projects/linux·blob assoc-array-fixes · refs

1531 lines · 39.4 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * linux/fs/file.c
4 *
5 * Copyright (C) 1998-1999, Stephen Tweedie and Bill Hawes
6 *
7 * Manage the dynamic fd arrays in the process files_struct.
8 */
9
10#include <linux/syscalls.h>
11#include <linux/export.h>
12#include <linux/fs.h>
13#include <linux/kernel.h>
14#include <linux/mm.h>
15#include <linux/sched/signal.h>
16#include <linux/slab.h>
17#include <linux/file.h>
18#include <linux/fdtable.h>
19#include <linux/bitops.h>
20#include <linux/spinlock.h>
21#include <linux/rcupdate.h>
22#include <linux/close_range.h>
23#include <linux/file_ref.h>
24#include <net/sock.h>
25#include <linux/init_task.h>
26
27#include "internal.h"
28
29static noinline bool __file_ref_put_badval(file_ref_t *ref, unsigned long cnt)
30{
31 /*
32 * If the reference count was already in the dead zone, then this
33 * put() operation is imbalanced. Warn, put the reference count back to
34 * DEAD and tell the caller to not deconstruct the object.
35 */
36 if (WARN_ONCE(cnt >= FILE_REF_RELEASED, "imbalanced put on file reference count")) {
37 atomic_long_set(&ref->refcnt, FILE_REF_DEAD);
38 return false;
39 }
40
41 /*
42 * This is a put() operation on a saturated refcount. Restore the
43 * mean saturation value and tell the caller to not deconstruct the
44 * object.
45 */
46 if (cnt > FILE_REF_MAXREF)
47 atomic_long_set(&ref->refcnt, FILE_REF_SATURATED);
48 return false;
49}
50
51/**
52 * __file_ref_put - Slowpath of file_ref_put()
53 * @ref: Pointer to the reference count
54 * @cnt: Current reference count
55 *
56 * Invoked when the reference count is outside of the valid zone.
57 *
58 * Return:
59 * True if this was the last reference with no future references
60 * possible. This signals the caller that it can safely schedule the
61 * object, which is protected by the reference counter, for
62 * deconstruction.
63 *
64 * False if there are still active references or the put() raced
65 * with a concurrent get()/put() pair. Caller is not allowed to
66 * deconstruct the protected object.
67 */
68bool __file_ref_put(file_ref_t *ref, unsigned long cnt)
69{
70 /* Did this drop the last reference? */
71 if (likely(cnt == FILE_REF_NOREF)) {
72 /*
73 * Carefully try to set the reference count to FILE_REF_DEAD.
74 *
75 * This can fail if a concurrent get() operation has
76 * elevated it again or the corresponding put() even marked
77 * it dead already. Both are valid situations and do not
78 * require a retry. If this fails the caller is not
79 * allowed to deconstruct the object.
80 */
81 if (!atomic_long_try_cmpxchg_release(&ref->refcnt, &cnt, FILE_REF_DEAD))
82 return false;
83
84 /*
85 * The caller can safely schedule the object for
86 * deconstruction. Provide acquire ordering.
87 */
88 smp_acquire__after_ctrl_dep();
89 return true;
90 }
91
92 return __file_ref_put_badval(ref, cnt);
93}
94EXPORT_SYMBOL_GPL(__file_ref_put);
95
96unsigned int sysctl_nr_open __read_mostly = 1024*1024;
97unsigned int sysctl_nr_open_min = BITS_PER_LONG;
98/* our min() is unusable in constant expressions ;-/ */
99#define __const_min(x, y) ((x) < (y) ? (x) : (y))
100unsigned int sysctl_nr_open_max =
101 __const_min(INT_MAX, ~(size_t)0/sizeof(void *)) & -BITS_PER_LONG;
102
103static void __free_fdtable(struct fdtable *fdt)
104{
105 kvfree(fdt->fd);
106 kvfree(fdt->open_fds);
107 kfree(fdt);
108}
109
110static void free_fdtable_rcu(struct rcu_head *rcu)
111{
112 __free_fdtable(container_of(rcu, struct fdtable, rcu));
113}
114
115#define BITBIT_NR(nr) BITS_TO_LONGS(BITS_TO_LONGS(nr))
116#define BITBIT_SIZE(nr) (BITBIT_NR(nr) * sizeof(long))
117
118#define fdt_words(fdt) ((fdt)->max_fds / BITS_PER_LONG) // words in ->open_fds
119/*
120 * Copy 'count' fd bits from the old table to the new table and clear the extra
121 * space if any. This does not copy the file pointers. Called with the files
122 * spinlock held for write.
123 */
124static inline void copy_fd_bitmaps(struct fdtable *nfdt, struct fdtable *ofdt,
125 unsigned int copy_words)
126{
127 unsigned int nwords = fdt_words(nfdt);
128
129 bitmap_copy_and_extend(nfdt->open_fds, ofdt->open_fds,
130 copy_words * BITS_PER_LONG, nwords * BITS_PER_LONG);
131 bitmap_copy_and_extend(nfdt->close_on_exec, ofdt->close_on_exec,
132 copy_words * BITS_PER_LONG, nwords * BITS_PER_LONG);
133 bitmap_copy_and_extend(nfdt->full_fds_bits, ofdt->full_fds_bits,
134 copy_words, nwords);
135}
136
137/*
138 * Copy all file descriptors from the old table to the new, expanded table and
139 * clear the extra space. Called with the files spinlock held for write.
140 */
141static void copy_fdtable(struct fdtable *nfdt, struct fdtable *ofdt)
142{
143 size_t cpy, set;
144
145 BUG_ON(nfdt->max_fds < ofdt->max_fds);
146
147 cpy = ofdt->max_fds * sizeof(struct file *);
148 set = (nfdt->max_fds - ofdt->max_fds) * sizeof(struct file *);
149 memcpy(nfdt->fd, ofdt->fd, cpy);
150 memset((char *)nfdt->fd + cpy, 0, set);
151
152 copy_fd_bitmaps(nfdt, ofdt, fdt_words(ofdt));
153}
154
155/*
156 * Note how the fdtable bitmap allocations very much have to be a multiple of
157 * BITS_PER_LONG. This is not only because we walk those things in chunks of
158 * 'unsigned long' in some places, but simply because that is how the Linux
159 * kernel bitmaps are defined to work: they are not "bits in an array of bytes",
160 * they are very much "bits in an array of unsigned long".
161 */
162static struct fdtable *alloc_fdtable(unsigned int slots_wanted)
163{
164 struct fdtable *fdt;
165 unsigned int nr;
166 void *data;
167
168 /*
169 * Figure out how many fds we actually want to support in this fdtable.
170 * Allocation steps are keyed to the size of the fdarray, since it
171 * grows far faster than any of the other dynamic data. We try to fit
172 * the fdarray into comfortable page-tuned chunks: starting at 1024B
173 * and growing in powers of two from there on. Since we called only
174 * with slots_wanted > BITS_PER_LONG (embedded instance in files->fdtab
175 * already gives BITS_PER_LONG slots), the above boils down to
176 * 1. use the smallest power of two large enough to give us that many
177 * slots.
178 * 2. on 32bit skip 64 and 128 - the minimal capacity we want there is
179 * 256 slots (i.e. 1Kb fd array).
180 * 3. on 64bit don't skip anything, 1Kb fd array means 128 slots there
181 * and we are never going to be asked for 64 or less.
182 */
183 if (IS_ENABLED(CONFIG_32BIT) && slots_wanted < 256)
184 nr = 256;
185 else
186 nr = roundup_pow_of_two(slots_wanted);
187 /*
188 * Note that this can drive nr *below* what we had passed if sysctl_nr_open
189 * had been set lower between the check in expand_files() and here.
190 *
191 * We make sure that nr remains a multiple of BITS_PER_LONG - otherwise
192 * bitmaps handling below becomes unpleasant, to put it mildly...
193 */
194 if (unlikely(nr > sysctl_nr_open)) {
195 nr = round_down(sysctl_nr_open, BITS_PER_LONG);
196 if (nr < slots_wanted)
197 return ERR_PTR(-EMFILE);
198 }
199
200 /*
201 * Check if the allocation size would exceed INT_MAX. kvmalloc_array()
202 * and kvmalloc() will warn if the allocation size is greater than
203 * INT_MAX, as filp_cache objects are not __GFP_NOWARN.
204 *
205 * This can happen when sysctl_nr_open is set to a very high value and
206 * a process tries to use a file descriptor near that limit. For example,
207 * if sysctl_nr_open is set to 1073741816 (0x3ffffff8) - which is what
208 * systemd typically sets it to - then trying to use a file descriptor
209 * close to that value will require allocating a file descriptor table
210 * that exceeds 8GB in size.
211 */
212 if (unlikely(nr > INT_MAX / sizeof(struct file *)))
213 return ERR_PTR(-EMFILE);
214
215 fdt = kmalloc_obj(struct fdtable, GFP_KERNEL_ACCOUNT);
216 if (!fdt)
217 goto out;
218 fdt->max_fds = nr;
219 data = kvmalloc_objs(struct file *, nr, GFP_KERNEL_ACCOUNT);
220 if (!data)
221 goto out_fdt;
222 fdt->fd = data;
223
224 data = kvmalloc(max_t(size_t,
225 2 * nr / BITS_PER_BYTE + BITBIT_SIZE(nr), L1_CACHE_BYTES),
226 GFP_KERNEL_ACCOUNT);
227 if (!data)
228 goto out_arr;
229 fdt->open_fds = data;
230 data += nr / BITS_PER_BYTE;
231 fdt->close_on_exec = data;
232 data += nr / BITS_PER_BYTE;
233 fdt->full_fds_bits = data;
234
235 return fdt;
236
237out_arr:
238 kvfree(fdt->fd);
239out_fdt:
240 kfree(fdt);
241out:
242 return ERR_PTR(-ENOMEM);
243}
244
245/*
246 * Expand the file descriptor table.
247 * This function will allocate a new fdtable and both fd array and fdset, of
248 * the given size.
249 * Return <0 error code on error; 0 on successful completion.
250 * The files->file_lock should be held on entry, and will be held on exit.
251 */
252static int expand_fdtable(struct files_struct *files, unsigned int nr)
253 __releases(files->file_lock)
254 __acquires(files->file_lock)
255{
256 struct fdtable *new_fdt, *cur_fdt;
257
258 spin_unlock(&files->file_lock);
259 new_fdt = alloc_fdtable(nr + 1);
260
261 /* make sure all fd_install() have seen resize_in_progress
262 * or have finished their rcu_read_lock_sched() section.
263 */
264 if (atomic_read(&files->count) > 1)
265 synchronize_rcu();
266
267 spin_lock(&files->file_lock);
268 if (IS_ERR(new_fdt))
269 return PTR_ERR(new_fdt);
270 cur_fdt = files_fdtable(files);
271 BUG_ON(nr < cur_fdt->max_fds);
272 copy_fdtable(new_fdt, cur_fdt);
273 rcu_assign_pointer(files->fdt, new_fdt);
274 if (cur_fdt != &files->fdtab)
275 call_rcu(&cur_fdt->rcu, free_fdtable_rcu);
276 /* coupled with smp_rmb() in fd_install() */
277 smp_wmb();
278 return 0;
279}
280
281/*
282 * Expand files.
283 * This function will expand the file structures, if the requested size exceeds
284 * the current capacity and there is room for expansion.
285 * Return <0 error code on error; 0 on success.
286 * The files->file_lock should be held on entry, and will be held on exit.
287 */
288static int expand_files(struct files_struct *files, unsigned int nr)
289 __releases(files->file_lock)
290 __acquires(files->file_lock)
291{
292 struct fdtable *fdt;
293 int error;
294
295repeat:
296 fdt = files_fdtable(files);
297
298 /* Do we need to expand? */
299 if (nr < fdt->max_fds)
300 return 0;
301
302 if (unlikely(files->resize_in_progress)) {
303 spin_unlock(&files->file_lock);
304 wait_event(files->resize_wait, !files->resize_in_progress);
305 spin_lock(&files->file_lock);
306 goto repeat;
307 }
308
309 /* Can we expand? */
310 if (unlikely(nr >= sysctl_nr_open))
311 return -EMFILE;
312
313 /* All good, so we try */
314 files->resize_in_progress = true;
315 error = expand_fdtable(files, nr);
316 files->resize_in_progress = false;
317
318 wake_up_all(&files->resize_wait);
319 return error;
320}
321
322static inline void __set_close_on_exec(unsigned int fd, struct fdtable *fdt,
323 bool set)
324{
325 if (set) {
326 __set_bit(fd, fdt->close_on_exec);
327 } else {
328 if (test_bit(fd, fdt->close_on_exec))
329 __clear_bit(fd, fdt->close_on_exec);
330 }
331}
332
333static inline void __set_open_fd(unsigned int fd, struct fdtable *fdt, bool set)
334{
335 __set_bit(fd, fdt->open_fds);
336 __set_close_on_exec(fd, fdt, set);
337 fd /= BITS_PER_LONG;
338 if (!~fdt->open_fds[fd])
339 __set_bit(fd, fdt->full_fds_bits);
340}
341
342static inline void __clear_open_fd(unsigned int fd, struct fdtable *fdt)
343{
344 __clear_bit(fd, fdt->open_fds);
345 fd /= BITS_PER_LONG;
346 if (test_bit(fd, fdt->full_fds_bits))
347 __clear_bit(fd, fdt->full_fds_bits);
348}
349
350static inline bool fd_is_open(unsigned int fd, const struct fdtable *fdt)
351{
352 return test_bit(fd, fdt->open_fds);
353}
354
355/*
356 * Note that a sane fdtable size always has to be a multiple of
357 * BITS_PER_LONG, since we have bitmaps that are sized by this.
358 *
359 * punch_hole is optional - when close_range() is asked to unshare
360 * and close, we don't need to copy descriptors in that range, so
361 * a smaller cloned descriptor table might suffice if the last
362 * currently opened descriptor falls into that range.
363 */
364static unsigned int sane_fdtable_size(struct fdtable *fdt, struct fd_range *punch_hole)
365{
366 unsigned int last = find_last_bit(fdt->open_fds, fdt->max_fds);
367
368 if (last == fdt->max_fds)
369 return NR_OPEN_DEFAULT;
370 if (punch_hole && punch_hole->to >= last && punch_hole->from <= last) {
371 last = find_last_bit(fdt->open_fds, punch_hole->from);
372 if (last == punch_hole->from)
373 return NR_OPEN_DEFAULT;
374 }
375 return ALIGN(last + 1, BITS_PER_LONG);
376}
377
378/*
379 * Allocate a new descriptor table and copy contents from the passed in
380 * instance. Returns a pointer to cloned table on success, ERR_PTR()
381 * on failure. For 'punch_hole' see sane_fdtable_size().
382 */
383struct files_struct *dup_fd(struct files_struct *oldf, struct fd_range *punch_hole)
384{
385 struct files_struct *newf;
386 struct file **old_fds, **new_fds;
387 unsigned int open_files, i;
388 struct fdtable *old_fdt, *new_fdt;
389
390 newf = kmem_cache_alloc(files_cachep, GFP_KERNEL);
391 if (!newf)
392 return ERR_PTR(-ENOMEM);
393
394 atomic_set(&newf->count, 1);
395
396 spin_lock_init(&newf->file_lock);
397 newf->resize_in_progress = false;
398 init_waitqueue_head(&newf->resize_wait);
399 newf->next_fd = 0;
400 new_fdt = &newf->fdtab;
401 new_fdt->max_fds = NR_OPEN_DEFAULT;
402 new_fdt->close_on_exec = newf->close_on_exec_init;
403 new_fdt->open_fds = newf->open_fds_init;
404 new_fdt->full_fds_bits = newf->full_fds_bits_init;
405 new_fdt->fd = &newf->fd_array[0];
406
407 spin_lock(&oldf->file_lock);
408 old_fdt = files_fdtable(oldf);
409 open_files = sane_fdtable_size(old_fdt, punch_hole);
410
411 /*
412 * Check whether we need to allocate a larger fd array and fd set.
413 */
414 while (unlikely(open_files > new_fdt->max_fds)) {
415 spin_unlock(&oldf->file_lock);
416
417 if (new_fdt != &newf->fdtab)
418 __free_fdtable(new_fdt);
419
420 new_fdt = alloc_fdtable(open_files);
421 if (IS_ERR(new_fdt)) {
422 kmem_cache_free(files_cachep, newf);
423 return ERR_CAST(new_fdt);
424 }
425
426 /*
427 * Reacquire the oldf lock and a pointer to its fd table
428 * who knows it may have a new bigger fd table. We need
429 * the latest pointer.
430 */
431 spin_lock(&oldf->file_lock);
432 old_fdt = files_fdtable(oldf);
433 open_files = sane_fdtable_size(old_fdt, punch_hole);
434 }
435
436 copy_fd_bitmaps(new_fdt, old_fdt, open_files / BITS_PER_LONG);
437
438 old_fds = old_fdt->fd;
439 new_fds = new_fdt->fd;
440
441 /*
442 * We may be racing against fd allocation from other threads using this
443 * files_struct, despite holding ->file_lock.
444 *
445 * alloc_fd() might have already claimed a slot, while fd_install()
446 * did not populate it yet. Note the latter operates locklessly, so
447 * the file can show up as we are walking the array below.
448 *
449 * At the same time we know no files will disappear as all other
450 * operations take the lock.
451 *
452 * Instead of trying to placate userspace racing with itself, we
453 * ref the file if we see it and mark the fd slot as unused otherwise.
454 */
455 for (i = open_files; i != 0; i--) {
456 struct file *f = rcu_dereference_raw(*old_fds++);
457 if (f) {
458 get_file(f);
459 } else {
460 __clear_open_fd(open_files - i, new_fdt);
461 }
462 rcu_assign_pointer(*new_fds++, f);
463 }
464 spin_unlock(&oldf->file_lock);
465
466 /* clear the remainder */
467 memset(new_fds, 0, (new_fdt->max_fds - open_files) * sizeof(struct file *));
468
469 rcu_assign_pointer(newf->fdt, new_fdt);
470
471 return newf;
472}
473
474static struct fdtable *close_files(struct files_struct * files)
475{
476 /*
477 * It is safe to dereference the fd table without RCU or
478 * ->file_lock because this is the last reference to the
479 * files structure.
480 */
481 struct fdtable *fdt = rcu_dereference_raw(files->fdt);
482 unsigned int i, j = 0;
483
484 for (;;) {
485 unsigned long set;
486 i = j * BITS_PER_LONG;
487 if (i >= fdt->max_fds)
488 break;
489 set = fdt->open_fds[j++];
490 while (set) {
491 if (set & 1) {
492 struct file *file = fdt->fd[i];
493 if (file) {
494 filp_close(file, files);
495 cond_resched();
496 }
497 }
498 i++;
499 set >>= 1;
500 }
501 }
502
503 return fdt;
504}
505
506void put_files_struct(struct files_struct *files)
507{
508 if (atomic_dec_and_test(&files->count)) {
509 struct fdtable *fdt = close_files(files);
510
511 /* free the arrays if they are not embedded */
512 if (fdt != &files->fdtab)
513 __free_fdtable(fdt);
514 kmem_cache_free(files_cachep, files);
515 }
516}
517
518void exit_files(struct task_struct *tsk)
519{
520 struct files_struct * files = tsk->files;
521
522 if (files) {
523 task_lock(tsk);
524 tsk->files = NULL;
525 task_unlock(tsk);
526 put_files_struct(files);
527 }
528}
529
530struct files_struct init_files = {
531 .count = ATOMIC_INIT(1),
532 .fdt = &init_files.fdtab,
533 .fdtab = {
534 .max_fds = NR_OPEN_DEFAULT,
535 .fd = &init_files.fd_array[0],
536 .close_on_exec = init_files.close_on_exec_init,
537 .open_fds = init_files.open_fds_init,
538 .full_fds_bits = init_files.full_fds_bits_init,
539 },
540 .file_lock = __SPIN_LOCK_UNLOCKED(init_files.file_lock),
541 .resize_wait = __WAIT_QUEUE_HEAD_INITIALIZER(init_files.resize_wait),
542};
543
544static unsigned int find_next_fd(struct fdtable *fdt, unsigned int start)
545{
546 unsigned int maxfd = fdt->max_fds; /* always multiple of BITS_PER_LONG */
547 unsigned int max_fds_words = maxfd / BITS_PER_LONG;
548 unsigned int fds_word_idx = start / BITS_PER_LONG;
549 unsigned int bit;
550
551 /*
552 * Try to avoid looking at the second level bitmap
553 */
554 bit = find_next_zero_bit(&fdt->open_fds[fds_word_idx], BITS_PER_LONG,
555 start & (BITS_PER_LONG - 1));
556 if (bit < BITS_PER_LONG)
557 return bit + (fds_word_idx * BITS_PER_LONG);
558
559 bit = BITS_PER_LONG *
560 find_next_zero_bit(fdt->full_fds_bits, max_fds_words, fds_word_idx + 1);
561 if (bit >= maxfd)
562 return maxfd;
563 return find_next_zero_bit(fdt->open_fds, maxfd, bit);
564}
565
566/*
567 * allocate a file descriptor, mark it busy.
568 */
569static int alloc_fd(unsigned start, unsigned end, unsigned flags)
570{
571 struct files_struct *files = current->files;
572 unsigned int fd;
573 int error;
574 struct fdtable *fdt;
575
576 spin_lock(&files->file_lock);
577repeat:
578 fdt = files_fdtable(files);
579 fd = start;
580 if (fd < files->next_fd)
581 fd = files->next_fd;
582
583 if (likely(fd < fdt->max_fds))
584 fd = find_next_fd(fdt, fd);
585
586 /*
587 * N.B. For clone tasks sharing a files structure, this test
588 * will limit the total number of files that can be opened.
589 */
590 error = -EMFILE;
591 if (unlikely(fd >= end))
592 goto out;
593
594 if (unlikely(fd >= fdt->max_fds)) {
595 error = expand_files(files, fd);
596 if (error < 0)
597 goto out;
598
599 goto repeat;
600 }
601
602 if (start <= files->next_fd)
603 files->next_fd = fd + 1;
604
605 __set_open_fd(fd, fdt, flags & O_CLOEXEC);
606 error = fd;
607 VFS_BUG_ON(rcu_access_pointer(fdt->fd[fd]) != NULL);
608
609out:
610 spin_unlock(&files->file_lock);
611 return error;
612}
613
614int __get_unused_fd_flags(unsigned flags, unsigned long nofile)
615{
616 return alloc_fd(0, nofile, flags);
617}
618
619int get_unused_fd_flags(unsigned flags)
620{
621 return __get_unused_fd_flags(flags, rlimit(RLIMIT_NOFILE));
622}
623EXPORT_SYMBOL(get_unused_fd_flags);
624
625static void __put_unused_fd(struct files_struct *files, unsigned int fd)
626{
627 struct fdtable *fdt = files_fdtable(files);
628 __clear_open_fd(fd, fdt);
629 if (fd < files->next_fd)
630 files->next_fd = fd;
631}
632
633void put_unused_fd(unsigned int fd)
634{
635 struct files_struct *files = current->files;
636 spin_lock(&files->file_lock);
637 __put_unused_fd(files, fd);
638 spin_unlock(&files->file_lock);
639}
640
641EXPORT_SYMBOL(put_unused_fd);
642
643/*
644 * Install a file pointer in the fd array while it is being resized.
645 *
646 * We need to make sure our update to the array does not get lost as the resizing
647 * thread can be copying the content as we modify it.
648 *
649 * We have two ways to do it:
650 * - go off CPU waiting for resize_in_progress to clear
651 * - take the spin lock
652 *
653 * The latter is trivial to implement and saves us from having to might_sleep()
654 * for debugging purposes.
655 *
656 * This is moved out of line from fd_install() to convince gcc to optimize that
657 * routine better.
658 */
659static void noinline fd_install_slowpath(unsigned int fd, struct file *file)
660{
661 struct files_struct *files = current->files;
662 struct fdtable *fdt;
663
664 spin_lock(&files->file_lock);
665 fdt = files_fdtable(files);
666 VFS_BUG_ON(rcu_access_pointer(fdt->fd[fd]) != NULL);
667 rcu_assign_pointer(fdt->fd[fd], file);
668 spin_unlock(&files->file_lock);
669}
670
671/**
672 * fd_install - install a file pointer in the fd array
673 * @fd: file descriptor to install the file in
674 * @file: the file to install
675 *
676 * This consumes the "file" refcount, so callers should treat it
677 * as if they had called fput(file).
678 */
679void fd_install(unsigned int fd, struct file *file)
680{
681 struct files_struct *files = current->files;
682 struct fdtable *fdt;
683
684 if (WARN_ON_ONCE(unlikely(file->f_mode & FMODE_BACKING)))
685 return;
686
687 rcu_read_lock_sched();
688 if (unlikely(files->resize_in_progress)) {
689 rcu_read_unlock_sched();
690 fd_install_slowpath(fd, file);
691 return;
692 }
693 /* coupled with smp_wmb() in expand_fdtable() */
694 smp_rmb();
695 fdt = rcu_dereference_sched(files->fdt);
696 VFS_BUG_ON(rcu_access_pointer(fdt->fd[fd]) != NULL);
697 rcu_assign_pointer(fdt->fd[fd], file);
698 rcu_read_unlock_sched();
699}
700
701EXPORT_SYMBOL(fd_install);
702
703/**
704 * file_close_fd_locked - return file associated with fd
705 * @files: file struct to retrieve file from
706 * @fd: file descriptor to retrieve file for
707 *
708 * Doesn't take a separate reference count.
709 *
710 * Context: files_lock must be held.
711 *
712 * Returns: The file associated with @fd (NULL if @fd is not open)
713 */
714struct file *file_close_fd_locked(struct files_struct *files, unsigned fd)
715{
716 struct fdtable *fdt = files_fdtable(files);
717 struct file *file;
718
719 lockdep_assert_held(&files->file_lock);
720
721 if (fd >= fdt->max_fds)
722 return NULL;
723
724 fd = array_index_nospec(fd, fdt->max_fds);
725 file = rcu_dereference_raw(fdt->fd[fd]);
726 if (file) {
727 rcu_assign_pointer(fdt->fd[fd], NULL);
728 __put_unused_fd(files, fd);
729 }
730 return file;
731}
732
733int close_fd(unsigned fd)
734{
735 struct files_struct *files = current->files;
736 struct file *file;
737
738 spin_lock(&files->file_lock);
739 file = file_close_fd_locked(files, fd);
740 spin_unlock(&files->file_lock);
741 if (!file)
742 return -EBADF;
743
744 return filp_close(file, files);
745}
746EXPORT_SYMBOL(close_fd);
747
748/**
749 * last_fd - return last valid index into fd table
750 * @fdt: File descriptor table.
751 *
752 * Context: Either rcu read lock or files_lock must be held.
753 *
754 * Returns: Last valid index into fdtable.
755 */
756static inline unsigned last_fd(struct fdtable *fdt)
757{
758 return fdt->max_fds - 1;
759}
760
761static inline void __range_cloexec(struct files_struct *cur_fds,
762 unsigned int fd, unsigned int max_fd)
763{
764 struct fdtable *fdt;
765
766 /* make sure we're using the correct maximum value */
767 spin_lock(&cur_fds->file_lock);
768 fdt = files_fdtable(cur_fds);
769 max_fd = min(last_fd(fdt), max_fd);
770 if (fd <= max_fd)
771 bitmap_set(fdt->close_on_exec, fd, max_fd - fd + 1);
772 spin_unlock(&cur_fds->file_lock);
773}
774
775static inline void __range_close(struct files_struct *files, unsigned int fd,
776 unsigned int max_fd)
777{
778 struct file *file;
779 struct fdtable *fdt;
780 unsigned n;
781
782 spin_lock(&files->file_lock);
783 fdt = files_fdtable(files);
784 n = last_fd(fdt);
785 max_fd = min(max_fd, n);
786
787 for (fd = find_next_bit(fdt->open_fds, max_fd + 1, fd);
788 fd <= max_fd;
789 fd = find_next_bit(fdt->open_fds, max_fd + 1, fd + 1)) {
790 file = file_close_fd_locked(files, fd);
791 if (file) {
792 spin_unlock(&files->file_lock);
793 filp_close(file, files);
794 cond_resched();
795 spin_lock(&files->file_lock);
796 fdt = files_fdtable(files);
797 } else if (need_resched()) {
798 spin_unlock(&files->file_lock);
799 cond_resched();
800 spin_lock(&files->file_lock);
801 fdt = files_fdtable(files);
802 }
803 }
804 spin_unlock(&files->file_lock);
805}
806
807/**
808 * sys_close_range() - Close all file descriptors in a given range.
809 *
810 * @fd: starting file descriptor to close
811 * @max_fd: last file descriptor to close
812 * @flags: CLOSE_RANGE flags.
813 *
814 * This closes a range of file descriptors. All file descriptors
815 * from @fd up to and including @max_fd are closed.
816 * Currently, errors to close a given file descriptor are ignored.
817 */
818SYSCALL_DEFINE3(close_range, unsigned int, fd, unsigned int, max_fd,
819 unsigned int, flags)
820{
821 struct task_struct *me = current;
822 struct files_struct *cur_fds = me->files, *fds = NULL;
823
824 if (flags & ~(CLOSE_RANGE_UNSHARE | CLOSE_RANGE_CLOEXEC))
825 return -EINVAL;
826
827 if (fd > max_fd)
828 return -EINVAL;
829
830 if ((flags & CLOSE_RANGE_UNSHARE) && atomic_read(&cur_fds->count) > 1) {
831 struct fd_range range = {fd, max_fd}, *punch_hole = &range;
832
833 /*
834 * If the caller requested all fds to be made cloexec we always
835 * copy all of the file descriptors since they still want to
836 * use them.
837 */
838 if (flags & CLOSE_RANGE_CLOEXEC)
839 punch_hole = NULL;
840
841 fds = dup_fd(cur_fds, punch_hole);
842 if (IS_ERR(fds))
843 return PTR_ERR(fds);
844 /*
845 * We used to share our file descriptor table, and have now
846 * created a private one, make sure we're using it below.
847 */
848 swap(cur_fds, fds);
849 }
850
851 if (flags & CLOSE_RANGE_CLOEXEC)
852 __range_cloexec(cur_fds, fd, max_fd);
853 else
854 __range_close(cur_fds, fd, max_fd);
855
856 if (fds) {
857 /*
858 * We're done closing the files we were supposed to. Time to install
859 * the new file descriptor table and drop the old one.
860 */
861 task_lock(me);
862 me->files = cur_fds;
863 task_unlock(me);
864 put_files_struct(fds);
865 }
866
867 return 0;
868}
869
870/**
871 * file_close_fd - return file associated with fd
872 * @fd: file descriptor to retrieve file for
873 *
874 * Doesn't take a separate reference count.
875 *
876 * Returns: The file associated with @fd (NULL if @fd is not open)
877 */
878struct file *file_close_fd(unsigned int fd)
879{
880 struct files_struct *files = current->files;
881 struct file *file;
882
883 spin_lock(&files->file_lock);
884 file = file_close_fd_locked(files, fd);
885 spin_unlock(&files->file_lock);
886
887 return file;
888}
889
890void do_close_on_exec(struct files_struct *files)
891{
892 unsigned i;
893 struct fdtable *fdt;
894
895 /* exec unshares first */
896 spin_lock(&files->file_lock);
897 for (i = 0; ; i++) {
898 unsigned long set;
899 unsigned fd = i * BITS_PER_LONG;
900 fdt = files_fdtable(files);
901 if (fd >= fdt->max_fds)
902 break;
903 set = fdt->close_on_exec[i];
904 if (!set)
905 continue;
906 fdt->close_on_exec[i] = 0;
907 for ( ; set ; fd++, set >>= 1) {
908 struct file *file;
909 if (!(set & 1))
910 continue;
911 file = fdt->fd[fd];
912 if (!file)
913 continue;
914 rcu_assign_pointer(fdt->fd[fd], NULL);
915 __put_unused_fd(files, fd);
916 spin_unlock(&files->file_lock);
917 filp_close(file, files);
918 cond_resched();
919 spin_lock(&files->file_lock);
920 }
921
922 }
923 spin_unlock(&files->file_lock);
924}
925
926static struct file *__get_file_rcu(struct file __rcu **f)
927{
928 struct file __rcu *file;
929 struct file __rcu *file_reloaded;
930 struct file __rcu *file_reloaded_cmp;
931
932 file = rcu_dereference_raw(*f);
933 if (!file)
934 return NULL;
935
936 if (unlikely(!file_ref_get(&file->f_ref)))
937 return ERR_PTR(-EAGAIN);
938
939 file_reloaded = rcu_dereference_raw(*f);
940
941 /*
942 * Ensure that all accesses have a dependency on the load from
943 * rcu_dereference_raw() above so we get correct ordering
944 * between reuse/allocation and the pointer check below.
945 */
946 file_reloaded_cmp = file_reloaded;
947 OPTIMIZER_HIDE_VAR(file_reloaded_cmp);
948
949 /*
950 * file_ref_get() above provided a full memory barrier when we
951 * acquired a reference.
952 *
953 * This is paired with the write barrier from assigning to the
954 * __rcu protected file pointer so that if that pointer still
955 * matches the current file, we know we have successfully
956 * acquired a reference to the right file.
957 *
958 * If the pointers don't match the file has been reallocated by
959 * SLAB_TYPESAFE_BY_RCU.
960 */
961 if (file == file_reloaded_cmp)
962 return file_reloaded;
963
964 fput(file);
965 return ERR_PTR(-EAGAIN);
966}
967
968/**
969 * get_file_rcu - try go get a reference to a file under rcu
970 * @f: the file to get a reference on
971 *
972 * This function tries to get a reference on @f carefully verifying that
973 * @f hasn't been reused.
974 *
975 * This function should rarely have to be used and only by users who
976 * understand the implications of SLAB_TYPESAFE_BY_RCU. Try to avoid it.
977 *
978 * Return: Returns @f with the reference count increased or NULL.
979 */
980struct file *get_file_rcu(struct file __rcu **f)
981{
982 for (;;) {
983 struct file __rcu *file;
984
985 file = __get_file_rcu(f);
986 if (!IS_ERR(file))
987 return file;
988 }
989}
990EXPORT_SYMBOL_GPL(get_file_rcu);
991
992/**
993 * get_file_active - try go get a reference to a file
994 * @f: the file to get a reference on
995 *
996 * In contast to get_file_rcu() the pointer itself isn't part of the
997 * reference counting.
998 *
999 * This function should rarely have to be used and only by users who
1000 * understand the implications of SLAB_TYPESAFE_BY_RCU. Try to avoid it.
1001 *
1002 * Return: Returns @f with the reference count increased or NULL.
1003 */
1004struct file *get_file_active(struct file **f)
1005{
1006 struct file __rcu *file;
1007
1008 rcu_read_lock();
1009 file = __get_file_rcu(f);
1010 rcu_read_unlock();
1011 if (IS_ERR(file))
1012 file = NULL;
1013 return file;
1014}
1015EXPORT_SYMBOL_GPL(get_file_active);
1016
1017static inline struct file *__fget_files_rcu(struct files_struct *files,
1018 unsigned int fd, fmode_t mask)
1019{
1020 for (;;) {
1021 struct file *file;
1022 struct fdtable *fdt = rcu_dereference_raw(files->fdt);
1023 struct file __rcu **fdentry;
1024 unsigned long nospec_mask;
1025
1026 /* Mask is a 0 for invalid fd's, ~0 for valid ones */
1027 nospec_mask = array_index_mask_nospec(fd, fdt->max_fds);
1028
1029 /*
1030 * fdentry points to the 'fd' offset, or fdt->fd[0].
1031 * Loading from fdt->fd[0] is always safe, because the
1032 * array always exists.
1033 */
1034 fdentry = fdt->fd + (fd & nospec_mask);
1035
1036 /* Do the load, then mask any invalid result */
1037 file = rcu_dereference_raw(*fdentry);
1038 file = (void *)(nospec_mask & (unsigned long)file);
1039 if (unlikely(!file))
1040 return NULL;
1041
1042 /*
1043 * Ok, we have a file pointer that was valid at
1044 * some point, but it might have become stale since.
1045 *
1046 * We need to confirm it by incrementing the refcount
1047 * and then check the lookup again.
1048 *
1049 * file_ref_get() gives us a full memory barrier. We
1050 * only really need an 'acquire' one to protect the
1051 * loads below, but we don't have that.
1052 */
1053 if (unlikely(!file_ref_get(&file->f_ref)))
1054 continue;
1055
1056 /*
1057 * Such a race can take two forms:
1058 *
1059 * (a) the file ref already went down to zero and the
1060 * file hasn't been reused yet or the file count
1061 * isn't zero but the file has already been reused.
1062 *
1063 * (b) the file table entry has changed under us.
1064 * Note that we don't need to re-check the 'fdt->fd'
1065 * pointer having changed, because it always goes
1066 * hand-in-hand with 'fdt'.
1067 *
1068 * If so, we need to put our ref and try again.
1069 */
1070 if (unlikely(file != rcu_dereference_raw(*fdentry)) ||
1071 unlikely(rcu_dereference_raw(files->fdt) != fdt)) {
1072 fput(file);
1073 continue;
1074 }
1075
1076 /*
1077 * This isn't the file we're looking for or we're not
1078 * allowed to get a reference to it.
1079 */
1080 if (unlikely(file->f_mode & mask)) {
1081 fput(file);
1082 return NULL;
1083 }
1084
1085 /*
1086 * Ok, we have a ref to the file, and checked that it
1087 * still exists.
1088 */
1089 return file;
1090 }
1091}
1092
1093static struct file *__fget_files(struct files_struct *files, unsigned int fd,
1094 fmode_t mask)
1095{
1096 struct file *file;
1097
1098 rcu_read_lock();
1099 file = __fget_files_rcu(files, fd, mask);
1100 rcu_read_unlock();
1101
1102 return file;
1103}
1104
1105static inline struct file *__fget(unsigned int fd, fmode_t mask)
1106{
1107 return __fget_files(current->files, fd, mask);
1108}
1109
1110struct file *fget(unsigned int fd)
1111{
1112 return __fget(fd, FMODE_PATH);
1113}
1114EXPORT_SYMBOL(fget);
1115
1116struct file *fget_raw(unsigned int fd)
1117{
1118 return __fget(fd, 0);
1119}
1120EXPORT_SYMBOL(fget_raw);
1121
1122struct file *fget_task(struct task_struct *task, unsigned int fd)
1123{
1124 struct file *file = NULL;
1125
1126 task_lock(task);
1127 if (task->files)
1128 file = __fget_files(task->files, fd, 0);
1129 task_unlock(task);
1130
1131 return file;
1132}
1133
1134struct file *fget_task_next(struct task_struct *task, unsigned int *ret_fd)
1135{
1136 struct files_struct *files;
1137 unsigned int fd = *ret_fd;
1138 struct file *file = NULL;
1139
1140 task_lock(task);
1141 files = task->files;
1142 if (files) {
1143 rcu_read_lock();
1144 for (; fd < files_fdtable(files)->max_fds; fd++) {
1145 file = __fget_files_rcu(files, fd, 0);
1146 if (file)
1147 break;
1148 }
1149 rcu_read_unlock();
1150 }
1151 task_unlock(task);
1152 *ret_fd = fd;
1153 return file;
1154}
1155EXPORT_SYMBOL(fget_task_next);
1156
1157/*
1158 * Lightweight file lookup - no refcnt increment if fd table isn't shared.
1159 *
1160 * You can use this instead of fget if you satisfy all of the following
1161 * conditions:
1162 * 1) You must call fput_light before exiting the syscall and returning control
1163 * to userspace (i.e. you cannot remember the returned struct file * after
1164 * returning to userspace).
1165 * 2) You must not call filp_close on the returned struct file * in between
1166 * calls to fget_light and fput_light.
1167 * 3) You must not clone the current task in between the calls to fget_light
1168 * and fput_light.
1169 *
1170 * The fput_needed flag returned by fget_light should be passed to the
1171 * corresponding fput_light.
1172 *
1173 * (As an exception to rule 2, you can call filp_close between fget_light and
1174 * fput_light provided that you capture a real refcount with get_file before
1175 * the call to filp_close, and ensure that this real refcount is fput *after*
1176 * the fput_light call.)
1177 *
1178 * See also the documentation in rust/kernel/file.rs.
1179 */
1180static inline struct fd __fget_light(unsigned int fd, fmode_t mask)
1181{
1182 struct files_struct *files = current->files;
1183 struct file *file;
1184
1185 /*
1186 * If another thread is concurrently calling close_fd() followed
1187 * by put_files_struct(), we must not observe the old table
1188 * entry combined with the new refcount - otherwise we could
1189 * return a file that is concurrently being freed.
1190 *
1191 * atomic_read_acquire() pairs with atomic_dec_and_test() in
1192 * put_files_struct().
1193 */
1194 if (likely(atomic_read_acquire(&files->count) == 1)) {
1195 file = files_lookup_fd_raw(files, fd);
1196 if (!file || unlikely(file->f_mode & mask))
1197 return EMPTY_FD;
1198 return BORROWED_FD(file);
1199 } else {
1200 file = __fget_files(files, fd, mask);
1201 if (!file)
1202 return EMPTY_FD;
1203 return CLONED_FD(file);
1204 }
1205}
1206struct fd fdget(unsigned int fd)
1207{
1208 return __fget_light(fd, FMODE_PATH);
1209}
1210EXPORT_SYMBOL(fdget);
1211
1212struct fd fdget_raw(unsigned int fd)
1213{
1214 return __fget_light(fd, 0);
1215}
1216
1217/*
1218 * Try to avoid f_pos locking. We only need it if the
1219 * file is marked for FMODE_ATOMIC_POS, and it can be
1220 * accessed multiple ways.
1221 *
1222 * Always do it for directories, because pidfd_getfd()
1223 * can make a file accessible even if it otherwise would
1224 * not be, and for directories this is a correctness
1225 * issue, not a "POSIX requirement".
1226 */
1227static inline bool file_needs_f_pos_lock(struct file *file)
1228{
1229 if (!(file->f_mode & FMODE_ATOMIC_POS))
1230 return false;
1231 if (__file_ref_read_raw(&file->f_ref) != FILE_REF_ONEREF)
1232 return true;
1233 if (file->f_op->iterate_shared)
1234 return true;
1235 return false;
1236}
1237
1238bool file_seek_cur_needs_f_lock(struct file *file)
1239{
1240 if (!(file->f_mode & FMODE_ATOMIC_POS) && !file->f_op->iterate_shared)
1241 return false;
1242
1243 /*
1244 * Note that we are not guaranteed to be called after fdget_pos() on
1245 * this file obj, in which case the caller is expected to provide the
1246 * appropriate locking.
1247 */
1248
1249 return true;
1250}
1251
1252struct fd fdget_pos(unsigned int fd)
1253{
1254 struct fd f = fdget(fd);
1255 struct file *file = fd_file(f);
1256
1257 if (likely(file) && file_needs_f_pos_lock(file)) {
1258 f.word |= FDPUT_POS_UNLOCK;
1259 mutex_lock(&file->f_pos_lock);
1260 }
1261 return f;
1262}
1263
1264void __f_unlock_pos(struct file *f)
1265{
1266 mutex_unlock(&f->f_pos_lock);
1267}
1268
1269/*
1270 * We only lock f_pos if we have threads or if the file might be
1271 * shared with another process. In both cases we'll have an elevated
1272 * file count (done either by fdget() or by fork()).
1273 */
1274
1275void set_close_on_exec(unsigned int fd, int flag)
1276{
1277 struct files_struct *files = current->files;
1278 spin_lock(&files->file_lock);
1279 __set_close_on_exec(fd, files_fdtable(files), flag);
1280 spin_unlock(&files->file_lock);
1281}
1282
1283bool get_close_on_exec(unsigned int fd)
1284{
1285 bool res;
1286 rcu_read_lock();
1287 res = close_on_exec(fd, current->files);
1288 rcu_read_unlock();
1289 return res;
1290}
1291
1292static int do_dup2(struct files_struct *files,
1293 struct file *file, unsigned fd, unsigned flags)
1294__releases(&files->file_lock)
1295{
1296 struct file *tofree;
1297 struct fdtable *fdt;
1298
1299 /*
1300 * dup2() is expected to close the file installed in the target fd slot
1301 * (if any). However, userspace hand-picking a fd may be racing against
1302 * its own threads which happened to allocate it in open() et al but did
1303 * not populate it yet.
1304 *
1305 * Broadly speaking we may be racing against the following:
1306 * fd = get_unused_fd_flags(); // fd slot reserved, ->fd[fd] == NULL
1307 * file = hard_work_goes_here();
1308 * fd_install(fd, file); // only now ->fd[fd] == file
1309 *
1310 * It is an invariant that a successfully allocated fd has a NULL entry
1311 * in the array until the matching fd_install().
1312 *
1313 * If we fit the window, we have the fd to populate, yet no target file
1314 * to close. Trying to ignore it and install our new file would violate
1315 * the invariant and make fd_install() overwrite our file.
1316 *
1317 * Things can be done(tm) to handle this. However, the issue does not
1318 * concern legitimate programs and we only need to make sure the kernel
1319 * does not trip over it.
1320 *
1321 * The simplest way out is to return an error if we find ourselves here.
1322 *
1323 * POSIX is silent on the issue, we return -EBUSY.
1324 */
1325 fdt = files_fdtable(files);
1326 fd = array_index_nospec(fd, fdt->max_fds);
1327 tofree = rcu_dereference_raw(fdt->fd[fd]);
1328 if (!tofree && fd_is_open(fd, fdt))
1329 goto Ebusy;
1330 get_file(file);
1331 rcu_assign_pointer(fdt->fd[fd], file);
1332 __set_open_fd(fd, fdt, flags & O_CLOEXEC);
1333 spin_unlock(&files->file_lock);
1334
1335 if (tofree)
1336 filp_close(tofree, files);
1337
1338 return fd;
1339
1340Ebusy:
1341 spin_unlock(&files->file_lock);
1342 return -EBUSY;
1343}
1344
1345int replace_fd(unsigned fd, struct file *file, unsigned flags)
1346{
1347 int err;
1348 struct files_struct *files = current->files;
1349
1350 if (!file)
1351 return close_fd(fd);
1352
1353 if (fd >= rlimit(RLIMIT_NOFILE))
1354 return -EBADF;
1355
1356 spin_lock(&files->file_lock);
1357 err = expand_files(files, fd);
1358 if (unlikely(err < 0))
1359 goto out_unlock;
1360 err = do_dup2(files, file, fd, flags);
1361 if (err < 0)
1362 return err;
1363 return 0;
1364
1365out_unlock:
1366 spin_unlock(&files->file_lock);
1367 return err;
1368}
1369
1370/**
1371 * receive_fd() - Install received file into file descriptor table
1372 * @file: struct file that was received from another process
1373 * @ufd: __user pointer to write new fd number to
1374 * @o_flags: the O_* flags to apply to the new fd entry
1375 *
1376 * Installs a received file into the file descriptor table, with appropriate
1377 * checks and count updates. Optionally writes the fd number to userspace, if
1378 * @ufd is non-NULL.
1379 *
1380 * This helper handles its own reference counting of the incoming
1381 * struct file.
1382 *
1383 * Returns newly install fd or -ve on error.
1384 */
1385int receive_fd(struct file *file, int __user *ufd, unsigned int o_flags)
1386{
1387 int error;
1388
1389 error = security_file_receive(file);
1390 if (error)
1391 return error;
1392
1393 FD_PREPARE(fdf, o_flags, file);
1394 if (fdf.err)
1395 return fdf.err;
1396 get_file(file);
1397
1398 if (ufd) {
1399 error = put_user(fd_prepare_fd(fdf), ufd);
1400 if (error)
1401 return error;
1402 }
1403
1404 __receive_sock(fd_prepare_file(fdf));
1405 return fd_publish(fdf);
1406}
1407EXPORT_SYMBOL_GPL(receive_fd);
1408
1409int receive_fd_replace(int new_fd, struct file *file, unsigned int o_flags)
1410{
1411 int error;
1412
1413 error = security_file_receive(file);
1414 if (error)
1415 return error;
1416 error = replace_fd(new_fd, file, o_flags);
1417 if (error)
1418 return error;
1419 __receive_sock(file);
1420 return new_fd;
1421}
1422
1423static int ksys_dup3(unsigned int oldfd, unsigned int newfd, int flags)
1424{
1425 int err = -EBADF;
1426 struct file *file;
1427 struct files_struct *files = current->files;
1428
1429 if ((flags & ~O_CLOEXEC) != 0)
1430 return -EINVAL;
1431
1432 if (unlikely(oldfd == newfd))
1433 return -EINVAL;
1434
1435 if (newfd >= rlimit(RLIMIT_NOFILE))
1436 return -EBADF;
1437
1438 spin_lock(&files->file_lock);
1439 err = expand_files(files, newfd);
1440 file = files_lookup_fd_locked(files, oldfd);
1441 if (unlikely(!file))
1442 goto Ebadf;
1443 if (unlikely(err < 0)) {
1444 if (err == -EMFILE)
1445 goto Ebadf;
1446 goto out_unlock;
1447 }
1448 return do_dup2(files, file, newfd, flags);
1449
1450Ebadf:
1451 err = -EBADF;
1452out_unlock:
1453 spin_unlock(&files->file_lock);
1454 return err;
1455}
1456
1457SYSCALL_DEFINE3(dup3, unsigned int, oldfd, unsigned int, newfd, int, flags)
1458{
1459 return ksys_dup3(oldfd, newfd, flags);
1460}
1461
1462SYSCALL_DEFINE2(dup2, unsigned int, oldfd, unsigned int, newfd)
1463{
1464 if (unlikely(newfd == oldfd)) { /* corner case */
1465 struct files_struct *files = current->files;
1466 struct file *f;
1467 int retval = oldfd;
1468
1469 rcu_read_lock();
1470 f = __fget_files_rcu(files, oldfd, 0);
1471 if (!f)
1472 retval = -EBADF;
1473 rcu_read_unlock();
1474 if (f)
1475 fput(f);
1476 return retval;
1477 }
1478 return ksys_dup3(oldfd, newfd, 0);
1479}
1480
1481SYSCALL_DEFINE1(dup, unsigned int, fildes)
1482{
1483 int ret = -EBADF;
1484 struct file *file = fget_raw(fildes);
1485
1486 if (file) {
1487 ret = get_unused_fd_flags(0);
1488 if (ret >= 0)
1489 fd_install(ret, file);
1490 else
1491 fput(file);
1492 }
1493 return ret;
1494}
1495
1496int f_dupfd(unsigned int from, struct file *file, unsigned flags)
1497{
1498 unsigned long nofile = rlimit(RLIMIT_NOFILE);
1499 int err;
1500 if (from >= nofile)
1501 return -EINVAL;
1502 err = alloc_fd(from, nofile, flags);
1503 if (err >= 0) {
1504 get_file(file);
1505 fd_install(err, file);
1506 }
1507 return err;
1508}
1509
1510int iterate_fd(struct files_struct *files, unsigned n,
1511 int (*f)(const void *, struct file *, unsigned),
1512 const void *p)
1513{
1514 struct fdtable *fdt;
1515 int res = 0;
1516 if (!files)
1517 return 0;
1518 spin_lock(&files->file_lock);
1519 for (fdt = files_fdtable(files); n < fdt->max_fds; n++) {
1520 struct file *file;
1521 file = rcu_dereference_check_fdtable(files, fdt->fd[n]);
1522 if (!file)
1523 continue;
1524 res = f(p, file, n);
1525 if (res)
1526 break;
1527 }
1528 spin_unlock(&files->file_lock);
1529 return res;
1530}
1531EXPORT_SYMBOL(iterate_fd);