| 1 | |
| 2 | |
| 3 | * linux/fs/file.c
|
| 4 | *
|
| 5 | * Copyright (C) 1998-1999, Stephen Tweedie and Bill Hawes
|
| 6 | *
|
| 7 | * Manage the dynamic fd arrays in the process files_struct.
|
| 8 |
|
| 9 |
|
| 10 | #include <linux/syscalls.h>
|
| 11 | #include <linux/export.h>
|
| 12 | #include <linux/fs.h>
|
| 13 | #include <linux/kernel.h>
|
| 14 | #include <linux/mm.h>
|
| 15 | #include <linux/sched/signal.h>
|
| 16 | #include <linux/slab.h>
|
| 17 | #include <linux/file.h>
|
| 18 | #include <linux/fdtable.h>
|
| 19 | #include <linux/bitops.h>
|
| 20 | #include <linux/spinlock.h>
|
| 21 | #include <linux/rcupdate.h>
|
| 22 | #include <linux/close_range.h>
|
| 23 | #include <linux/file_ref.h>
|
| 24 | #include <net/sock.h>
|
| 25 | #include <linux/init_task.h>
|
| 26 |
|
| 27 | #include "internal.h"
|
| 28 |
|
| 29 | static noinline bool __file_ref_put_badval(file_ref_t *ref, unsigned long cnt)
|
| 30 | {
|
| 31 | |
| 32 | * If the reference count was already in the dead zone, then this
|
| 33 | * put() operation is imbalanced. Warn, put the reference count back to
|
| 34 | * DEAD and tell the caller to not deconstruct the object.
|
| 35 |
|
| 36 | if (WARN_ONCE(cnt >= FILE_REF_RELEASED, "imbalanced put on file reference count")) {
|
| 37 | atomic_long_set(&ref->refcnt, FILE_REF_DEAD);
|
| 38 | return false;
|
| 39 | }
|
| 40 |
|
| 41 | |
| 42 | * This is a put() operation on a saturated refcount. Restore the
|
| 43 | * mean saturation value and tell the caller to not deconstruct the
|
| 44 | * object.
|
| 45 |
|
| 46 | if (cnt > FILE_REF_MAXREF)
|
| 47 | atomic_long_set(&ref->refcnt, FILE_REF_SATURATED);
|
| 48 | return false;
|
| 49 | }
|
| 50 |
|
| 51 | |
| 52 | * __file_ref_put - Slowpath of file_ref_put()
|
| 53 | * @ref: Pointer to the reference count
|
| 54 | * @cnt: Current reference count
|
| 55 | *
|
| 56 | * Invoked when the reference count is outside of the valid zone.
|
| 57 | *
|
| 58 | * Return:
|
| 59 | * True if this was the last reference with no future references
|
| 60 | * possible. This signals the caller that it can safely schedule the
|
| 61 | * object, which is protected by the reference counter, for
|
| 62 | * deconstruction.
|
| 63 | *
|
| 64 | * False if there are still active references or the put() raced
|
| 65 | * with a concurrent get()/put() pair. Caller is not allowed to
|
| 66 | * deconstruct the protected object.
|
| 67 |
|
| 68 | bool __file_ref_put(file_ref_t *ref, unsigned long cnt)
|
| 69 | {
|
| 70 |
|
| 71 | if (likely(cnt == FILE_REF_NOREF)) {
|
| 72 | |
| 73 | * Carefully try to set the reference count to FILE_REF_DEAD.
|
| 74 | *
|
| 75 | * This can fail if a concurrent get() operation has
|
| 76 | * elevated it again or the corresponding put() even marked
|
| 77 | * it dead already. Both are valid situations and do not
|
| 78 | * require a retry. If this fails the caller is not
|
| 79 | * allowed to deconstruct the object.
|
| 80 |
|
| 81 | if (!atomic_long_try_cmpxchg_release(&ref->refcnt, &cnt, FILE_REF_DEAD))
|
| 82 | return false;
|
| 83 |
|
| 84 | |
| 85 | * The caller can safely schedule the object for
|
| 86 | * deconstruction. Provide acquire ordering.
|
| 87 |
|
| 88 | smp_acquire__after_ctrl_dep();
|
| 89 | return true;
|
| 90 | }
|
| 91 |
|
| 92 | return __file_ref_put_badval(ref, cnt);
|
| 93 | }
|
| 94 | EXPORT_SYMBOL_GPL(__file_ref_put);
|
| 95 |
|
| 96 | unsigned int sysctl_nr_open __read_mostly = 1024*1024;
|
| 97 | unsigned int sysctl_nr_open_min = BITS_PER_LONG;
|
| 98 |
|
| 99 | #define __const_min(x, y) ((x) < (y) ? (x) : (y))
|
| 100 | unsigned int sysctl_nr_open_max =
|
| 101 | __const_min(INT_MAX, ~(size_t)0/sizeof(void *)) & -BITS_PER_LONG;
|
| 102 |
|
| 103 | static void __free_fdtable(struct fdtable *fdt)
|
| 104 | {
|
| 105 | kvfree(fdt->fd);
|
| 106 | kvfree(fdt->open_fds);
|
| 107 | kfree(fdt);
|
| 108 | }
|
| 109 |
|
| 110 | static void free_fdtable_rcu(struct rcu_head *rcu)
|
| 111 | {
|
| 112 | __free_fdtable(container_of(rcu, struct fdtable, rcu));
|
| 113 | }
|
| 114 |
|
| 115 | #define BITBIT_NR(nr) BITS_TO_LONGS(BITS_TO_LONGS(nr))
|
| 116 | #define BITBIT_SIZE(nr) (BITBIT_NR(nr) * sizeof(long))
|
| 117 |
|
| 118 | #define fdt_words(fdt) ((fdt)->max_fds / BITS_PER_LONG)
|
| 119 | |
| 120 | * Copy 'count' fd bits from the old table to the new table and clear the extra
|
| 121 | * space if any. This does not copy the file pointers. Called with the files
|
| 122 | * spinlock held for write.
|
| 123 |
|
| 124 | static inline void copy_fd_bitmaps(struct fdtable *nfdt, struct fdtable *ofdt,
|
| 125 | unsigned int copy_words)
|
| 126 | {
|
| 127 | unsigned int nwords = fdt_words(nfdt);
|
| 128 |
|
| 129 | bitmap_copy_and_extend(nfdt->open_fds, ofdt->open_fds,
|
| 130 | copy_words * BITS_PER_LONG, nwords * BITS_PER_LONG);
|
| 131 | bitmap_copy_and_extend(nfdt->close_on_exec, ofdt->close_on_exec,
|
| 132 | copy_words * BITS_PER_LONG, nwords * BITS_PER_LONG);
|
| 133 | bitmap_copy_and_extend(nfdt->full_fds_bits, ofdt->full_fds_bits,
|
| 134 | copy_words, nwords);
|
| 135 | }
|
| 136 |
|
| 137 | |
| 138 | * Copy all file descriptors from the old table to the new, expanded table and
|
| 139 | * clear the extra space. Called with the files spinlock held for write.
|
| 140 |
|
| 141 | static void copy_fdtable(struct fdtable *nfdt, struct fdtable *ofdt)
|
| 142 | {
|
| 143 | size_t cpy, set;
|
| 144 |
|
| 145 | BUG_ON(nfdt->max_fds < ofdt->max_fds);
|
| 146 |
|
| 147 | cpy = ofdt->max_fds * sizeof(struct file *);
|
| 148 | set = (nfdt->max_fds - ofdt->max_fds) * sizeof(struct file *);
|
| 149 | memcpy(nfdt->fd, ofdt->fd, cpy);
|
| 150 | memset((char *)nfdt->fd + cpy, 0, set);
|
| 151 |
|
| 152 | copy_fd_bitmaps(nfdt, ofdt, fdt_words(ofdt));
|
| 153 | }
|
| 154 |
|
| 155 | |
| 156 | * Note how the fdtable bitmap allocations very much have to be a multiple of
|
| 157 | * BITS_PER_LONG. This is not only because we walk those things in chunks of
|
| 158 | * 'unsigned long' in some places, but simply because that is how the Linux
|
| 159 | * kernel bitmaps are defined to work: they are not "bits in an array of bytes",
|
| 160 | * they are very much "bits in an array of unsigned long".
|
| 161 |
|
| 162 | static struct fdtable *alloc_fdtable(unsigned int slots_wanted)
|
| 163 | {
|
| 164 | struct fdtable *fdt;
|
| 165 | unsigned int nr;
|
| 166 | void *data;
|
| 167 |
|
| 168 | |
| 169 | * Figure out how many fds we actually want to support in this fdtable.
|
| 170 | * Allocation steps are keyed to the size of the fdarray, since it
|
| 171 | * grows far faster than any of the other dynamic data. We try to fit
|
| 172 | * the fdarray into comfortable page-tuned chunks: starting at 1024B
|
| 173 | * and growing in powers of two from there on. Since we called only
|
| 174 | * with slots_wanted > BITS_PER_LONG (embedded instance in files->fdtab
|
| 175 | * already gives BITS_PER_LONG slots), the above boils down to
|
| 176 | * 1. use the smallest power of two large enough to give us that many
|
| 177 | * slots.
|
| 178 | * 2. on 32bit skip 64 and 128 - the minimal capacity we want there is
|
| 179 | * 256 slots (i.e. 1Kb fd array).
|
| 180 | * 3. on 64bit don't skip anything, 1Kb fd array means 128 slots there
|
| 181 | * and we are never going to be asked for 64 or less.
|
| 182 |
|
| 183 | if (IS_ENABLED(CONFIG_32BIT) && slots_wanted < 256)
|
| 184 | nr = 256;
|
| 185 | else
|
| 186 | nr = roundup_pow_of_two(slots_wanted);
|
| 187 | |
| 188 | * Note that this can drive nr *below* what we had passed if sysctl_nr_open
|
| 189 | * had been set lower between the check in expand_files() and here.
|
| 190 | *
|
| 191 | * We make sure that nr remains a multiple of BITS_PER_LONG - otherwise
|
| 192 | * bitmaps handling below becomes unpleasant, to put it mildly...
|
| 193 |
|
| 194 | if (unlikely(nr > sysctl_nr_open)) {
|
| 195 | nr = round_down(sysctl_nr_open, BITS_PER_LONG);
|
| 196 | if (nr < slots_wanted)
|
| 197 | return ERR_PTR(-EMFILE);
|
| 198 | }
|
| 199 |
|
| 200 | |
| 201 | * Check if the allocation size would exceed INT_MAX. kvmalloc_array()
|
| 202 | * and kvmalloc() will warn if the allocation size is greater than
|
| 203 | * INT_MAX, as filp_cache objects are not __GFP_NOWARN.
|
| 204 | *
|
| 205 | * This can happen when sysctl_nr_open is set to a very high value and
|
| 206 | * a process tries to use a file descriptor near that limit. For example,
|
| 207 | * if sysctl_nr_open is set to 1073741816 (0x3ffffff8) - which is what
|
| 208 | * systemd typically sets it to - then trying to use a file descriptor
|
| 209 | * close to that value will require allocating a file descriptor table
|
| 210 | * that exceeds 8GB in size.
|
| 211 |
|
| 212 | if (unlikely(nr > INT_MAX / sizeof(struct file *)))
|
| 213 | return ERR_PTR(-EMFILE);
|
| 214 |
|
| 215 | fdt = kmalloc_obj(struct fdtable, GFP_KERNEL_ACCOUNT);
|
| 216 | if (!fdt)
|
| 217 | goto out;
|
| 218 | fdt->max_fds = nr;
|
| 219 | data = kvmalloc_objs(struct file *, nr, GFP_KERNEL_ACCOUNT);
|
| 220 | if (!data)
|
| 221 | goto out_fdt;
|
| 222 | fdt->fd = data;
|
| 223 |
|
| 224 | data = kvmalloc(max_t(size_t,
|
| 225 | 2 * nr / BITS_PER_BYTE + BITBIT_SIZE(nr), L1_CACHE_BYTES),
|
| 226 | GFP_KERNEL_ACCOUNT);
|
| 227 | if (!data)
|
| 228 | goto out_arr;
|
| 229 | fdt->open_fds = data;
|
| 230 | data += nr / BITS_PER_BYTE;
|
| 231 | fdt->close_on_exec = data;
|
| 232 | data += nr / BITS_PER_BYTE;
|
| 233 | fdt->full_fds_bits = data;
|
| 234 |
|
| 235 | return fdt;
|
| 236 |
|
| 237 | out_arr:
|
| 238 | kvfree(fdt->fd);
|
| 239 | out_fdt:
|
| 240 | kfree(fdt);
|
| 241 | out:
|
| 242 | return ERR_PTR(-ENOMEM);
|
| 243 | }
|
| 244 |
|
| 245 | |
| 246 | * Expand the file descriptor table.
|
| 247 | * This function will allocate a new fdtable and both fd array and fdset, of
|
| 248 | * the given size.
|
| 249 | * Return <0 error code on error; 0 on successful completion.
|
| 250 | * The files->file_lock should be held on entry, and will be held on exit.
|
| 251 |
|
| 252 | static int expand_fdtable(struct files_struct *files, unsigned int nr)
|
| 253 | __releases(files->file_lock)
|
| 254 | __acquires(files->file_lock)
|
| 255 | {
|
| 256 | struct fdtable *new_fdt, *cur_fdt;
|
| 257 |
|
| 258 | spin_unlock(&files->file_lock);
|
| 259 | new_fdt = alloc_fdtable(nr + 1);
|
| 260 |
|
| 261 | |
| 262 | * or have finished their rcu_read_lock_sched() section.
|
| 263 |
|
| 264 | if (atomic_read(&files->count) > 1)
|
| 265 | synchronize_rcu();
|
| 266 |
|
| 267 | spin_lock(&files->file_lock);
|
| 268 | if (IS_ERR(new_fdt))
|
| 269 | return PTR_ERR(new_fdt);
|
| 270 | cur_fdt = files_fdtable(files);
|
| 271 | BUG_ON(nr < cur_fdt->max_fds);
|
| 272 | copy_fdtable(new_fdt, cur_fdt);
|
| 273 | rcu_assign_pointer(files->fdt, new_fdt);
|
| 274 | if (cur_fdt != &files->fdtab)
|
| 275 | call_rcu(&cur_fdt->rcu, free_fdtable_rcu);
|
| 276 |
|
| 277 | smp_wmb();
|
| 278 | return 0;
|
| 279 | }
|
| 280 |
|
| 281 | |
| 282 | * Expand files.
|
| 283 | * This function will expand the file structures, if the requested size exceeds
|
| 284 | * the current capacity and there is room for expansion.
|
| 285 | * Return <0 error code on error; 0 on success.
|
| 286 | * The files->file_lock should be held on entry, and will be held on exit.
|
| 287 |
|
| 288 | static int expand_files(struct files_struct *files, unsigned int nr)
|
| 289 | __releases(files->file_lock)
|
| 290 | __acquires(files->file_lock)
|
| 291 | {
|
| 292 | struct fdtable *fdt;
|
| 293 | int error;
|
| 294 |
|
| 295 | repeat:
|
| 296 | fdt = files_fdtable(files);
|
| 297 |
|
| 298 |
|
| 299 | if (nr < fdt->max_fds)
|
| 300 | return 0;
|
| 301 |
|
| 302 | if (unlikely(files->resize_in_progress)) {
|
| 303 | spin_unlock(&files->file_lock);
|
| 304 | wait_event(files->resize_wait, !files->resize_in_progress);
|
| 305 | spin_lock(&files->file_lock);
|
| 306 | goto repeat;
|
| 307 | }
|
| 308 |
|
| 309 |
|
| 310 | if (unlikely(nr >= sysctl_nr_open))
|
| 311 | return -EMFILE;
|
| 312 |
|
| 313 |
|
| 314 | files->resize_in_progress = true;
|
| 315 | error = expand_fdtable(files, nr);
|
| 316 | files->resize_in_progress = false;
|
| 317 |
|
| 318 | wake_up_all(&files->resize_wait);
|
| 319 | return error;
|
| 320 | }
|
| 321 |
|
| 322 | static inline void __set_close_on_exec(unsigned int fd, struct fdtable *fdt,
|
| 323 | bool set)
|
| 324 | {
|
| 325 | if (set) {
|
| 326 | __set_bit(fd, fdt->close_on_exec);
|
| 327 | } else {
|
| 328 | if (test_bit(fd, fdt->close_on_exec))
|
| 329 | __clear_bit(fd, fdt->close_on_exec);
|
| 330 | }
|
| 331 | }
|
| 332 |
|
| 333 | static inline void __set_open_fd(unsigned int fd, struct fdtable *fdt, bool set)
|
| 334 | {
|
| 335 | __set_bit(fd, fdt->open_fds);
|
| 336 | __set_close_on_exec(fd, fdt, set);
|
| 337 | fd /= BITS_PER_LONG;
|
| 338 | if (!~fdt->open_fds[fd])
|
| 339 | __set_bit(fd, fdt->full_fds_bits);
|
| 340 | }
|
| 341 |
|
| 342 | static inline void __clear_open_fd(unsigned int fd, struct fdtable *fdt)
|
| 343 | {
|
| 344 | __clear_bit(fd, fdt->open_fds);
|
| 345 | fd /= BITS_PER_LONG;
|
| 346 | if (test_bit(fd, fdt->full_fds_bits))
|
| 347 | __clear_bit(fd, fdt->full_fds_bits);
|
| 348 | }
|
| 349 |
|
| 350 | static inline bool fd_is_open(unsigned int fd, const struct fdtable *fdt)
|
| 351 | {
|
| 352 | return test_bit(fd, fdt->open_fds);
|
| 353 | }
|
| 354 |
|
| 355 |
|
| 356 | static unsigned long fd_range_word(struct fd_range *range, unsigned int i)
|
| 357 | {
|
| 358 | unsigned int first = i * BITS_PER_LONG;
|
| 359 | unsigned int last = first + BITS_PER_LONG - 1;
|
| 360 |
|
| 361 | if (range->to < first || range->from > last)
|
| 362 | return 0;
|
| 363 | return GENMASK(min(range->to, last) - first,
|
| 364 | max(range->from, first) - first);
|
| 365 | }
|
| 366 |
|
| 367 |
|
| 368 | static unsigned long dup_fd_dropped_word(struct fdtable *fdt, unsigned int i,
|
| 369 | struct fd_range *range)
|
| 370 | {
|
| 371 | unsigned long dropped;
|
| 372 |
|
| 373 | if (!range)
|
| 374 | return 0;
|
| 375 | dropped = fd_range_word(range, i);
|
| 376 | if (range->flags & FD_RANGE_EXCEPT)
|
| 377 | dropped = ~dropped;
|
| 378 | if (range->flags & FD_RANGE_CLOEXEC_ONLY)
|
| 379 | dropped &= fdt->close_on_exec[i];
|
| 380 | return dropped;
|
| 381 | }
|
| 382 |
|
| 383 | |
| 384 | * Note that a sane fdtable size always has to be a multiple of
|
| 385 | * BITS_PER_LONG, since we have bitmaps that are sized by this.
|
| 386 | *
|
| 387 | * range is optional. When close_range() is asked to unshare dup_fd()
|
| 388 | * will leave any files behind according to the range and its flags. The
|
| 389 | * cloned table only has to reach the last open descriptor that is
|
| 390 | * carried over.
|
| 391 |
|
| 392 | static unsigned int sane_fdtable_size(struct fdtable *fdt, struct fd_range *range)
|
| 393 | {
|
| 394 | unsigned int last = find_last_bit(fdt->open_fds, fdt->max_fds);
|
| 395 | unsigned int i;
|
| 396 |
|
| 397 | if (last == fdt->max_fds)
|
| 398 | return NR_OPEN_DEFAULT;
|
| 399 | if (!range)
|
| 400 | return ALIGN(last + 1, BITS_PER_LONG);
|
| 401 |
|
| 402 | if (range->flags & FD_RANGE_CLOEXEC_ONLY) {
|
| 403 |
|
| 404 | i = last / BITS_PER_LONG + 1;
|
| 405 | while (i--) {
|
| 406 | unsigned long dropped = dup_fd_dropped_word(fdt, i, range);
|
| 407 |
|
| 408 | if (fdt->open_fds[i] & ~dropped)
|
| 409 | return (i + 1) * BITS_PER_LONG;
|
| 410 | }
|
| 411 | return NR_OPEN_DEFAULT;
|
| 412 | }
|
| 413 |
|
| 414 | if (range->flags & FD_RANGE_EXCEPT) {
|
| 415 |
|
| 416 | if (last > range->to) {
|
| 417 | last = find_last_bit(fdt->open_fds, range->to + 1);
|
| 418 | if (last > range->to)
|
| 419 | return NR_OPEN_DEFAULT;
|
| 420 | }
|
| 421 | if (last < range->from)
|
| 422 | return NR_OPEN_DEFAULT;
|
| 423 | } else if (last >= range->from && last <= range->to) {
|
| 424 |
|
| 425 | last = find_last_bit(fdt->open_fds, range->from);
|
| 426 | if (last == range->from)
|
| 427 | return NR_OPEN_DEFAULT;
|
| 428 | }
|
| 429 | return ALIGN(last + 1, BITS_PER_LONG);
|
| 430 | }
|
| 431 |
|
| 432 | |
| 433 | * Allocate a new descriptor table and copy contents from the passed in
|
| 434 | * instance. Returns a pointer to cloned table on success, ERR_PTR()
|
| 435 | * on failure. For 'range' see sane_fdtable_size().
|
| 436 |
|
| 437 | struct files_struct *dup_fd(struct files_struct *oldf, struct fd_range *range)
|
| 438 | {
|
| 439 | struct files_struct *newf;
|
| 440 | struct file **old_fds, **new_fds;
|
| 441 | unsigned int open_files, fd;
|
| 442 | unsigned long dropped = 0;
|
| 443 | struct fdtable *old_fdt, *new_fdt;
|
| 444 |
|
| 445 | newf = kmem_cache_alloc(files_cachep, GFP_KERNEL);
|
| 446 | if (!newf)
|
| 447 | return ERR_PTR(-ENOMEM);
|
| 448 |
|
| 449 | atomic_set(&newf->count, 1);
|
| 450 |
|
| 451 | spin_lock_init(&newf->file_lock);
|
| 452 | newf->resize_in_progress = false;
|
| 453 | init_waitqueue_head(&newf->resize_wait);
|
| 454 | newf->next_fd = 0;
|
| 455 | new_fdt = &newf->fdtab;
|
| 456 | new_fdt->max_fds = NR_OPEN_DEFAULT;
|
| 457 | new_fdt->close_on_exec = newf->close_on_exec_init;
|
| 458 | new_fdt->open_fds = newf->open_fds_init;
|
| 459 | new_fdt->full_fds_bits = newf->full_fds_bits_init;
|
| 460 | new_fdt->fd = &newf->fd_array[0];
|
| 461 |
|
| 462 | spin_lock(&oldf->file_lock);
|
| 463 | old_fdt = files_fdtable(oldf);
|
| 464 | open_files = sane_fdtable_size(old_fdt, range);
|
| 465 |
|
| 466 | |
| 467 | * Check whether we need to allocate a larger fd array and fd set.
|
| 468 |
|
| 469 | while (unlikely(open_files > new_fdt->max_fds)) {
|
| 470 | spin_unlock(&oldf->file_lock);
|
| 471 |
|
| 472 | if (new_fdt != &newf->fdtab)
|
| 473 | __free_fdtable(new_fdt);
|
| 474 |
|
| 475 | new_fdt = alloc_fdtable(open_files);
|
| 476 | if (IS_ERR(new_fdt)) {
|
| 477 | kmem_cache_free(files_cachep, newf);
|
| 478 | return ERR_CAST(new_fdt);
|
| 479 | }
|
| 480 |
|
| 481 | |
| 482 | * Reacquire the oldf lock and a pointer to its fd table
|
| 483 | * who knows it may have a new bigger fd table. We need
|
| 484 | * the latest pointer.
|
| 485 |
|
| 486 | spin_lock(&oldf->file_lock);
|
| 487 | old_fdt = files_fdtable(oldf);
|
| 488 | open_files = sane_fdtable_size(old_fdt, range);
|
| 489 | }
|
| 490 |
|
| 491 | copy_fd_bitmaps(new_fdt, old_fdt, open_files / BITS_PER_LONG);
|
| 492 |
|
| 493 | old_fds = old_fdt->fd;
|
| 494 | new_fds = new_fdt->fd;
|
| 495 |
|
| 496 | |
| 497 | * We may be racing against fd allocation from other threads using this
|
| 498 | * files_struct, despite holding ->file_lock.
|
| 499 | *
|
| 500 | * alloc_fd() might have already claimed a slot, while fd_install()
|
| 501 | * did not populate it yet. Note the latter operates locklessly, so
|
| 502 | * the file can show up as we are walking the array below.
|
| 503 | *
|
| 504 | * At the same time we know no files will disappear as all other
|
| 505 | * operations take the lock.
|
| 506 | *
|
| 507 | * Instead of trying to placate userspace racing with itself, we
|
| 508 | * ref the file if we see it and mark the fd slot as unused otherwise.
|
| 509 | * Descriptors dup_fd() is asked to leave behind get the same treatment.
|
| 510 |
|
| 511 | for (fd = 0; fd < open_files; fd++) {
|
| 512 | struct file *f = rcu_dereference_raw(*old_fds++);
|
| 513 |
|
| 514 | if (!(fd % BITS_PER_LONG))
|
| 515 | dropped = dup_fd_dropped_word(old_fdt, fd / BITS_PER_LONG, range);
|
| 516 | if (f && !(dropped & BIT_MASK(fd))) {
|
| 517 | get_file(f);
|
| 518 | } else {
|
| 519 | f = NULL;
|
| 520 | __clear_open_fd(fd, new_fdt);
|
| 521 | }
|
| 522 | rcu_assign_pointer(*new_fds++, f);
|
| 523 | }
|
| 524 | spin_unlock(&oldf->file_lock);
|
| 525 |
|
| 526 |
|
| 527 | memset(new_fds, 0, (new_fdt->max_fds - open_files) * sizeof(struct file *));
|
| 528 |
|
| 529 | rcu_assign_pointer(newf->fdt, new_fdt);
|
| 530 |
|
| 531 | return newf;
|
| 532 | }
|
| 533 |
|
| 534 | |
| 535 | * Unshare file descriptor table if it is being shared
|
| 536 |
|
| 537 | int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp)
|
| 538 | {
|
| 539 | struct files_struct *fd = current->files;
|
| 540 |
|
| 541 | if ((unshare_flags & CLONE_FILES) &&
|
| 542 | (fd && atomic_read(&fd->count) > 1)) {
|
| 543 | fd = dup_fd(fd, NULL);
|
| 544 | if (IS_ERR(fd))
|
| 545 | return PTR_ERR(fd);
|
| 546 | *new_fdp = fd;
|
| 547 | }
|
| 548 |
|
| 549 | return 0;
|
| 550 | }
|
| 551 |
|
| 552 | static struct fdtable *close_files(struct files_struct *files)
|
| 553 | {
|
| 554 | |
| 555 | * It is safe to dereference the fd table without RCU or
|
| 556 | * ->file_lock because this is the last reference to the
|
| 557 | * files structure.
|
| 558 |
|
| 559 | struct fdtable *fdt = rcu_dereference_raw(files->fdt);
|
| 560 | unsigned int j = fdt->max_fds / BITS_PER_LONG;
|
| 561 |
|
| 562 |
|
| 563 | while (j--) {
|
| 564 | unsigned long set = fdt->open_fds[j];
|
| 565 |
|
| 566 | while (set) {
|
| 567 | unsigned int bit = __fls(set);
|
| 568 | struct file *file = fdt->fd[j * BITS_PER_LONG + bit];
|
| 569 |
|
| 570 | set ^= 1UL << bit;
|
| 571 | if (file) {
|
| 572 | filp_close_sync(file, files);
|
| 573 | cond_resched();
|
| 574 | }
|
| 575 | }
|
| 576 | }
|
| 577 |
|
| 578 | return fdt;
|
| 579 | }
|
| 580 |
|
| 581 | void put_files_struct(struct files_struct *files)
|
| 582 | {
|
| 583 | if (atomic_dec_and_test(&files->count)) {
|
| 584 | struct fdtable *fdt = close_files(files);
|
| 585 |
|
| 586 |
|
| 587 | if (fdt != &files->fdtab)
|
| 588 | __free_fdtable(fdt);
|
| 589 | kmem_cache_free(files_cachep, files);
|
| 590 | }
|
| 591 | }
|
| 592 |
|
| 593 |
|
| 594 | void switch_files_struct(struct task_struct *tsk, struct files_struct *files)
|
| 595 | {
|
| 596 | scoped_guard(task_lock, tsk)
|
| 597 | swap(tsk->files, files);
|
| 598 | put_files_struct(files);
|
| 599 | }
|
| 600 |
|
| 601 | void exit_files(struct task_struct *tsk)
|
| 602 | {
|
| 603 | if (tsk->files)
|
| 604 | switch_files_struct(tsk, NULL);
|
| 605 | }
|
| 606 |
|
| 607 | struct files_struct init_files = {
|
| 608 | .count = ATOMIC_INIT(1),
|
| 609 | .fdt = &init_files.fdtab,
|
| 610 | .fdtab = {
|
| 611 | .max_fds = NR_OPEN_DEFAULT,
|
| 612 | .fd = &init_files.fd_array[0],
|
| 613 | .close_on_exec = init_files.close_on_exec_init,
|
| 614 | .open_fds = init_files.open_fds_init,
|
| 615 | .full_fds_bits = init_files.full_fds_bits_init,
|
| 616 | },
|
| 617 | .file_lock = __SPIN_LOCK_UNLOCKED(init_files.file_lock),
|
| 618 | .resize_wait = __WAIT_QUEUE_HEAD_INITIALIZER(init_files.resize_wait),
|
| 619 | };
|
| 620 |
|
| 621 | static unsigned int find_next_fd(struct fdtable *fdt, unsigned int start)
|
| 622 | {
|
| 623 | unsigned int maxfd = fdt->max_fds;
|
| 624 | unsigned int max_fds_words = maxfd / BITS_PER_LONG;
|
| 625 | unsigned int fds_word_idx = start / BITS_PER_LONG;
|
| 626 | unsigned int bit;
|
| 627 |
|
| 628 | |
| 629 | * Try to avoid looking at the second level bitmap
|
| 630 |
|
| 631 | bit = find_next_zero_bit(&fdt->open_fds[fds_word_idx], BITS_PER_LONG,
|
| 632 | start & (BITS_PER_LONG - 1));
|
| 633 | if (bit < BITS_PER_LONG)
|
| 634 | return bit + (fds_word_idx * BITS_PER_LONG);
|
| 635 |
|
| 636 | bit = BITS_PER_LONG *
|
| 637 | find_next_zero_bit(fdt->full_fds_bits, max_fds_words, fds_word_idx + 1);
|
| 638 | if (bit >= maxfd)
|
| 639 | return maxfd;
|
| 640 | return find_next_zero_bit(fdt->open_fds, maxfd, bit);
|
| 641 | }
|
| 642 |
|
| 643 | |
| 644 | * allocate a file descriptor, mark it busy.
|
| 645 |
|
| 646 | static int alloc_fd(unsigned start, unsigned end, unsigned flags)
|
| 647 | {
|
| 648 | struct files_struct *files = current->files;
|
| 649 | unsigned int fd;
|
| 650 | int error;
|
| 651 | struct fdtable *fdt;
|
| 652 |
|
| 653 | spin_lock(&files->file_lock);
|
| 654 | repeat:
|
| 655 | fdt = files_fdtable(files);
|
| 656 | fd = start;
|
| 657 | if (fd < files->next_fd)
|
| 658 | fd = files->next_fd;
|
| 659 |
|
| 660 | if (likely(fd < fdt->max_fds))
|
| 661 | fd = find_next_fd(fdt, fd);
|
| 662 |
|
| 663 | |
| 664 | * N.B. For clone tasks sharing a files structure, this test
|
| 665 | * will limit the total number of files that can be opened.
|
| 666 |
|
| 667 | error = -EMFILE;
|
| 668 | if (unlikely(fd >= end))
|
| 669 | goto out;
|
| 670 |
|
| 671 | if (unlikely(fd >= fdt->max_fds)) {
|
| 672 | error = expand_files(files, fd);
|
| 673 | if (error < 0)
|
| 674 | goto out;
|
| 675 |
|
| 676 | goto repeat;
|
| 677 | }
|
| 678 |
|
| 679 | if (start <= files->next_fd)
|
| 680 | files->next_fd = fd + 1;
|
| 681 |
|
| 682 | __set_open_fd(fd, fdt, flags & O_CLOEXEC);
|
| 683 | error = fd;
|
| 684 | VFS_BUG_ON(rcu_access_pointer(fdt->fd[fd]) != NULL);
|
| 685 |
|
| 686 | out:
|
| 687 | spin_unlock(&files->file_lock);
|
| 688 | return error;
|
| 689 | }
|
| 690 |
|
| 691 | int __get_unused_fd_flags(unsigned flags, unsigned long nofile)
|
| 692 | {
|
| 693 | return alloc_fd(0, nofile, flags);
|
| 694 | }
|
| 695 |
|
| 696 | int get_unused_fd_flags(unsigned flags)
|
| 697 | {
|
| 698 | return __get_unused_fd_flags(flags, rlimit(RLIMIT_NOFILE));
|
| 699 | }
|
| 700 | EXPORT_SYMBOL(get_unused_fd_flags);
|
| 701 |
|
| 702 | static void __put_unused_fd(struct files_struct *files, unsigned int fd)
|
| 703 | {
|
| 704 | struct fdtable *fdt = files_fdtable(files);
|
| 705 | __clear_open_fd(fd, fdt);
|
| 706 | if (fd < files->next_fd)
|
| 707 | files->next_fd = fd;
|
| 708 | }
|
| 709 |
|
| 710 | void put_unused_fd(unsigned int fd)
|
| 711 | {
|
| 712 | struct files_struct *files = current->files;
|
| 713 | spin_lock(&files->file_lock);
|
| 714 | __put_unused_fd(files, fd);
|
| 715 | spin_unlock(&files->file_lock);
|
| 716 | }
|
| 717 |
|
| 718 | EXPORT_SYMBOL(put_unused_fd);
|
| 719 |
|
| 720 | |
| 721 | * Install a file pointer in the fd array while it is being resized.
|
| 722 | *
|
| 723 | * We need to make sure our update to the array does not get lost as the resizing
|
| 724 | * thread can be copying the content as we modify it.
|
| 725 | *
|
| 726 | * We have two ways to do it:
|
| 727 | * - go off CPU waiting for resize_in_progress to clear
|
| 728 | * - take the spin lock
|
| 729 | *
|
| 730 | * The latter is trivial to implement and saves us from having to might_sleep()
|
| 731 | * for debugging purposes.
|
| 732 | *
|
| 733 | * This is moved out of line from fd_install() to convince gcc to optimize that
|
| 734 | * routine better.
|
| 735 |
|
| 736 | static void noinline fd_install_slowpath(unsigned int fd, struct file *file)
|
| 737 | {
|
| 738 | struct files_struct *files = current->files;
|
| 739 | struct fdtable *fdt;
|
| 740 |
|
| 741 | spin_lock(&files->file_lock);
|
| 742 | fdt = files_fdtable(files);
|
| 743 | VFS_BUG_ON(rcu_access_pointer(fdt->fd[fd]) != NULL);
|
| 744 | rcu_assign_pointer(fdt->fd[fd], file);
|
| 745 | spin_unlock(&files->file_lock);
|
| 746 | }
|
| 747 |
|
| 748 | |
| 749 | * fd_install - install a file pointer in the fd array
|
| 750 | * @fd: file descriptor to install the file in
|
| 751 | * @file: the file to install
|
| 752 | *
|
| 753 | * This consumes the "file" refcount, so callers should treat it
|
| 754 | * as if they had called fput(file).
|
| 755 |
|
| 756 | void fd_install(unsigned int fd, struct file *file)
|
| 757 | {
|
| 758 | struct files_struct *files = current->files;
|
| 759 | struct fdtable *fdt;
|
| 760 |
|
| 761 | if (WARN_ON_ONCE(unlikely(file->f_mode & FMODE_BACKING)))
|
| 762 | return;
|
| 763 |
|
| 764 | rcu_read_lock_sched();
|
| 765 | if (unlikely(files->resize_in_progress)) {
|
| 766 | rcu_read_unlock_sched();
|
| 767 | fd_install_slowpath(fd, file);
|
| 768 | return;
|
| 769 | }
|
| 770 |
|
| 771 | smp_rmb();
|
| 772 | fdt = rcu_dereference_sched(files->fdt);
|
| 773 | VFS_BUG_ON(rcu_access_pointer(fdt->fd[fd]) != NULL);
|
| 774 | rcu_assign_pointer(fdt->fd[fd], file);
|
| 775 | rcu_read_unlock_sched();
|
| 776 | }
|
| 777 |
|
| 778 | EXPORT_SYMBOL(fd_install);
|
| 779 |
|
| 780 | |
| 781 | * file_close_fd_locked - return file associated with fd
|
| 782 | * @files: file struct to retrieve file from
|
| 783 | * @fd: file descriptor to retrieve file for
|
| 784 | *
|
| 785 | * Doesn't take a separate reference count.
|
| 786 | *
|
| 787 | * Context: files_lock must be held.
|
| 788 | *
|
| 789 | * Returns: The file associated with @fd (NULL if @fd is not open)
|
| 790 |
|
| 791 | struct file *file_close_fd_locked(struct files_struct *files, unsigned fd)
|
| 792 | {
|
| 793 | struct fdtable *fdt = files_fdtable(files);
|
| 794 | struct file *file;
|
| 795 |
|
| 796 | lockdep_assert_held(&files->file_lock);
|
| 797 |
|
| 798 | if (fd >= fdt->max_fds)
|
| 799 | return NULL;
|
| 800 |
|
| 801 | fd = array_index_nospec(fd, fdt->max_fds);
|
| 802 | file = rcu_dereference_raw(fdt->fd[fd]);
|
| 803 | if (file) {
|
| 804 | rcu_assign_pointer(fdt->fd[fd], NULL);
|
| 805 | __put_unused_fd(files, fd);
|
| 806 | }
|
| 807 | return file;
|
| 808 | }
|
| 809 |
|
| 810 | int close_fd(unsigned fd)
|
| 811 | {
|
| 812 | struct file *file;
|
| 813 |
|
| 814 | file = file_close_fd(fd);
|
| 815 | if (!file)
|
| 816 | return -EBADF;
|
| 817 |
|
| 818 | return filp_close(file, current->files);
|
| 819 | }
|
| 820 | EXPORT_SYMBOL(close_fd);
|
| 821 |
|
| 822 | |
| 823 | * last_fd - return last valid index into fd table
|
| 824 | * @fdt: File descriptor table.
|
| 825 | *
|
| 826 | * Context: Either rcu read lock or files_lock must be held.
|
| 827 | *
|
| 828 | * Returns: Last valid index into fdtable.
|
| 829 |
|
| 830 | static inline unsigned last_fd(struct fdtable *fdt)
|
| 831 | {
|
| 832 | return fdt->max_fds - 1;
|
| 833 | }
|
| 834 |
|
| 835 | static inline void __range_cloexec(struct files_struct *cur_fds,
|
| 836 | struct fd_range *range)
|
| 837 | {
|
| 838 | struct fdtable *fdt;
|
| 839 | unsigned int last;
|
| 840 |
|
| 841 | spin_lock(&cur_fds->file_lock);
|
| 842 | fdt = files_fdtable(cur_fds);
|
| 843 |
|
| 844 | last = last_fd(fdt);
|
| 845 | if (!(range->flags & FD_RANGE_EXCEPT)) {
|
| 846 | if (range->from <= last)
|
| 847 | bitmap_set(fdt->close_on_exec, range->from,
|
| 848 | min(range->to, last) - range->from + 1);
|
| 849 | } else {
|
| 850 | if (range->from > 0)
|
| 851 | bitmap_set(fdt->close_on_exec, 0,
|
| 852 | min(range->from - 1, last) + 1);
|
| 853 | if (range->to < last)
|
| 854 | bitmap_set(fdt->close_on_exec, range->to + 1,
|
| 855 | last - range->to);
|
| 856 | }
|
| 857 | spin_unlock(&cur_fds->file_lock);
|
| 858 | }
|
| 859 |
|
| 860 |
|
| 861 | static inline unsigned int last_fd_to_close(struct fdtable *fdt, unsigned int n,
|
| 862 | struct fd_range *range)
|
| 863 | {
|
| 864 | unsigned int i, lo = 0;
|
| 865 |
|
| 866 | if (!(range->flags & FD_RANGE_EXCEPT))
|
| 867 | lo = range->from / BITS_PER_LONG;
|
| 868 | for (i = n ? (n - 1) / BITS_PER_LONG + 1 : 0; i-- > lo; ) {
|
| 869 | unsigned long set = fdt->open_fds[i];
|
| 870 |
|
| 871 | if (!set) {
|
| 872 |
|
| 873 | unsigned int last = find_last_bit(fdt->open_fds, i * BITS_PER_LONG);
|
| 874 |
|
| 875 | if (last >= i * BITS_PER_LONG)
|
| 876 | break;
|
| 877 | i = last / BITS_PER_LONG + 1;
|
| 878 | continue;
|
| 879 | }
|
| 880 |
|
| 881 | if ((range->flags & FD_RANGE_EXCEPT) &&
|
| 882 | i * BITS_PER_LONG >= range->from &&
|
| 883 | i * BITS_PER_LONG + BITS_PER_LONG - 1 <= range->to) {
|
| 884 | if (!range->from)
|
| 885 | break;
|
| 886 | i = (range->from - 1) / BITS_PER_LONG + 1;
|
| 887 | continue;
|
| 888 | }
|
| 889 | set &= dup_fd_dropped_word(fdt, i, range);
|
| 890 | if (i == (n - 1) / BITS_PER_LONG)
|
| 891 | set &= BITMAP_LAST_WORD_MASK(n);
|
| 892 | if (set)
|
| 893 | return i * BITS_PER_LONG + __fls(set);
|
| 894 | }
|
| 895 | return n;
|
| 896 | }
|
| 897 |
|
| 898 | static inline void __range_close(struct files_struct *files,
|
| 899 | struct fd_range *range)
|
| 900 | {
|
| 901 | struct file *file;
|
| 902 | struct fdtable *fdt;
|
| 903 | unsigned int fd, n;
|
| 904 |
|
| 905 | spin_lock(&files->file_lock);
|
| 906 | fdt = files_fdtable(files);
|
| 907 | if (range->flags & FD_RANGE_EXCEPT)
|
| 908 |
|
| 909 | n = fdt->max_fds;
|
| 910 | else
|
| 911 | n = min(range->to, last_fd(fdt)) + 1;
|
| 912 |
|
| 913 |
|
| 914 | while ((fd = last_fd_to_close(fdt, n, range)) < n) {
|
| 915 | n = fd;
|
| 916 | file = file_close_fd_locked(files, fd);
|
| 917 | if (file) {
|
| 918 | spin_unlock(&files->file_lock);
|
| 919 | filp_close_sync(file, files);
|
| 920 | cond_resched();
|
| 921 | spin_lock(&files->file_lock);
|
| 922 | fdt = files_fdtable(files);
|
| 923 | } else if (need_resched()) {
|
| 924 | spin_unlock(&files->file_lock);
|
| 925 | cond_resched();
|
| 926 | spin_lock(&files->file_lock);
|
| 927 | fdt = files_fdtable(files);
|
| 928 | }
|
| 929 | }
|
| 930 | spin_unlock(&files->file_lock);
|
| 931 | }
|
| 932 |
|
| 933 | |
| 934 | * sys_close_range() - Close all file descriptors in a given range.
|
| 935 | *
|
| 936 | * @fd: starting file descriptor to close
|
| 937 | * @max_fd: last file descriptor to close
|
| 938 | * @flags: CLOSE_RANGE flags.
|
| 939 | *
|
| 940 | * This closes a range of file descriptors. All file descriptors
|
| 941 | * from @fd up to and including @max_fd are closed.
|
| 942 | * Currently, errors to close a given file descriptor are ignored.
|
| 943 | *
|
| 944 | * With CLOSE_RANGE_EXCEPT the range names what to leave alone instead:
|
| 945 | * every open file descriptor outside of [@fd, @max_fd] is closed, or
|
| 946 | * marked close-on-exec with CLOSE_RANGE_CLOEXEC.
|
| 947 | *
|
| 948 | * With CLOSE_RANGE_CLOEXEC_ONLY only file descriptors that have
|
| 949 | * close-on-exec set are closed. Together with CLOSE_RANGE_EXCEPT the
|
| 950 | * range names the close-on-exec file descriptors to keep. To keep none
|
| 951 | * of them, name a range that cannot hold an open file descriptor, e.g.
|
| 952 | * close_range(~0U, ~0U, ...).
|
| 953 |
|
| 954 | SYSCALL_DEFINE3(close_range, unsigned int, fd, unsigned int, max_fd,
|
| 955 | unsigned int, flags)
|
| 956 | {
|
| 957 | struct task_struct *me = current;
|
| 958 | struct files_struct *cur_fds = me->files, *fds = NULL;
|
| 959 | struct fd_range range = {fd, max_fd};
|
| 960 |
|
| 961 | if (flags & ~(CLOSE_RANGE_UNSHARE | CLOSE_RANGE_CLOEXEC |
|
| 962 | CLOSE_RANGE_EXCEPT | CLOSE_RANGE_CLOEXEC_ONLY))
|
| 963 | return -EINVAL;
|
| 964 |
|
| 965 |
|
| 966 | if (hweight32(flags & (CLOSE_RANGE_CLOEXEC |
|
| 967 | CLOSE_RANGE_CLOEXEC_ONLY)) > 1)
|
| 968 | return -EINVAL;
|
| 969 |
|
| 970 | if (fd > max_fd)
|
| 971 | return -EINVAL;
|
| 972 |
|
| 973 | if (flags & CLOSE_RANGE_EXCEPT)
|
| 974 | range.flags |= FD_RANGE_EXCEPT;
|
| 975 | if (flags & CLOSE_RANGE_CLOEXEC_ONLY)
|
| 976 | range.flags |= FD_RANGE_CLOEXEC_ONLY;
|
| 977 |
|
| 978 | if ((flags & CLOSE_RANGE_UNSHARE) && atomic_read(&cur_fds->count) > 1) {
|
| 979 | struct fd_range *drop = ⦥
|
| 980 |
|
| 981 | |
| 982 | * If the caller requested all fds to be made cloexec we always
|
| 983 | * copy all of the file descriptors since they still want to
|
| 984 | * use them.
|
| 985 |
|
| 986 | if (flags & CLOSE_RANGE_CLOEXEC)
|
| 987 | drop = NULL;
|
| 988 |
|
| 989 | fds = dup_fd(cur_fds, drop);
|
| 990 | if (IS_ERR(fds))
|
| 991 | return PTR_ERR(fds);
|
| 992 | |
| 993 | * We used to share our file descriptor table, and have now
|
| 994 | * created a private one, make sure we're using it below.
|
| 995 |
|
| 996 | swap(cur_fds, fds);
|
| 997 | }
|
| 998 |
|
| 999 | if (flags & CLOSE_RANGE_CLOEXEC) {
|
| 1000 | __range_cloexec(cur_fds, &range);
|
| 1001 | } else if (!fds) {
|
| 1002 |
|
| 1003 | __range_close(cur_fds, &range);
|
| 1004 | }
|
| 1005 |
|
| 1006 | if (fds) {
|
| 1007 | |
| 1008 | * We're done closing the files we were supposed to. Time to install
|
| 1009 | * the new file descriptor table and drop the old one.
|
| 1010 |
|
| 1011 | switch_files_struct(me, cur_fds);
|
| 1012 | }
|
| 1013 |
|
| 1014 | return 0;
|
| 1015 | }
|
| 1016 |
|
| 1017 | |
| 1018 | * file_close_fd - return file associated with fd
|
| 1019 | * @fd: file descriptor to retrieve file for
|
| 1020 | *
|
| 1021 | * Doesn't take a separate reference count.
|
| 1022 | *
|
| 1023 | * Returns: The file associated with @fd (NULL if @fd is not open)
|
| 1024 |
|
| 1025 | struct file *file_close_fd(unsigned int fd)
|
| 1026 | {
|
| 1027 | struct files_struct *files = current->files;
|
| 1028 | struct file *file;
|
| 1029 |
|
| 1030 | spin_lock(&files->file_lock);
|
| 1031 | file = file_close_fd_locked(files, fd);
|
| 1032 | spin_unlock(&files->file_lock);
|
| 1033 |
|
| 1034 | return file;
|
| 1035 | }
|
| 1036 |
|
| 1037 | void close_cloexec_files(struct files_struct *files)
|
| 1038 | {
|
| 1039 | unsigned i;
|
| 1040 | struct fdtable *fdt;
|
| 1041 |
|
| 1042 |
|
| 1043 | spin_lock(&files->file_lock);
|
| 1044 | fdt = files_fdtable(files);
|
| 1045 |
|
| 1046 | for (i = fdt->max_fds / BITS_PER_LONG; i--; ) {
|
| 1047 | unsigned long set;
|
| 1048 |
|
| 1049 | fdt = files_fdtable(files);
|
| 1050 | set = fdt->close_on_exec[i];
|
| 1051 | if (!set)
|
| 1052 | continue;
|
| 1053 | fdt->close_on_exec[i] = 0;
|
| 1054 | while (set) {
|
| 1055 | unsigned int bit = __fls(set);
|
| 1056 | unsigned fd = i * BITS_PER_LONG + bit;
|
| 1057 | struct file *file;
|
| 1058 |
|
| 1059 | set ^= 1UL << bit;
|
| 1060 | file = fdt->fd[fd];
|
| 1061 | if (!file)
|
| 1062 | continue;
|
| 1063 | rcu_assign_pointer(fdt->fd[fd], NULL);
|
| 1064 | __put_unused_fd(files, fd);
|
| 1065 | spin_unlock(&files->file_lock);
|
| 1066 | filp_close_sync(file, files);
|
| 1067 | cond_resched();
|
| 1068 | spin_lock(&files->file_lock);
|
| 1069 | }
|
| 1070 |
|
| 1071 | }
|
| 1072 | spin_unlock(&files->file_lock);
|
| 1073 | }
|
| 1074 |
|
| 1075 | static struct file *__get_file_rcu(struct file __rcu **f)
|
| 1076 | {
|
| 1077 | struct file __rcu *file;
|
| 1078 | struct file __rcu *file_reloaded;
|
| 1079 | struct file __rcu *file_reloaded_cmp;
|
| 1080 |
|
| 1081 | file = rcu_dereference_raw(*f);
|
| 1082 | if (!file)
|
| 1083 | return NULL;
|
| 1084 |
|
| 1085 | if (unlikely(!file_ref_get(&file->f_ref)))
|
| 1086 | return ERR_PTR(-EAGAIN);
|
| 1087 |
|
| 1088 | file_reloaded = rcu_dereference_raw(*f);
|
| 1089 |
|
| 1090 | |
| 1091 | * Ensure that all accesses have a dependency on the load from
|
| 1092 | * rcu_dereference_raw() above so we get correct ordering
|
| 1093 | * between reuse/allocation and the pointer check below.
|
| 1094 |
|
| 1095 | file_reloaded_cmp = file_reloaded;
|
| 1096 | OPTIMIZER_HIDE_VAR(file_reloaded_cmp);
|
| 1097 |
|
| 1098 | |
| 1099 | * file_ref_get() above provided a full memory barrier when we
|
| 1100 | * acquired a reference.
|
| 1101 | *
|
| 1102 | * This is paired with the write barrier from assigning to the
|
| 1103 | * __rcu protected file pointer so that if that pointer still
|
| 1104 | * matches the current file, we know we have successfully
|
| 1105 | * acquired a reference to the right file.
|
| 1106 | *
|
| 1107 | * If the pointers don't match the file has been reallocated by
|
| 1108 | * SLAB_TYPESAFE_BY_RCU.
|
| 1109 |
|
| 1110 | if (file == file_reloaded_cmp)
|
| 1111 | return file_reloaded;
|
| 1112 |
|
| 1113 | fput(file);
|
| 1114 | return ERR_PTR(-EAGAIN);
|
| 1115 | }
|
| 1116 |
|
| 1117 | |
| 1118 | * get_file_rcu - try go get a reference to a file under rcu
|
| 1119 | * @f: the file to get a reference on
|
| 1120 | *
|
| 1121 | * This function tries to get a reference on @f carefully verifying that
|
| 1122 | * @f hasn't been reused.
|
| 1123 | *
|
| 1124 | * This function should rarely have to be used and only by users who
|
| 1125 | * understand the implications of SLAB_TYPESAFE_BY_RCU. Try to avoid it.
|
| 1126 | *
|
| 1127 | * Return: Returns @f with the reference count increased or NULL.
|
| 1128 |
|
| 1129 | struct file *get_file_rcu(struct file __rcu **f)
|
| 1130 | {
|
| 1131 | for (;;) {
|
| 1132 | struct file __rcu *file;
|
| 1133 |
|
| 1134 | file = __get_file_rcu(f);
|
| 1135 | if (!IS_ERR(file))
|
| 1136 | return file;
|
| 1137 | }
|
| 1138 | }
|
| 1139 | EXPORT_SYMBOL_GPL(get_file_rcu);
|
| 1140 |
|
| 1141 | |
| 1142 | * get_file_active - try go get a reference to a file
|
| 1143 | * @f: the file to get a reference on
|
| 1144 | *
|
| 1145 | * In contast to get_file_rcu() the pointer itself isn't part of the
|
| 1146 | * reference counting.
|
| 1147 | *
|
| 1148 | * This function should rarely have to be used and only by users who
|
| 1149 | * understand the implications of SLAB_TYPESAFE_BY_RCU. Try to avoid it.
|
| 1150 | *
|
| 1151 | * Return: Returns @f with the reference count increased or NULL.
|
| 1152 |
|
| 1153 | struct file *get_file_active(struct file **f)
|
| 1154 | {
|
| 1155 | struct file __rcu *file;
|
| 1156 |
|
| 1157 | rcu_read_lock();
|
| 1158 | file = __get_file_rcu(f);
|
| 1159 | rcu_read_unlock();
|
| 1160 | if (IS_ERR(file))
|
| 1161 | file = NULL;
|
| 1162 | return file;
|
| 1163 | }
|
| 1164 | EXPORT_SYMBOL_GPL(get_file_active);
|
| 1165 |
|
| 1166 | static inline struct file *__fget_files_rcu(struct files_struct *files,
|
| 1167 | unsigned int fd, fmode_t mask)
|
| 1168 | {
|
| 1169 | for (;;) {
|
| 1170 | struct file *file;
|
| 1171 | struct fdtable *fdt = rcu_dereference_raw(files->fdt);
|
| 1172 | struct file __rcu **fdentry;
|
| 1173 | unsigned long nospec_mask;
|
| 1174 |
|
| 1175 |
|
| 1176 | nospec_mask = array_index_mask_nospec(fd, fdt->max_fds);
|
| 1177 |
|
| 1178 | |
| 1179 | * fdentry points to the 'fd' offset, or fdt->fd[0].
|
| 1180 | * Loading from fdt->fd[0] is always safe, because the
|
| 1181 | * array always exists.
|
| 1182 |
|
| 1183 | fdentry = fdt->fd + (fd & nospec_mask);
|
| 1184 |
|
| 1185 |
|
| 1186 | file = rcu_dereference_raw(*fdentry);
|
| 1187 | file = (void *)(nospec_mask & (unsigned long)file);
|
| 1188 | if (unlikely(!file))
|
| 1189 | return NULL;
|
| 1190 |
|
| 1191 | |
| 1192 | * Ok, we have a file pointer that was valid at
|
| 1193 | * some point, but it might have become stale since.
|
| 1194 | *
|
| 1195 | * We need to confirm it by incrementing the refcount
|
| 1196 | * and then check the lookup again.
|
| 1197 | *
|
| 1198 | * file_ref_get() gives us a full memory barrier. We
|
| 1199 | * only really need an 'acquire' one to protect the
|
| 1200 | * loads below, but we don't have that.
|
| 1201 |
|
| 1202 | if (unlikely(!file_ref_get(&file->f_ref)))
|
| 1203 | continue;
|
| 1204 |
|
| 1205 | |
| 1206 | * Such a race can take two forms:
|
| 1207 | *
|
| 1208 | * (a) the file ref already went down to zero and the
|
| 1209 | * file hasn't been reused yet or the file count
|
| 1210 | * isn't zero but the file has already been reused.
|
| 1211 | *
|
| 1212 | * (b) the file table entry has changed under us.
|
| 1213 | * Note that we don't need to re-check the 'fdt->fd'
|
| 1214 | * pointer having changed, because it always goes
|
| 1215 | * hand-in-hand with 'fdt'.
|
| 1216 | *
|
| 1217 | * If so, we need to put our ref and try again.
|
| 1218 |
|
| 1219 | if (unlikely(file != rcu_dereference_raw(*fdentry)) ||
|
| 1220 | unlikely(rcu_dereference_raw(files->fdt) != fdt)) {
|
| 1221 | fput(file);
|
| 1222 | continue;
|
| 1223 | }
|
| 1224 |
|
| 1225 | |
| 1226 | * This isn't the file we're looking for or we're not
|
| 1227 | * allowed to get a reference to it.
|
| 1228 |
|
| 1229 | if (unlikely(file->f_mode & mask)) {
|
| 1230 | fput(file);
|
| 1231 | return NULL;
|
| 1232 | }
|
| 1233 |
|
| 1234 | |
| 1235 | * Ok, we have a ref to the file, and checked that it
|
| 1236 | * still exists.
|
| 1237 |
|
| 1238 | return file;
|
| 1239 | }
|
| 1240 | }
|
| 1241 |
|
| 1242 | static struct file *__fget_files(struct files_struct *files, unsigned int fd,
|
| 1243 | fmode_t mask)
|
| 1244 | {
|
| 1245 | struct file *file;
|
| 1246 |
|
| 1247 | rcu_read_lock();
|
| 1248 | file = __fget_files_rcu(files, fd, mask);
|
| 1249 | rcu_read_unlock();
|
| 1250 |
|
| 1251 | return file;
|
| 1252 | }
|
| 1253 |
|
| 1254 | static inline struct file *__fget(unsigned int fd, fmode_t mask)
|
| 1255 | {
|
| 1256 | return __fget_files(current->files, fd, mask);
|
| 1257 | }
|
| 1258 |
|
| 1259 | struct file *fget(unsigned int fd)
|
| 1260 | {
|
| 1261 | return __fget(fd, FMODE_PATH);
|
| 1262 | }
|
| 1263 | EXPORT_SYMBOL(fget);
|
| 1264 |
|
| 1265 | struct file *fget_raw(unsigned int fd)
|
| 1266 | {
|
| 1267 | return __fget(fd, 0);
|
| 1268 | }
|
| 1269 | EXPORT_SYMBOL(fget_raw);
|
| 1270 |
|
| 1271 | struct file *fget_task(struct task_struct *task, unsigned int fd)
|
| 1272 | {
|
| 1273 | struct file *file = NULL;
|
| 1274 |
|
| 1275 | task_lock(task);
|
| 1276 | if (task->files)
|
| 1277 | file = __fget_files(task->files, fd, 0);
|
| 1278 | task_unlock(task);
|
| 1279 |
|
| 1280 | return file;
|
| 1281 | }
|
| 1282 |
|
| 1283 | struct file *fget_task_next(struct task_struct *task, unsigned int *ret_fd)
|
| 1284 | {
|
| 1285 | struct files_struct *files;
|
| 1286 | unsigned int fd = *ret_fd;
|
| 1287 | struct file *file = NULL;
|
| 1288 |
|
| 1289 | task_lock(task);
|
| 1290 | files = task->files;
|
| 1291 | if (files) {
|
| 1292 | rcu_read_lock();
|
| 1293 | for (; fd < files_fdtable(files)->max_fds; fd++) {
|
| 1294 | file = __fget_files_rcu(files, fd, 0);
|
| 1295 | if (file)
|
| 1296 | break;
|
| 1297 | }
|
| 1298 | rcu_read_unlock();
|
| 1299 | }
|
| 1300 | task_unlock(task);
|
| 1301 | *ret_fd = fd;
|
| 1302 | return file;
|
| 1303 | }
|
| 1304 | EXPORT_SYMBOL(fget_task_next);
|
| 1305 |
|
| 1306 | |
| 1307 | * Lightweight file lookup - no refcnt increment if fd table isn't shared.
|
| 1308 | *
|
| 1309 | * You can use this instead of fget if you satisfy all of the following
|
| 1310 | * conditions:
|
| 1311 | * 1) You must call fput_light before exiting the syscall and returning control
|
| 1312 | * to userspace (i.e. you cannot remember the returned struct file * after
|
| 1313 | * returning to userspace).
|
| 1314 | * 2) You must not call filp_close on the returned struct file * in between
|
| 1315 | * calls to fget_light and fput_light.
|
| 1316 | * 3) You must not clone the current task in between the calls to fget_light
|
| 1317 | * and fput_light.
|
| 1318 | *
|
| 1319 | * The fput_needed flag returned by fget_light should be passed to the
|
| 1320 | * corresponding fput_light.
|
| 1321 | *
|
| 1322 | * (As an exception to rule 2, you can call filp_close between fget_light and
|
| 1323 | * fput_light provided that you capture a real refcount with get_file before
|
| 1324 | * the call to filp_close, and ensure that this real refcount is fput *after*
|
| 1325 | * the fput_light call.)
|
| 1326 | *
|
| 1327 | * See also the documentation in rust/kernel/file.rs.
|
| 1328 |
|
| 1329 | static inline struct fd __fget_light(unsigned int fd, fmode_t mask)
|
| 1330 | {
|
| 1331 | struct files_struct *files = current->files;
|
| 1332 | struct file *file;
|
| 1333 |
|
| 1334 | |
| 1335 | * If another thread is concurrently calling close_fd() followed
|
| 1336 | * by put_files_struct(), we must not observe the old table
|
| 1337 | * entry combined with the new refcount - otherwise we could
|
| 1338 | * return a file that is concurrently being freed.
|
| 1339 | *
|
| 1340 | * atomic_read_acquire() pairs with atomic_dec_and_test() in
|
| 1341 | * put_files_struct().
|
| 1342 |
|
| 1343 | if (likely(atomic_read_acquire(&files->count) == 1)) {
|
| 1344 | file = files_lookup_fd_raw(files, fd);
|
| 1345 | if (!file || unlikely(file->f_mode & mask))
|
| 1346 | return EMPTY_FD;
|
| 1347 | return BORROWED_FD(file);
|
| 1348 | } else {
|
| 1349 | file = __fget_files(files, fd, mask);
|
| 1350 | if (!file)
|
| 1351 | return EMPTY_FD;
|
| 1352 | return CLONED_FD(file);
|
| 1353 | }
|
| 1354 | }
|
| 1355 | struct fd fdget(unsigned int fd)
|
| 1356 | {
|
| 1357 | return __fget_light(fd, FMODE_PATH);
|
| 1358 | }
|
| 1359 | EXPORT_SYMBOL(fdget);
|
| 1360 |
|
| 1361 | struct fd fdget_raw(unsigned int fd)
|
| 1362 | {
|
| 1363 | return __fget_light(fd, 0);
|
| 1364 | }
|
| 1365 |
|
| 1366 | |
| 1367 | * Try to avoid f_pos locking. We only need it if the
|
| 1368 | * file is marked for FMODE_ATOMIC_POS, and it can be
|
| 1369 | * accessed multiple ways.
|
| 1370 | *
|
| 1371 | * Always do it for directories, because pidfd_getfd()
|
| 1372 | * can make a file accessible even if it otherwise would
|
| 1373 | * not be, and for directories this is a correctness
|
| 1374 | * issue, not a "POSIX requirement".
|
| 1375 |
|
| 1376 | static inline bool file_needs_f_pos_lock(struct file *file)
|
| 1377 | {
|
| 1378 | if (!(file->f_mode & FMODE_ATOMIC_POS))
|
| 1379 | return false;
|
| 1380 | if (__file_ref_read_raw(&file->f_ref) != FILE_REF_ONEREF)
|
| 1381 | return true;
|
| 1382 | if (file->f_op->iterate_shared)
|
| 1383 | return true;
|
| 1384 | return false;
|
| 1385 | }
|
| 1386 |
|
| 1387 | bool file_seek_cur_needs_f_lock(struct file *file)
|
| 1388 | {
|
| 1389 | if (!(file->f_mode & FMODE_ATOMIC_POS) && !file->f_op->iterate_shared)
|
| 1390 | return false;
|
| 1391 |
|
| 1392 | |
| 1393 | * Note that we are not guaranteed to be called after fdget_pos() on
|
| 1394 | * this file obj, in which case the caller is expected to provide the
|
| 1395 | * appropriate locking.
|
| 1396 |
|
| 1397 |
|
| 1398 | return true;
|
| 1399 | }
|
| 1400 |
|
| 1401 | struct fd fdget_pos(unsigned int fd)
|
| 1402 | {
|
| 1403 | struct fd f = fdget(fd);
|
| 1404 | struct file *file = fd_file(f);
|
| 1405 |
|
| 1406 | if (likely(file) && file_needs_f_pos_lock(file)) {
|
| 1407 | f.word |= FDPUT_POS_UNLOCK;
|
| 1408 | mutex_lock(&file->f_pos_lock);
|
| 1409 | }
|
| 1410 | return f;
|
| 1411 | }
|
| 1412 |
|
| 1413 | void __f_unlock_pos(struct file *f)
|
| 1414 | {
|
| 1415 | mutex_unlock(&f->f_pos_lock);
|
| 1416 | }
|
| 1417 |
|
| 1418 | |
| 1419 | * We only lock f_pos if we have threads or if the file might be
|
| 1420 | * shared with another process. In both cases we'll have an elevated
|
| 1421 | * file count (done either by fdget() or by fork()).
|
| 1422 |
|
| 1423 |
|
| 1424 | void set_close_on_exec(unsigned int fd, int flag)
|
| 1425 | {
|
| 1426 | struct files_struct *files = current->files;
|
| 1427 | spin_lock(&files->file_lock);
|
| 1428 | __set_close_on_exec(fd, files_fdtable(files), flag);
|
| 1429 | spin_unlock(&files->file_lock);
|
| 1430 | }
|
| 1431 |
|
| 1432 | bool get_close_on_exec(unsigned int fd)
|
| 1433 | {
|
| 1434 | bool res;
|
| 1435 | rcu_read_lock();
|
| 1436 | res = close_on_exec(fd, current->files);
|
| 1437 | rcu_read_unlock();
|
| 1438 | return res;
|
| 1439 | }
|
| 1440 |
|
| 1441 | static int do_dup2(struct files_struct *files,
|
| 1442 | struct file *file, unsigned fd, unsigned flags)
|
| 1443 | __releases(&files->file_lock)
|
| 1444 | {
|
| 1445 | struct file *tofree;
|
| 1446 | struct fdtable *fdt;
|
| 1447 |
|
| 1448 | |
| 1449 | * dup2() is expected to close the file installed in the target fd slot
|
| 1450 | * (if any). However, userspace hand-picking a fd may be racing against
|
| 1451 | * its own threads which happened to allocate it in open() et al but did
|
| 1452 | * not populate it yet.
|
| 1453 | *
|
| 1454 | * Broadly speaking we may be racing against the following:
|
| 1455 | * fd = get_unused_fd_flags(); // fd slot reserved, ->fd[fd] == NULL
|
| 1456 | * file = hard_work_goes_here();
|
| 1457 | * fd_install(fd, file); // only now ->fd[fd] == file
|
| 1458 | *
|
| 1459 | * It is an invariant that a successfully allocated fd has a NULL entry
|
| 1460 | * in the array until the matching fd_install().
|
| 1461 | *
|
| 1462 | * If we fit the window, we have the fd to populate, yet no target file
|
| 1463 | * to close. Trying to ignore it and install our new file would violate
|
| 1464 | * the invariant and make fd_install() overwrite our file.
|
| 1465 | *
|
| 1466 | * Things can be done(tm) to handle this. However, the issue does not
|
| 1467 | * concern legitimate programs and we only need to make sure the kernel
|
| 1468 | * does not trip over it.
|
| 1469 | *
|
| 1470 | * The simplest way out is to return an error if we find ourselves here.
|
| 1471 | *
|
| 1472 | * POSIX is silent on the issue, we return -EBUSY.
|
| 1473 |
|
| 1474 | fdt = files_fdtable(files);
|
| 1475 | fd = array_index_nospec(fd, fdt->max_fds);
|
| 1476 | tofree = rcu_dereference_raw(fdt->fd[fd]);
|
| 1477 | if (!tofree && fd_is_open(fd, fdt))
|
| 1478 | goto Ebusy;
|
| 1479 | get_file(file);
|
| 1480 | rcu_assign_pointer(fdt->fd[fd], file);
|
| 1481 | __set_open_fd(fd, fdt, flags & O_CLOEXEC);
|
| 1482 | spin_unlock(&files->file_lock);
|
| 1483 |
|
| 1484 | if (tofree)
|
| 1485 | filp_close(tofree, files);
|
| 1486 |
|
| 1487 | return fd;
|
| 1488 |
|
| 1489 | Ebusy:
|
| 1490 | spin_unlock(&files->file_lock);
|
| 1491 | return -EBUSY;
|
| 1492 | }
|
| 1493 |
|
| 1494 | int replace_fd(unsigned fd, struct file *file, unsigned flags)
|
| 1495 | {
|
| 1496 | int err;
|
| 1497 | struct files_struct *files = current->files;
|
| 1498 |
|
| 1499 | if (!file)
|
| 1500 | return close_fd(fd);
|
| 1501 |
|
| 1502 | if (fd >= rlimit(RLIMIT_NOFILE))
|
| 1503 | return -EBADF;
|
| 1504 |
|
| 1505 | spin_lock(&files->file_lock);
|
| 1506 | err = expand_files(files, fd);
|
| 1507 | if (unlikely(err < 0))
|
| 1508 | goto out_unlock;
|
| 1509 | err = do_dup2(files, file, fd, flags);
|
| 1510 | if (err < 0)
|
| 1511 | return err;
|
| 1512 | return 0;
|
| 1513 |
|
| 1514 | out_unlock:
|
| 1515 | spin_unlock(&files->file_lock);
|
| 1516 | return err;
|
| 1517 | }
|
| 1518 |
|
| 1519 | |
| 1520 | * receive_fd() - Install received file into file descriptor table
|
| 1521 | * @file: struct file that was received from another process
|
| 1522 | * @ufd: __user pointer to write new fd number to
|
| 1523 | * @o_flags: the O_* flags to apply to the new fd entry
|
| 1524 | *
|
| 1525 | * Installs a received file into the file descriptor table, with appropriate
|
| 1526 | * checks and count updates. Optionally writes the fd number to userspace, if
|
| 1527 | * @ufd is non-NULL.
|
| 1528 | *
|
| 1529 | * This helper handles its own reference counting of the incoming
|
| 1530 | * struct file.
|
| 1531 | *
|
| 1532 | * Returns newly install fd or -ve on error.
|
| 1533 |
|
| 1534 | int receive_fd(struct file *file, int __user *ufd, unsigned int o_flags)
|
| 1535 | {
|
| 1536 | int error;
|
| 1537 |
|
| 1538 | error = security_file_receive(file);
|
| 1539 | if (error)
|
| 1540 | return error;
|
| 1541 |
|
| 1542 | FD_PREPARE(fdf, o_flags, file);
|
| 1543 | if (fdf->fd < 0)
|
| 1544 | return fdf->fd;
|
| 1545 | get_file(file);
|
| 1546 |
|
| 1547 | if (ufd) {
|
| 1548 | error = put_user(fdf->fd, ufd);
|
| 1549 | if (error)
|
| 1550 | return error;
|
| 1551 | }
|
| 1552 |
|
| 1553 | __receive_sock(fdf->file);
|
| 1554 | return fd_publish(fdf);
|
| 1555 | }
|
| 1556 | EXPORT_SYMBOL_GPL(receive_fd);
|
| 1557 |
|
| 1558 | int receive_fd_replace(int new_fd, struct file *file, unsigned int o_flags)
|
| 1559 | {
|
| 1560 | int error;
|
| 1561 |
|
| 1562 | error = security_file_receive(file);
|
| 1563 | if (error)
|
| 1564 | return error;
|
| 1565 | error = replace_fd(new_fd, file, o_flags);
|
| 1566 | if (error)
|
| 1567 | return error;
|
| 1568 | __receive_sock(file);
|
| 1569 | return new_fd;
|
| 1570 | }
|
| 1571 |
|
| 1572 | static int ksys_dup3(unsigned int oldfd, unsigned int newfd, int flags)
|
| 1573 | {
|
| 1574 | int err = -EBADF;
|
| 1575 | struct file *file;
|
| 1576 | struct files_struct *files = current->files;
|
| 1577 |
|
| 1578 | if ((flags & ~O_CLOEXEC) != 0)
|
| 1579 | return -EINVAL;
|
| 1580 |
|
| 1581 | if (unlikely(oldfd == newfd))
|
| 1582 | return -EINVAL;
|
| 1583 |
|
| 1584 | if (newfd >= rlimit(RLIMIT_NOFILE))
|
| 1585 | return -EBADF;
|
| 1586 |
|
| 1587 | spin_lock(&files->file_lock);
|
| 1588 | err = expand_files(files, newfd);
|
| 1589 | file = files_lookup_fd_locked(files, oldfd);
|
| 1590 | if (unlikely(!file))
|
| 1591 | goto Ebadf;
|
| 1592 | if (unlikely(err < 0)) {
|
| 1593 | if (err == -EMFILE)
|
| 1594 | goto Ebadf;
|
| 1595 | goto out_unlock;
|
| 1596 | }
|
| 1597 | return do_dup2(files, file, newfd, flags);
|
| 1598 |
|
| 1599 | Ebadf:
|
| 1600 | err = -EBADF;
|
| 1601 | out_unlock:
|
| 1602 | spin_unlock(&files->file_lock);
|
| 1603 | return err;
|
| 1604 | }
|
| 1605 |
|
| 1606 | SYSCALL_DEFINE3(dup3, unsigned int, oldfd, unsigned int, newfd, int, flags)
|
| 1607 | {
|
| 1608 | return ksys_dup3(oldfd, newfd, flags);
|
| 1609 | }
|
| 1610 |
|
| 1611 | SYSCALL_DEFINE2(dup2, unsigned int, oldfd, unsigned int, newfd)
|
| 1612 | {
|
| 1613 | if (unlikely(newfd == oldfd)) {
|
| 1614 | struct files_struct *files = current->files;
|
| 1615 | struct file *f;
|
| 1616 | int retval = oldfd;
|
| 1617 |
|
| 1618 | rcu_read_lock();
|
| 1619 | f = __fget_files_rcu(files, oldfd, 0);
|
| 1620 | if (!f)
|
| 1621 | retval = -EBADF;
|
| 1622 | rcu_read_unlock();
|
| 1623 | if (f)
|
| 1624 | fput(f);
|
| 1625 | return retval;
|
| 1626 | }
|
| 1627 | return ksys_dup3(oldfd, newfd, 0);
|
| 1628 | }
|
| 1629 |
|
| 1630 | SYSCALL_DEFINE1(dup, unsigned int, fildes)
|
| 1631 | {
|
| 1632 | int ret = -EBADF;
|
| 1633 | struct file *file = fget_raw(fildes);
|
| 1634 |
|
| 1635 | if (file) {
|
| 1636 | ret = get_unused_fd_flags(0);
|
| 1637 | if (ret >= 0)
|
| 1638 | fd_install(ret, file);
|
| 1639 | else
|
| 1640 | fput(file);
|
| 1641 | }
|
| 1642 | return ret;
|
| 1643 | }
|
| 1644 |
|
| 1645 | int f_dupfd(unsigned int from, struct file *file, unsigned flags)
|
| 1646 | {
|
| 1647 | unsigned long nofile = rlimit(RLIMIT_NOFILE);
|
| 1648 | int err;
|
| 1649 | if (from >= nofile)
|
| 1650 | return -EINVAL;
|
| 1651 | err = alloc_fd(from, nofile, flags);
|
| 1652 | if (err >= 0) {
|
| 1653 | get_file(file);
|
| 1654 | fd_install(err, file);
|
| 1655 | }
|
| 1656 | return err;
|
| 1657 | }
|
| 1658 |
|
| 1659 | int iterate_fd(struct files_struct *files, unsigned n,
|
| 1660 | int (*f)(const void *, struct file *, unsigned),
|
| 1661 | const void *p)
|
| 1662 | {
|
| 1663 | struct fdtable *fdt;
|
| 1664 | int res = 0;
|
| 1665 | if (!files)
|
| 1666 | return 0;
|
| 1667 | spin_lock(&files->file_lock);
|
| 1668 | for (fdt = files_fdtable(files); n < fdt->max_fds; n++) {
|
| 1669 | struct file *file;
|
| 1670 | file = rcu_dereference_check_fdtable(files, fdt->fd[n]);
|
| 1671 | if (!file)
|
| 1672 | continue;
|
| 1673 | res = f(p, file, n);
|
| 1674 | if (res)
|
| 1675 | break;
|
| 1676 | }
|
| 1677 | spin_unlock(&files->file_lock);
|
| 1678 | return res;
|
| 1679 | }
|
| 1680 | EXPORT_SYMBOL(iterate_fd);
|
| 1681 |
|
| 1682 | #ifdef CONFIG_FDTABLE_KUNIT_TEST
|
| 1683 | #include "tests/fdtable_kunit.c"
|
| 1684 | #endif
|