projects/linux·blob 2c455ddc · refs

1822 lines · 43.8 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * linux/fs/read_write.c
4 *
5 * Copyright (C) 1991, 1992 Linus Torvalds
6 */
7
8#include <linux/slab.h>
9#include <linux/stat.h>
10#include <linux/sched/xacct.h>
11#include <linux/fcntl.h>
12#include <linux/file.h>
13#include <linux/uio.h>
14#include <linux/fsnotify.h>
15#include <linux/security.h>
16#include <linux/export.h>
17#include <linux/syscalls.h>
18#include <linux/pagemap.h>
19#include <linux/splice.h>
20#include <linux/compat.h>
21#include <linux/mount.h>
22#include <linux/fs.h>
23#include <linux/filelock.h>
24#include "internal.h"
25
26#include <linux/uaccess.h>
27#include <asm/unistd.h>
28
29const struct file_operations generic_ro_fops = {
30 .llseek = generic_file_llseek,
31 .read_iter = generic_file_read_iter,
32 .mmap_prepare = generic_file_readonly_mmap_prepare,
33 .splice_read = filemap_splice_read,
34 .setlease = generic_setlease,
35};
36
37EXPORT_SYMBOL(generic_ro_fops);
38
39static inline bool unsigned_offsets(struct file *file)
40{
41 return file->f_op->fop_flags & FOP_UNSIGNED_OFFSET;
42}
43
44/**
45 * vfs_setpos_cookie - update the file offset for lseek and reset cookie
46 * @file: file structure in question
47 * @offset: file offset to seek to
48 * @maxsize: maximum file size
49 * @cookie: cookie to reset
50 *
51 * Update the file offset to the value specified by @offset if the given
52 * offset is valid and it is not equal to the current file offset and
53 * reset the specified cookie to indicate that a seek happened.
54 *
55 * Return the specified offset on success and -EINVAL on invalid offset.
56 */
57static loff_t vfs_setpos_cookie(struct file *file, loff_t offset,
58 loff_t maxsize, u64 *cookie)
59{
60 if (offset < 0 && !unsigned_offsets(file))
61 return -EINVAL;
62 if (offset > maxsize)
63 return -EINVAL;
64
65 if (offset != file->f_pos) {
66 file->f_pos = offset;
67 if (cookie)
68 *cookie = 0;
69 }
70 return offset;
71}
72
73/**
74 * vfs_setpos - update the file offset for lseek
75 * @file: file structure in question
76 * @offset: file offset to seek to
77 * @maxsize: maximum file size
78 *
79 * This is a low-level filesystem helper for updating the file offset to
80 * the value specified by @offset if the given offset is valid and it is
81 * not equal to the current file offset.
82 *
83 * Return the specified offset on success and -EINVAL on invalid offset.
84 */
85loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize)
86{
87 return vfs_setpos_cookie(file, offset, maxsize, NULL);
88}
89EXPORT_SYMBOL(vfs_setpos);
90
91/**
92 * must_set_pos - check whether f_pos has to be updated
93 * @file: file to seek on
94 * @offset: offset to use
95 * @whence: type of seek operation
96 * @eof: end of file
97 *
98 * Check whether f_pos needs to be updated and update @offset according
99 * to @whence.
100 *
101 * Return: 0 if f_pos doesn't need to be updated, 1 if f_pos has to be
102 * updated, and negative error code on failure.
103 */
104static int must_set_pos(struct file *file, loff_t *offset, int whence, loff_t eof)
105{
106 switch (whence) {
107 case SEEK_END:
108 *offset += eof;
109 break;
110 case SEEK_CUR:
111 /*
112 * Here we special-case the lseek(fd, 0, SEEK_CUR)
113 * position-querying operation. Avoid rewriting the "same"
114 * f_pos value back to the file because a concurrent read(),
115 * write() or lseek() might have altered it
116 */
117 if (*offset == 0) {
118 *offset = file->f_pos;
119 return 0;
120 }
121 break;
122 case SEEK_DATA:
123 /*
124 * In the generic case the entire file is data, so as long as
125 * offset isn't at the end of the file then the offset is data.
126 */
127 if ((unsigned long long)*offset >= eof)
128 return -ENXIO;
129 break;
130 case SEEK_HOLE:
131 /*
132 * There is a virtual hole at the end of the file, so as long as
133 * offset isn't i_size or larger, return i_size.
134 */
135 if ((unsigned long long)*offset >= eof)
136 return -ENXIO;
137 *offset = eof;
138 break;
139 }
140
141 return 1;
142}
143
144/**
145 * generic_file_llseek_size - generic llseek implementation for regular files
146 * @file: file structure to seek on
147 * @offset: file offset to seek to
148 * @whence: type of seek
149 * @maxsize: max size of this file in file system
150 * @eof: offset used for SEEK_END position
151 *
152 * This is a variant of generic_file_llseek that allows passing in a custom
153 * maximum file size and a custom EOF position, for e.g. hashed directories
154 *
155 * Synchronization:
156 * SEEK_SET and SEEK_END are unsynchronized (but atomic on 64bit platforms)
157 * SEEK_CUR is synchronized against other SEEK_CURs, but not read/writes.
158 * read/writes behave like SEEK_SET against seeks.
159 */
160loff_t
161generic_file_llseek_size(struct file *file, loff_t offset, int whence,
162 loff_t maxsize, loff_t eof)
163{
164 int ret;
165
166 ret = must_set_pos(file, &offset, whence, eof);
167 if (ret < 0)
168 return ret;
169 if (ret == 0)
170 return offset;
171
172 if (whence == SEEK_CUR) {
173 /*
174 * If the file requires locking via f_pos_lock we know
175 * that mutual exclusion for SEEK_CUR on the same file
176 * is guaranteed. If the file isn't locked, we take
177 * f_lock to protect against f_pos races with other
178 * SEEK_CURs.
179 */
180 if (file_seek_cur_needs_f_lock(file)) {
181 guard(spinlock)(&file->f_lock);
182 return vfs_setpos(file, file->f_pos + offset, maxsize);
183 }
184 return vfs_setpos(file, file->f_pos + offset, maxsize);
185 }
186
187 return vfs_setpos(file, offset, maxsize);
188}
189EXPORT_SYMBOL(generic_file_llseek_size);
190
191/**
192 * generic_llseek_cookie - versioned llseek implementation
193 * @file: file structure to seek on
194 * @offset: file offset to seek to
195 * @whence: type of seek
196 * @cookie: cookie to update
197 *
198 * See generic_file_llseek for a general description and locking assumptions.
199 *
200 * In contrast to generic_file_llseek, this function also resets a
201 * specified cookie to indicate a seek took place.
202 */
203loff_t generic_llseek_cookie(struct file *file, loff_t offset, int whence,
204 u64 *cookie)
205{
206 struct inode *inode = file->f_mapping->host;
207 loff_t maxsize = inode->i_sb->s_maxbytes;
208 loff_t eof = i_size_read(inode);
209 int ret;
210
211 if (WARN_ON_ONCE(!cookie))
212 return -EINVAL;
213
214 /*
215 * Require that this is only used for directories that guarantee
216 * synchronization between readdir and seek so that an update to
217 * @cookie is correctly synchronized with concurrent readdir.
218 */
219 if (WARN_ON_ONCE(!(file->f_mode & FMODE_ATOMIC_POS)))
220 return -EINVAL;
221
222 ret = must_set_pos(file, &offset, whence, eof);
223 if (ret < 0)
224 return ret;
225 if (ret == 0)
226 return offset;
227
228 /* No need to hold f_lock because we know that f_pos_lock is held. */
229 if (whence == SEEK_CUR)
230 return vfs_setpos_cookie(file, file->f_pos + offset, maxsize, cookie);
231
232 return vfs_setpos_cookie(file, offset, maxsize, cookie);
233}
234EXPORT_SYMBOL(generic_llseek_cookie);
235
236/**
237 * generic_file_llseek - generic llseek implementation for regular files
238 * @file: file structure to seek on
239 * @offset: file offset to seek to
240 * @whence: type of seek
241 *
242 * This is a generic implementation of ->llseek useable for all normal local
243 * filesystems. It just updates the file offset to the value specified by
244 * @offset and @whence.
245 */
246loff_t generic_file_llseek(struct file *file, loff_t offset, int whence)
247{
248 struct inode *inode = file->f_mapping->host;
249
250 return generic_file_llseek_size(file, offset, whence,
251 inode->i_sb->s_maxbytes,
252 i_size_read(inode));
253}
254EXPORT_SYMBOL(generic_file_llseek);
255
256/**
257 * fixed_size_llseek - llseek implementation for fixed-sized devices
258 * @file: file structure to seek on
259 * @offset: file offset to seek to
260 * @whence: type of seek
261 * @size: size of the file
262 *
263 */
264loff_t fixed_size_llseek(struct file *file, loff_t offset, int whence, loff_t size)
265{
266 switch (whence) {
267 case SEEK_SET: case SEEK_CUR: case SEEK_END:
268 return generic_file_llseek_size(file, offset, whence,
269 size, size);
270 default:
271 return -EINVAL;
272 }
273}
274EXPORT_SYMBOL(fixed_size_llseek);
275
276/**
277 * no_seek_end_llseek - llseek implementation for files without SEEK_END
278 * @file: file structure to seek on
279 * @offset: file offset to seek to
280 * @whence: type of seek
281 *
282 */
283loff_t no_seek_end_llseek(struct file *file, loff_t offset, int whence)
284{
285 switch (whence) {
286 case SEEK_SET: case SEEK_CUR:
287 return generic_file_llseek_size(file, offset, whence,
288 OFFSET_MAX, 0);
289 default:
290 return -EINVAL;
291 }
292}
293EXPORT_SYMBOL(no_seek_end_llseek);
294
295/**
296 * no_seek_end_llseek_size - llseek implementation for files without SEEK_END
297 * @file: file structure to seek on
298 * @offset: file offset to seek to
299 * @whence: type of seek
300 * @size: maximal offset allowed
301 *
302 */
303loff_t no_seek_end_llseek_size(struct file *file, loff_t offset, int whence, loff_t size)
304{
305 switch (whence) {
306 case SEEK_SET: case SEEK_CUR:
307 return generic_file_llseek_size(file, offset, whence,
308 size, 0);
309 default:
310 return -EINVAL;
311 }
312}
313EXPORT_SYMBOL(no_seek_end_llseek_size);
314
315/**
316 * noop_llseek - No Operation Performed llseek implementation
317 * @file: file structure to seek on
318 * @offset: file offset to seek to
319 * @whence: type of seek
320 *
321 * This is an implementation of ->llseek useable for the rare special case when
322 * userspace expects the seek to succeed but the (device) file is actually not
323 * able to perform the seek. In this case you use noop_llseek() instead of
324 * falling back to the default implementation of ->llseek.
325 */
326loff_t noop_llseek(struct file *file, loff_t offset, int whence)
327{
328 return file->f_pos;
329}
330EXPORT_SYMBOL(noop_llseek);
331
332loff_t default_llseek(struct file *file, loff_t offset, int whence)
333{
334 struct inode *inode = file_inode(file);
335 loff_t retval;
336
337 retval = inode_lock_killable(inode);
338 if (retval)
339 return retval;
340 switch (whence) {
341 case SEEK_END:
342 offset += i_size_read(inode);
343 break;
344 case SEEK_CUR:
345 if (offset == 0) {
346 retval = file->f_pos;
347 goto out;
348 }
349 offset += file->f_pos;
350 break;
351 case SEEK_DATA:
352 /*
353 * In the generic case the entire file is data, so as
354 * long as offset isn't at the end of the file then the
355 * offset is data.
356 */
357 if (offset >= inode->i_size) {
358 retval = -ENXIO;
359 goto out;
360 }
361 break;
362 case SEEK_HOLE:
363 /*
364 * There is a virtual hole at the end of the file, so
365 * as long as offset isn't i_size or larger, return
366 * i_size.
367 */
368 if (offset >= inode->i_size) {
369 retval = -ENXIO;
370 goto out;
371 }
372 offset = inode->i_size;
373 break;
374 }
375 retval = -EINVAL;
376 if (offset >= 0 || unsigned_offsets(file)) {
377 if (offset != file->f_pos)
378 file->f_pos = offset;
379 retval = offset;
380 }
381out:
382 inode_unlock(inode);
383 return retval;
384}
385EXPORT_SYMBOL(default_llseek);
386
387loff_t vfs_llseek(struct file *file, loff_t offset, int whence)
388{
389 if (!(file->f_mode & FMODE_LSEEK))
390 return -ESPIPE;
391 return file->f_op->llseek(file, offset, whence);
392}
393EXPORT_SYMBOL(vfs_llseek);
394
395static off_t ksys_lseek(unsigned int fd, off_t offset, unsigned int whence)
396{
397 off_t retval;
398 CLASS(fd_pos, f)(fd);
399 if (fd_empty(f))
400 return -EBADF;
401
402 retval = -EINVAL;
403 if (whence <= SEEK_MAX) {
404 loff_t res = vfs_llseek(fd_file(f), offset, whence);
405 retval = res;
406 if (res != (loff_t)retval)
407 retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */
408 }
409 return retval;
410}
411
412SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, whence)
413{
414 return ksys_lseek(fd, offset, whence);
415}
416
417#ifdef CONFIG_COMPAT
418COMPAT_SYSCALL_DEFINE3(lseek, unsigned int, fd, compat_off_t, offset, unsigned int, whence)
419{
420 return ksys_lseek(fd, offset, whence);
421}
422#endif
423
424#if !defined(CONFIG_64BIT) || defined(CONFIG_COMPAT) || \
425 defined(__ARCH_WANT_SYS_LLSEEK)
426SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
427 unsigned long, offset_low, loff_t __user *, result,
428 unsigned int, whence)
429{
430 int retval;
431 CLASS(fd_pos, f)(fd);
432 loff_t offset;
433
434 if (fd_empty(f))
435 return -EBADF;
436
437 if (whence > SEEK_MAX)
438 return -EINVAL;
439
440 offset = vfs_llseek(fd_file(f), ((loff_t) offset_high << 32) | offset_low,
441 whence);
442
443 retval = (int)offset;
444 if (offset >= 0) {
445 retval = -EFAULT;
446 if (!copy_to_user(result, &offset, sizeof(offset)))
447 retval = 0;
448 }
449 return retval;
450}
451#endif
452
453int rw_verify_area(int read_write, struct file *file, const loff_t *ppos, size_t count)
454{
455 int mask = read_write == READ ? MAY_READ : MAY_WRITE;
456 int ret;
457
458 if (unlikely((ssize_t) count < 0))
459 return -EINVAL;
460
461 if (ppos) {
462 loff_t pos = *ppos;
463
464 if (unlikely(pos < 0)) {
465 if (!unsigned_offsets(file))
466 return -EINVAL;
467 if (count >= -pos) /* both values are in 0..LLONG_MAX */
468 return -EOVERFLOW;
469 } else if (unlikely((loff_t) (pos + count) < 0)) {
470 if (!unsigned_offsets(file))
471 return -EINVAL;
472 }
473 }
474
475 ret = security_file_permission(file, mask);
476 if (ret)
477 return ret;
478
479 return fsnotify_file_area_perm(file, mask, ppos, count);
480}
481EXPORT_SYMBOL(rw_verify_area);
482
483static ssize_t new_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
484{
485 struct kiocb kiocb;
486 struct iov_iter iter;
487 ssize_t ret;
488
489 init_sync_kiocb(&kiocb, filp);
490 kiocb.ki_pos = (ppos ? *ppos : 0);
491 iov_iter_ubuf(&iter, ITER_DEST, buf, len);
492
493 ret = filp->f_op->read_iter(&kiocb, &iter);
494 BUG_ON(ret == -EIOCBQUEUED);
495 if (ppos)
496 *ppos = kiocb.ki_pos;
497 return ret;
498}
499
500static int warn_unsupported(struct file *file, const char *op)
501{
502 pr_warn_ratelimited(
503 "kernel %s not supported for file %pD4 (pid: %d comm: %.20s)\n",
504 op, file, current->pid, current->comm);
505 return -EINVAL;
506}
507
508ssize_t __kernel_read(struct file *file, void *buf, size_t count, loff_t *pos)
509{
510 struct kvec iov = {
511 .iov_base = buf,
512 .iov_len = min_t(size_t, count, MAX_RW_COUNT),
513 };
514 struct kiocb kiocb;
515 struct iov_iter iter;
516 ssize_t ret;
517
518 if (WARN_ON_ONCE(!(file->f_mode & FMODE_READ)))
519 return -EINVAL;
520 if (!(file->f_mode & FMODE_CAN_READ))
521 return -EINVAL;
522 /*
523 * Also fail if ->read_iter and ->read are both wired up as that
524 * implies very convoluted semantics.
525 */
526 if (unlikely(!file->f_op->read_iter || file->f_op->read))
527 return warn_unsupported(file, "read");
528
529 init_sync_kiocb(&kiocb, file);
530 kiocb.ki_pos = pos ? *pos : 0;
531 iov_iter_kvec(&iter, ITER_DEST, &iov, 1, iov.iov_len);
532 ret = file->f_op->read_iter(&kiocb, &iter);
533 if (ret > 0) {
534 if (pos)
535 *pos = kiocb.ki_pos;
536 fsnotify_access(file);
537 add_rchar(current, ret);
538 }
539 inc_syscr(current);
540 return ret;
541}
542
543ssize_t kernel_read(struct file *file, void *buf, size_t count, loff_t *pos)
544{
545 ssize_t ret;
546
547 ret = rw_verify_area(READ, file, pos, count);
548 if (ret)
549 return ret;
550 return __kernel_read(file, buf, count, pos);
551}
552EXPORT_SYMBOL(kernel_read);
553
554ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
555{
556 ssize_t ret;
557
558 if (!(file->f_mode & FMODE_READ))
559 return -EBADF;
560 if (!(file->f_mode & FMODE_CAN_READ))
561 return -EINVAL;
562 if (unlikely(!access_ok(buf, count)))
563 return -EFAULT;
564
565 ret = rw_verify_area(READ, file, pos, count);
566 if (ret)
567 return ret;
568 if (count > MAX_RW_COUNT)
569 count = MAX_RW_COUNT;
570
571 if (file->f_op->read)
572 ret = file->f_op->read(file, buf, count, pos);
573 else if (file->f_op->read_iter)
574 ret = new_sync_read(file, buf, count, pos);
575 else
576 ret = -EINVAL;
577 if (ret > 0) {
578 fsnotify_access(file);
579 add_rchar(current, ret);
580 }
581 inc_syscr(current);
582 return ret;
583}
584
585static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
586{
587 struct kiocb kiocb;
588 struct iov_iter iter;
589 ssize_t ret;
590
591 init_sync_kiocb(&kiocb, filp);
592 kiocb.ki_pos = (ppos ? *ppos : 0);
593 iov_iter_ubuf(&iter, ITER_SOURCE, (void __user *)buf, len);
594
595 ret = filp->f_op->write_iter(&kiocb, &iter);
596 BUG_ON(ret == -EIOCBQUEUED);
597 if (ret > 0 && ppos)
598 *ppos = kiocb.ki_pos;
599 return ret;
600}
601
602/* caller is responsible for file_start_write/file_end_write */
603ssize_t __kernel_write_iter(struct file *file, struct iov_iter *from, loff_t *pos)
604{
605 struct kiocb kiocb;
606 ssize_t ret;
607
608 if (WARN_ON_ONCE(!(file->f_mode & FMODE_WRITE)))
609 return -EBADF;
610 if (!(file->f_mode & FMODE_CAN_WRITE))
611 return -EINVAL;
612 /*
613 * Also fail if ->write_iter and ->write are both wired up as that
614 * implies very convoluted semantics.
615 */
616 if (unlikely(!file->f_op->write_iter || file->f_op->write))
617 return warn_unsupported(file, "write");
618
619 init_sync_kiocb(&kiocb, file);
620 kiocb.ki_pos = pos ? *pos : 0;
621 ret = file->f_op->write_iter(&kiocb, from);
622 if (ret > 0) {
623 if (pos)
624 *pos = kiocb.ki_pos;
625 fsnotify_modify(file);
626 add_wchar(current, ret);
627 }
628 inc_syscw(current);
629 return ret;
630}
631
632/* caller is responsible for file_start_write/file_end_write */
633ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos)
634{
635 struct kvec iov = {
636 .iov_base = (void *)buf,
637 .iov_len = min_t(size_t, count, MAX_RW_COUNT),
638 };
639 struct iov_iter iter;
640 iov_iter_kvec(&iter, ITER_SOURCE, &iov, 1, iov.iov_len);
641 return __kernel_write_iter(file, &iter, pos);
642}
643/*
644 * autofs is one of the few internal kernel users that actually
645 * wants this _and_ can be built as a module. So we need to export
646 * this symbol for autofs, even though it really isn't appropriate
647 * for any other kernel modules.
648 */
649EXPORT_SYMBOL_FOR_MODULES(__kernel_write, "autofs4");
650
651ssize_t kernel_write(struct file *file, const void *buf, size_t count,
652 loff_t *pos)
653{
654 ssize_t ret;
655
656 ret = rw_verify_area(WRITE, file, pos, count);
657 if (ret)
658 return ret;
659
660 file_start_write(file);
661 ret = __kernel_write(file, buf, count, pos);
662 file_end_write(file);
663 return ret;
664}
665EXPORT_SYMBOL(kernel_write);
666
667ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
668{
669 ssize_t ret;
670
671 if (!(file->f_mode & FMODE_WRITE))
672 return -EBADF;
673 if (!(file->f_mode & FMODE_CAN_WRITE))
674 return -EINVAL;
675 if (unlikely(!access_ok(buf, count)))
676 return -EFAULT;
677
678 ret = rw_verify_area(WRITE, file, pos, count);
679 if (ret)
680 return ret;
681 if (count > MAX_RW_COUNT)
682 count = MAX_RW_COUNT;
683 file_start_write(file);
684 if (file->f_op->write)
685 ret = file->f_op->write(file, buf, count, pos);
686 else if (file->f_op->write_iter)
687 ret = new_sync_write(file, buf, count, pos);
688 else
689 ret = -EINVAL;
690 if (ret > 0) {
691 fsnotify_modify(file);
692 add_wchar(current, ret);
693 }
694 inc_syscw(current);
695 file_end_write(file);
696 return ret;
697}
698
699/* file_ppos returns &file->f_pos or NULL if file is stream */
700static inline loff_t *file_ppos(struct file *file)
701{
702 return file->f_mode & FMODE_STREAM ? NULL : &file->f_pos;
703}
704
705ssize_t ksys_read(unsigned int fd, char __user *buf, size_t count)
706{
707 CLASS(fd_pos, f)(fd);
708 ssize_t ret = -EBADF;
709
710 if (!fd_empty(f)) {
711 loff_t pos, *ppos = file_ppos(fd_file(f));
712 if (ppos) {
713 pos = *ppos;
714 ppos = &pos;
715 }
716 ret = vfs_read(fd_file(f), buf, count, ppos);
717 if (ret >= 0 && ppos)
718 fd_file(f)->f_pos = pos;
719 }
720 return ret;
721}
722
723SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
724{
725 return ksys_read(fd, buf, count);
726}
727
728ssize_t ksys_write(unsigned int fd, const char __user *buf, size_t count)
729{
730 CLASS(fd_pos, f)(fd);
731 ssize_t ret = -EBADF;
732
733 if (!fd_empty(f)) {
734 loff_t pos, *ppos = file_ppos(fd_file(f));
735 if (ppos) {
736 pos = *ppos;
737 ppos = &pos;
738 }
739 ret = vfs_write(fd_file(f), buf, count, ppos);
740 if (ret >= 0 && ppos)
741 fd_file(f)->f_pos = pos;
742 }
743
744 return ret;
745}
746
747SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
748 size_t, count)
749{
750 return ksys_write(fd, buf, count);
751}
752
753ssize_t ksys_pread64(unsigned int fd, char __user *buf, size_t count,
754 loff_t pos)
755{
756 if (pos < 0)
757 return -EINVAL;
758
759 CLASS(fd, f)(fd);
760 if (fd_empty(f))
761 return -EBADF;
762
763 if (fd_file(f)->f_mode & FMODE_PREAD)
764 return vfs_read(fd_file(f), buf, count, &pos);
765
766 return -ESPIPE;
767}
768
769SYSCALL_DEFINE4(pread64, unsigned int, fd, char __user *, buf,
770 size_t, count, loff_t, pos)
771{
772 return ksys_pread64(fd, buf, count, pos);
773}
774
775#if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_PREAD64)
776COMPAT_SYSCALL_DEFINE5(pread64, unsigned int, fd, char __user *, buf,
777 size_t, count, compat_arg_u64_dual(pos))
778{
779 return ksys_pread64(fd, buf, count, compat_arg_u64_glue(pos));
780}
781#endif
782
783ssize_t ksys_pwrite64(unsigned int fd, const char __user *buf,
784 size_t count, loff_t pos)
785{
786 if (pos < 0)
787 return -EINVAL;
788
789 CLASS(fd, f)(fd);
790 if (fd_empty(f))
791 return -EBADF;
792
793 if (fd_file(f)->f_mode & FMODE_PWRITE)
794 return vfs_write(fd_file(f), buf, count, &pos);
795
796 return -ESPIPE;
797}
798
799SYSCALL_DEFINE4(pwrite64, unsigned int, fd, const char __user *, buf,
800 size_t, count, loff_t, pos)
801{
802 return ksys_pwrite64(fd, buf, count, pos);
803}
804
805#if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_PWRITE64)
806COMPAT_SYSCALL_DEFINE5(pwrite64, unsigned int, fd, const char __user *, buf,
807 size_t, count, compat_arg_u64_dual(pos))
808{
809 return ksys_pwrite64(fd, buf, count, compat_arg_u64_glue(pos));
810}
811#endif
812
813static ssize_t do_iter_readv_writev(struct file *filp, struct iov_iter *iter,
814 loff_t *ppos, int type, rwf_t flags)
815{
816 struct kiocb kiocb;
817 ssize_t ret;
818
819 init_sync_kiocb(&kiocb, filp);
820 ret = kiocb_set_rw_flags(&kiocb, flags, type);
821 if (ret)
822 return ret;
823 kiocb.ki_pos = (ppos ? *ppos : 0);
824
825 if (type == READ)
826 ret = filp->f_op->read_iter(&kiocb, iter);
827 else
828 ret = filp->f_op->write_iter(&kiocb, iter);
829 BUG_ON(ret == -EIOCBQUEUED);
830 if (ppos)
831 *ppos = kiocb.ki_pos;
832 return ret;
833}
834
835/* Do it by hand, with file-ops */
836static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter,
837 loff_t *ppos, int type, rwf_t flags)
838{
839 ssize_t ret = 0;
840
841 if (flags & ~RWF_HIPRI)
842 return -EOPNOTSUPP;
843
844 while (iov_iter_count(iter)) {
845 ssize_t nr;
846
847 if (type == READ) {
848 nr = filp->f_op->read(filp, iter_iov_addr(iter),
849 iter_iov_len(iter), ppos);
850 } else {
851 nr = filp->f_op->write(filp, iter_iov_addr(iter),
852 iter_iov_len(iter), ppos);
853 }
854
855 if (nr < 0) {
856 if (!ret)
857 ret = nr;
858 break;
859 }
860 ret += nr;
861 if (nr != iter_iov_len(iter))
862 break;
863 iov_iter_advance(iter, nr);
864 }
865
866 return ret;
867}
868
869ssize_t vfs_iocb_iter_read(struct file *file, struct kiocb *iocb,
870 struct iov_iter *iter)
871{
872 size_t tot_len;
873 ssize_t ret = 0;
874
875 if (!file->f_op->read_iter)
876 return -EINVAL;
877 if (!(file->f_mode & FMODE_READ))
878 return -EBADF;
879 if (!(file->f_mode & FMODE_CAN_READ))
880 return -EINVAL;
881
882 tot_len = iov_iter_count(iter);
883 if (!tot_len)
884 goto out;
885 ret = rw_verify_area(READ, file, &iocb->ki_pos, tot_len);
886 if (ret < 0)
887 return ret;
888
889 ret = file->f_op->read_iter(iocb, iter);
890out:
891 if (ret >= 0)
892 fsnotify_access(file);
893 return ret;
894}
895EXPORT_SYMBOL(vfs_iocb_iter_read);
896
897ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos,
898 rwf_t flags)
899{
900 size_t tot_len;
901 ssize_t ret = 0;
902
903 if (!file->f_op->read_iter)
904 return -EINVAL;
905 if (!(file->f_mode & FMODE_READ))
906 return -EBADF;
907 if (!(file->f_mode & FMODE_CAN_READ))
908 return -EINVAL;
909
910 tot_len = iov_iter_count(iter);
911 if (!tot_len)
912 goto out;
913 ret = rw_verify_area(READ, file, ppos, tot_len);
914 if (ret < 0)
915 return ret;
916
917 ret = do_iter_readv_writev(file, iter, ppos, READ, flags);
918out:
919 if (ret >= 0)
920 fsnotify_access(file);
921 return ret;
922}
923EXPORT_SYMBOL(vfs_iter_read);
924
925/*
926 * Caller is responsible for calling kiocb_end_write() on completion
927 * if async iocb was queued.
928 */
929ssize_t vfs_iocb_iter_write(struct file *file, struct kiocb *iocb,
930 struct iov_iter *iter)
931{
932 size_t tot_len;
933 ssize_t ret = 0;
934
935 if (!file->f_op->write_iter)
936 return -EINVAL;
937 if (!(file->f_mode & FMODE_WRITE))
938 return -EBADF;
939 if (!(file->f_mode & FMODE_CAN_WRITE))
940 return -EINVAL;
941
942 tot_len = iov_iter_count(iter);
943 if (!tot_len)
944 return 0;
945 ret = rw_verify_area(WRITE, file, &iocb->ki_pos, tot_len);
946 if (ret < 0)
947 return ret;
948
949 kiocb_start_write(iocb);
950 ret = file->f_op->write_iter(iocb, iter);
951 if (ret != -EIOCBQUEUED)
952 kiocb_end_write(iocb);
953 if (ret > 0)
954 fsnotify_modify(file);
955
956 return ret;
957}
958EXPORT_SYMBOL(vfs_iocb_iter_write);
959
960ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos,
961 rwf_t flags)
962{
963 size_t tot_len;
964 ssize_t ret;
965
966 if (!(file->f_mode & FMODE_WRITE))
967 return -EBADF;
968 if (!(file->f_mode & FMODE_CAN_WRITE))
969 return -EINVAL;
970 if (!file->f_op->write_iter)
971 return -EINVAL;
972
973 tot_len = iov_iter_count(iter);
974 if (!tot_len)
975 return 0;
976
977 ret = rw_verify_area(WRITE, file, ppos, tot_len);
978 if (ret < 0)
979 return ret;
980
981 file_start_write(file);
982 ret = do_iter_readv_writev(file, iter, ppos, WRITE, flags);
983 if (ret > 0)
984 fsnotify_modify(file);
985 file_end_write(file);
986
987 return ret;
988}
989EXPORT_SYMBOL(vfs_iter_write);
990
991static ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
992 unsigned long vlen, loff_t *pos, rwf_t flags)
993{
994 struct iovec iovstack[UIO_FASTIOV];
995 struct iovec *iov = iovstack;
996 struct iov_iter iter;
997 size_t tot_len;
998 ssize_t ret = 0;
999
1000 if (!(file->f_mode & FMODE_READ))
1001 return -EBADF;
1002 if (!(file->f_mode & FMODE_CAN_READ))
1003 return -EINVAL;
1004
1005 ret = import_iovec(ITER_DEST, vec, vlen, ARRAY_SIZE(iovstack), &iov,
1006 &iter);
1007 if (ret < 0)
1008 return ret;
1009
1010 tot_len = iov_iter_count(&iter);
1011 if (!tot_len)
1012 goto out;
1013
1014 ret = rw_verify_area(READ, file, pos, tot_len);
1015 if (ret < 0)
1016 goto out;
1017
1018 if (file->f_op->read_iter)
1019 ret = do_iter_readv_writev(file, &iter, pos, READ, flags);
1020 else
1021 ret = do_loop_readv_writev(file, &iter, pos, READ, flags);
1022out:
1023 if (ret >= 0)
1024 fsnotify_access(file);
1025 kfree(iov);
1026 return ret;
1027}
1028
1029static ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
1030 unsigned long vlen, loff_t *pos, rwf_t flags)
1031{
1032 struct iovec iovstack[UIO_FASTIOV];
1033 struct iovec *iov = iovstack;
1034 struct iov_iter iter;
1035 size_t tot_len;
1036 ssize_t ret = 0;
1037
1038 if (!(file->f_mode & FMODE_WRITE))
1039 return -EBADF;
1040 if (!(file->f_mode & FMODE_CAN_WRITE))
1041 return -EINVAL;
1042
1043 ret = import_iovec(ITER_SOURCE, vec, vlen, ARRAY_SIZE(iovstack), &iov,
1044 &iter);
1045 if (ret < 0)
1046 return ret;
1047
1048 tot_len = iov_iter_count(&iter);
1049 if (!tot_len)
1050 goto out;
1051
1052 ret = rw_verify_area(WRITE, file, pos, tot_len);
1053 if (ret < 0)
1054 goto out;
1055
1056 file_start_write(file);
1057 if (file->f_op->write_iter)
1058 ret = do_iter_readv_writev(file, &iter, pos, WRITE, flags);
1059 else
1060 ret = do_loop_readv_writev(file, &iter, pos, WRITE, flags);
1061 if (ret > 0)
1062 fsnotify_modify(file);
1063 file_end_write(file);
1064out:
1065 kfree(iov);
1066 return ret;
1067}
1068
1069static ssize_t do_readv(unsigned long fd, const struct iovec __user *vec,
1070 unsigned long vlen, rwf_t flags)
1071{
1072 CLASS(fd_pos, f)(fd);
1073 ssize_t ret = -EBADF;
1074
1075 if (!fd_empty(f)) {
1076 loff_t pos, *ppos = file_ppos(fd_file(f));
1077 if (ppos) {
1078 pos = *ppos;
1079 ppos = &pos;
1080 }
1081 ret = vfs_readv(fd_file(f), vec, vlen, ppos, flags);
1082 if (ret >= 0 && ppos)
1083 fd_file(f)->f_pos = pos;
1084 }
1085
1086 if (ret > 0)
1087 add_rchar(current, ret);
1088 inc_syscr(current);
1089 return ret;
1090}
1091
1092static ssize_t do_writev(unsigned long fd, const struct iovec __user *vec,
1093 unsigned long vlen, rwf_t flags)
1094{
1095 CLASS(fd_pos, f)(fd);
1096 ssize_t ret = -EBADF;
1097
1098 if (!fd_empty(f)) {
1099 loff_t pos, *ppos = file_ppos(fd_file(f));
1100 if (ppos) {
1101 pos = *ppos;
1102 ppos = &pos;
1103 }
1104 ret = vfs_writev(fd_file(f), vec, vlen, ppos, flags);
1105 if (ret >= 0 && ppos)
1106 fd_file(f)->f_pos = pos;
1107 }
1108
1109 if (ret > 0)
1110 add_wchar(current, ret);
1111 inc_syscw(current);
1112 return ret;
1113}
1114
1115static inline loff_t pos_from_hilo(unsigned long high, unsigned long low)
1116{
1117#define HALF_LONG_BITS (BITS_PER_LONG / 2)
1118 return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low;
1119}
1120
1121static ssize_t do_preadv(unsigned long fd, const struct iovec __user *vec,
1122 unsigned long vlen, loff_t pos, rwf_t flags)
1123{
1124 ssize_t ret = -EBADF;
1125
1126 if (pos < 0)
1127 return -EINVAL;
1128
1129 CLASS(fd, f)(fd);
1130 if (!fd_empty(f)) {
1131 ret = -ESPIPE;
1132 if (fd_file(f)->f_mode & FMODE_PREAD)
1133 ret = vfs_readv(fd_file(f), vec, vlen, &pos, flags);
1134 }
1135
1136 if (ret > 0)
1137 add_rchar(current, ret);
1138 inc_syscr(current);
1139 return ret;
1140}
1141
1142static ssize_t do_pwritev(unsigned long fd, const struct iovec __user *vec,
1143 unsigned long vlen, loff_t pos, rwf_t flags)
1144{
1145 ssize_t ret = -EBADF;
1146
1147 if (pos < 0)
1148 return -EINVAL;
1149
1150 CLASS(fd, f)(fd);
1151 if (!fd_empty(f)) {
1152 ret = -ESPIPE;
1153 if (fd_file(f)->f_mode & FMODE_PWRITE)
1154 ret = vfs_writev(fd_file(f), vec, vlen, &pos, flags);
1155 }
1156
1157 if (ret > 0)
1158 add_wchar(current, ret);
1159 inc_syscw(current);
1160 return ret;
1161}
1162
1163SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
1164 unsigned long, vlen)
1165{
1166 return do_readv(fd, vec, vlen, 0);
1167}
1168
1169SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
1170 unsigned long, vlen)
1171{
1172 return do_writev(fd, vec, vlen, 0);
1173}
1174
1175SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
1176 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
1177{
1178 loff_t pos = pos_from_hilo(pos_h, pos_l);
1179
1180 return do_preadv(fd, vec, vlen, pos, 0);
1181}
1182
1183SYSCALL_DEFINE6(preadv2, unsigned long, fd, const struct iovec __user *, vec,
1184 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
1185 rwf_t, flags)
1186{
1187 loff_t pos = pos_from_hilo(pos_h, pos_l);
1188
1189 if (pos == -1)
1190 return do_readv(fd, vec, vlen, flags);
1191
1192 return do_preadv(fd, vec, vlen, pos, flags);
1193}
1194
1195SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
1196 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
1197{
1198 loff_t pos = pos_from_hilo(pos_h, pos_l);
1199
1200 return do_pwritev(fd, vec, vlen, pos, 0);
1201}
1202
1203SYSCALL_DEFINE6(pwritev2, unsigned long, fd, const struct iovec __user *, vec,
1204 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
1205 rwf_t, flags)
1206{
1207 loff_t pos = pos_from_hilo(pos_h, pos_l);
1208
1209 if (pos == -1)
1210 return do_writev(fd, vec, vlen, flags);
1211
1212 return do_pwritev(fd, vec, vlen, pos, flags);
1213}
1214
1215/*
1216 * Various compat syscalls. Note that they all pretend to take a native
1217 * iovec - import_iovec will properly treat those as compat_iovecs based on
1218 * in_compat_syscall().
1219 */
1220#ifdef CONFIG_COMPAT
1221#ifdef __ARCH_WANT_COMPAT_SYS_PREADV64
1222COMPAT_SYSCALL_DEFINE4(preadv64, unsigned long, fd,
1223 const struct iovec __user *, vec,
1224 unsigned long, vlen, loff_t, pos)
1225{
1226 return do_preadv(fd, vec, vlen, pos, 0);
1227}
1228#endif
1229
1230COMPAT_SYSCALL_DEFINE5(preadv, compat_ulong_t, fd,
1231 const struct iovec __user *, vec,
1232 compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
1233{
1234 loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1235
1236 return do_preadv(fd, vec, vlen, pos, 0);
1237}
1238
1239#ifdef __ARCH_WANT_COMPAT_SYS_PREADV64V2
1240COMPAT_SYSCALL_DEFINE5(preadv64v2, unsigned long, fd,
1241 const struct iovec __user *, vec,
1242 unsigned long, vlen, loff_t, pos, rwf_t, flags)
1243{
1244 if (pos == -1)
1245 return do_readv(fd, vec, vlen, flags);
1246 return do_preadv(fd, vec, vlen, pos, flags);
1247}
1248#endif
1249
1250COMPAT_SYSCALL_DEFINE6(preadv2, compat_ulong_t, fd,
1251 const struct iovec __user *, vec,
1252 compat_ulong_t, vlen, u32, pos_low, u32, pos_high,
1253 rwf_t, flags)
1254{
1255 loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1256
1257 if (pos == -1)
1258 return do_readv(fd, vec, vlen, flags);
1259 return do_preadv(fd, vec, vlen, pos, flags);
1260}
1261
1262#ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64
1263COMPAT_SYSCALL_DEFINE4(pwritev64, unsigned long, fd,
1264 const struct iovec __user *, vec,
1265 unsigned long, vlen, loff_t, pos)
1266{
1267 return do_pwritev(fd, vec, vlen, pos, 0);
1268}
1269#endif
1270
1271COMPAT_SYSCALL_DEFINE5(pwritev, compat_ulong_t, fd,
1272 const struct iovec __user *,vec,
1273 compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
1274{
1275 loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1276
1277 return do_pwritev(fd, vec, vlen, pos, 0);
1278}
1279
1280#ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64V2
1281COMPAT_SYSCALL_DEFINE5(pwritev64v2, unsigned long, fd,
1282 const struct iovec __user *, vec,
1283 unsigned long, vlen, loff_t, pos, rwf_t, flags)
1284{
1285 if (pos == -1)
1286 return do_writev(fd, vec, vlen, flags);
1287 return do_pwritev(fd, vec, vlen, pos, flags);
1288}
1289#endif
1290
1291COMPAT_SYSCALL_DEFINE6(pwritev2, compat_ulong_t, fd,
1292 const struct iovec __user *,vec,
1293 compat_ulong_t, vlen, u32, pos_low, u32, pos_high, rwf_t, flags)
1294{
1295 loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1296
1297 if (pos == -1)
1298 return do_writev(fd, vec, vlen, flags);
1299 return do_pwritev(fd, vec, vlen, pos, flags);
1300}
1301#endif /* CONFIG_COMPAT */
1302
1303static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
1304 size_t count, loff_t max)
1305{
1306 struct inode *in_inode, *out_inode;
1307 struct pipe_inode_info *opipe;
1308 loff_t pos;
1309 loff_t out_pos;
1310 ssize_t retval;
1311 int fl;
1312
1313 /*
1314 * Get input file, and verify that it is ok..
1315 */
1316 CLASS(fd, in)(in_fd);
1317 if (fd_empty(in))
1318 return -EBADF;
1319 if (!(fd_file(in)->f_mode & FMODE_READ))
1320 return -EBADF;
1321 if (!ppos) {
1322 pos = fd_file(in)->f_pos;
1323 } else {
1324 pos = *ppos;
1325 if (!(fd_file(in)->f_mode & FMODE_PREAD))
1326 return -ESPIPE;
1327 }
1328 retval = rw_verify_area(READ, fd_file(in), &pos, count);
1329 if (retval < 0)
1330 return retval;
1331 if (count > MAX_RW_COUNT)
1332 count = MAX_RW_COUNT;
1333
1334 /*
1335 * Get output file, and verify that it is ok..
1336 */
1337 CLASS(fd, out)(out_fd);
1338 if (fd_empty(out))
1339 return -EBADF;
1340 if (!(fd_file(out)->f_mode & FMODE_WRITE))
1341 return -EBADF;
1342 in_inode = file_inode(fd_file(in));
1343 out_inode = file_inode(fd_file(out));
1344 out_pos = fd_file(out)->f_pos;
1345
1346 if (!max)
1347 max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
1348
1349 if (unlikely(pos + count > max)) {
1350 if (pos >= max)
1351 return -EOVERFLOW;
1352 count = max - pos;
1353 }
1354
1355 fl = 0;
1356#if 0
1357 /*
1358 * We need to debate whether we can enable this or not. The
1359 * man page documents EAGAIN return for the output at least,
1360 * and the application is arguably buggy if it doesn't expect
1361 * EAGAIN on a non-blocking file descriptor.
1362 */
1363 if (fd_file(in)->f_flags & O_NONBLOCK)
1364 fl = SPLICE_F_NONBLOCK;
1365#endif
1366 opipe = get_pipe_info(fd_file(out), true);
1367 if (!opipe) {
1368 retval = rw_verify_area(WRITE, fd_file(out), &out_pos, count);
1369 if (retval < 0)
1370 return retval;
1371 retval = do_splice_direct(fd_file(in), &pos, fd_file(out), &out_pos,
1372 count, fl);
1373 } else {
1374 if (fd_file(out)->f_flags & O_NONBLOCK)
1375 fl |= SPLICE_F_NONBLOCK;
1376
1377 retval = splice_file_to_pipe(fd_file(in), opipe, &pos, count, fl);
1378 }
1379
1380 if (retval > 0) {
1381 add_rchar(current, retval);
1382 add_wchar(current, retval);
1383 fsnotify_access(fd_file(in));
1384 fsnotify_modify(fd_file(out));
1385 fd_file(out)->f_pos = out_pos;
1386 if (ppos)
1387 *ppos = pos;
1388 else
1389 fd_file(in)->f_pos = pos;
1390 }
1391
1392 inc_syscr(current);
1393 inc_syscw(current);
1394 if (pos > max)
1395 retval = -EOVERFLOW;
1396 return retval;
1397}
1398
1399SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
1400{
1401 loff_t pos;
1402 off_t off;
1403 ssize_t ret;
1404
1405 if (offset) {
1406 if (unlikely(get_user(off, offset)))
1407 return -EFAULT;
1408 pos = off;
1409 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1410 if (unlikely(put_user(pos, offset)))
1411 return -EFAULT;
1412 return ret;
1413 }
1414
1415 return do_sendfile(out_fd, in_fd, NULL, count, 0);
1416}
1417
1418SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
1419{
1420 loff_t pos;
1421 ssize_t ret;
1422
1423 if (offset) {
1424 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1425 return -EFAULT;
1426 ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1427 if (unlikely(put_user(pos, offset)))
1428 return -EFAULT;
1429 return ret;
1430 }
1431
1432 return do_sendfile(out_fd, in_fd, NULL, count, 0);
1433}
1434
1435#ifdef CONFIG_COMPAT
1436COMPAT_SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd,
1437 compat_off_t __user *, offset, compat_size_t, count)
1438{
1439 loff_t pos;
1440 off_t off;
1441 ssize_t ret;
1442
1443 if (offset) {
1444 if (unlikely(get_user(off, offset)))
1445 return -EFAULT;
1446 pos = off;
1447 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1448 if (unlikely(put_user(pos, offset)))
1449 return -EFAULT;
1450 return ret;
1451 }
1452
1453 return do_sendfile(out_fd, in_fd, NULL, count, 0);
1454}
1455
1456COMPAT_SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd,
1457 compat_loff_t __user *, offset, compat_size_t, count)
1458{
1459 loff_t pos;
1460 ssize_t ret;
1461
1462 if (offset) {
1463 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1464 return -EFAULT;
1465 ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1466 if (unlikely(put_user(pos, offset)))
1467 return -EFAULT;
1468 return ret;
1469 }
1470
1471 return do_sendfile(out_fd, in_fd, NULL, count, 0);
1472}
1473#endif
1474
1475/*
1476 * Performs necessary checks before doing a file copy
1477 *
1478 * Can adjust amount of bytes to copy via @req_count argument.
1479 * Returns appropriate error code that caller should return or
1480 * zero in case the copy should be allowed.
1481 */
1482static int generic_copy_file_checks(struct file *file_in, loff_t pos_in,
1483 struct file *file_out, loff_t pos_out,
1484 size_t *req_count, unsigned int flags)
1485{
1486 struct inode *inode_in = file_inode(file_in);
1487 struct inode *inode_out = file_inode(file_out);
1488 uint64_t count = *req_count;
1489 loff_t size_in;
1490 int ret;
1491
1492 ret = generic_file_rw_checks(file_in, file_out);
1493 if (ret)
1494 return ret;
1495
1496 /*
1497 * We allow some filesystems to handle cross sb copy, but passing
1498 * a file of the wrong filesystem type to filesystem driver can result
1499 * in an attempt to dereference the wrong type of ->private_data, so
1500 * avoid doing that until we really have a good reason.
1501 *
1502 * nfs and cifs define several different file_system_type structures
1503 * and several different sets of file_operations, but they all end up
1504 * using the same ->copy_file_range() function pointer.
1505 */
1506 if (flags & COPY_FILE_SPLICE) {
1507 /* cross sb splice is allowed */
1508 } else if (file_out->f_op->copy_file_range) {
1509 if (file_in->f_op->copy_file_range !=
1510 file_out->f_op->copy_file_range)
1511 return -EXDEV;
1512 } else if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb) {
1513 return -EXDEV;
1514 }
1515
1516 /* Don't touch certain kinds of inodes */
1517 if (IS_IMMUTABLE(inode_out))
1518 return -EPERM;
1519
1520 if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out))
1521 return -ETXTBSY;
1522
1523 /* Ensure offsets don't wrap. */
1524 if (pos_in + count < pos_in || pos_out + count < pos_out)
1525 return -EOVERFLOW;
1526
1527 /* Shorten the copy to EOF */
1528 size_in = i_size_read(inode_in);
1529 if (pos_in >= size_in)
1530 count = 0;
1531 else
1532 count = min(count, size_in - (uint64_t)pos_in);
1533
1534 ret = generic_write_check_limits(file_out, pos_out, &count);
1535 if (ret)
1536 return ret;
1537
1538 /* Don't allow overlapped copying within the same file. */
1539 if (inode_in == inode_out &&
1540 pos_out + count > pos_in &&
1541 pos_out < pos_in + count)
1542 return -EINVAL;
1543
1544 *req_count = count;
1545 return 0;
1546}
1547
1548/*
1549 * copy_file_range() differs from regular file read and write in that it
1550 * specifically allows return partial success. When it does so is up to
1551 * the copy_file_range method.
1552 */
1553ssize_t vfs_copy_file_range(struct file *file_in, loff_t pos_in,
1554 struct file *file_out, loff_t pos_out,
1555 size_t len, unsigned int flags)
1556{
1557 ssize_t ret;
1558 bool splice = flags & COPY_FILE_SPLICE;
1559 bool samesb = file_inode(file_in)->i_sb == file_inode(file_out)->i_sb;
1560
1561 if (flags & ~COPY_FILE_SPLICE)
1562 return -EINVAL;
1563
1564 ret = generic_copy_file_checks(file_in, pos_in, file_out, pos_out, &len,
1565 flags);
1566 if (unlikely(ret))
1567 return ret;
1568
1569 ret = rw_verify_area(READ, file_in, &pos_in, len);
1570 if (unlikely(ret))
1571 return ret;
1572
1573 ret = rw_verify_area(WRITE, file_out, &pos_out, len);
1574 if (unlikely(ret))
1575 return ret;
1576
1577 if (len == 0)
1578 return 0;
1579
1580 /*
1581 * Make sure return value doesn't overflow in 32bit compat mode. Also
1582 * limit the size for all cases except when calling ->copy_file_range().
1583 */
1584 if (splice || !file_out->f_op->copy_file_range || in_compat_syscall())
1585 len = min_t(size_t, MAX_RW_COUNT, len);
1586
1587 file_start_write(file_out);
1588
1589 /*
1590 * Cloning is supported by more file systems, so we implement copy on
1591 * same sb using clone, but for filesystems where both clone and copy
1592 * are supported (e.g. nfs,cifs), we only call the copy method.
1593 */
1594 if (!splice && file_out->f_op->copy_file_range) {
1595 ret = file_out->f_op->copy_file_range(file_in, pos_in,
1596 file_out, pos_out,
1597 len, flags);
1598 } else if (!splice && file_in->f_op->remap_file_range && samesb) {
1599 ret = file_in->f_op->remap_file_range(file_in, pos_in,
1600 file_out, pos_out, len, REMAP_FILE_CAN_SHORTEN);
1601 /* fallback to splice */
1602 if (ret <= 0)
1603 splice = true;
1604 } else if (samesb) {
1605 /* Fallback to splice for same sb copy for backward compat */
1606 splice = true;
1607 }
1608
1609 file_end_write(file_out);
1610
1611 if (!splice)
1612 goto done;
1613
1614 /*
1615 * We can get here for same sb copy of filesystems that do not implement
1616 * ->copy_file_range() in case filesystem does not support clone or in
1617 * case filesystem supports clone but rejected the clone request (e.g.
1618 * because it was not block aligned).
1619 *
1620 * In both cases, fall back to kernel copy so we are able to maintain a
1621 * consistent story about which filesystems support copy_file_range()
1622 * and which filesystems do not, that will allow userspace tools to
1623 * make consistent desicions w.r.t using copy_file_range().
1624 *
1625 * We also get here if caller (e.g. nfsd) requested COPY_FILE_SPLICE
1626 * for server-side-copy between any two sb.
1627 *
1628 * In any case, we call do_splice_direct() and not splice_file_range(),
1629 * without file_start_write() held, to avoid possible deadlocks related
1630 * to splicing from input file, while file_start_write() is held on
1631 * the output file on a different sb.
1632 */
1633 ret = do_splice_direct(file_in, &pos_in, file_out, &pos_out, len, 0);
1634done:
1635 if (ret > 0) {
1636 fsnotify_access(file_in);
1637 add_rchar(current, ret);
1638 fsnotify_modify(file_out);
1639 add_wchar(current, ret);
1640 }
1641
1642 inc_syscr(current);
1643 inc_syscw(current);
1644
1645 return ret;
1646}
1647EXPORT_SYMBOL(vfs_copy_file_range);
1648
1649SYSCALL_DEFINE6(copy_file_range, int, fd_in, loff_t __user *, off_in,
1650 int, fd_out, loff_t __user *, off_out,
1651 size_t, len, unsigned int, flags)
1652{
1653 loff_t pos_in;
1654 loff_t pos_out;
1655 ssize_t ret = -EBADF;
1656
1657 CLASS(fd, f_in)(fd_in);
1658 if (fd_empty(f_in))
1659 return -EBADF;
1660
1661 CLASS(fd, f_out)(fd_out);
1662 if (fd_empty(f_out))
1663 return -EBADF;
1664
1665 if (off_in) {
1666 if (copy_from_user(&pos_in, off_in, sizeof(loff_t)))
1667 return -EFAULT;
1668 } else {
1669 pos_in = fd_file(f_in)->f_pos;
1670 }
1671
1672 if (off_out) {
1673 if (copy_from_user(&pos_out, off_out, sizeof(loff_t)))
1674 return -EFAULT;
1675 } else {
1676 pos_out = fd_file(f_out)->f_pos;
1677 }
1678
1679 if (flags != 0)
1680 return -EINVAL;
1681
1682 ret = vfs_copy_file_range(fd_file(f_in), pos_in, fd_file(f_out), pos_out, len,
1683 flags);
1684 if (ret > 0) {
1685 pos_in += ret;
1686 pos_out += ret;
1687
1688 if (off_in) {
1689 if (copy_to_user(off_in, &pos_in, sizeof(loff_t)))
1690 ret = -EFAULT;
1691 } else {
1692 fd_file(f_in)->f_pos = pos_in;
1693 }
1694
1695 if (off_out) {
1696 if (copy_to_user(off_out, &pos_out, sizeof(loff_t)))
1697 ret = -EFAULT;
1698 } else {
1699 fd_file(f_out)->f_pos = pos_out;
1700 }
1701 }
1702 return ret;
1703}
1704
1705/*
1706 * Don't operate on ranges the page cache doesn't support, and don't exceed the
1707 * LFS limits. If pos is under the limit it becomes a short access. If it
1708 * exceeds the limit we return -EFBIG.
1709 */
1710int generic_write_check_limits(struct file *file, loff_t pos, loff_t *count)
1711{
1712 struct inode *inode = file->f_mapping->host;
1713 loff_t max_size = inode->i_sb->s_maxbytes;
1714 loff_t limit = rlimit(RLIMIT_FSIZE);
1715
1716 if (limit != RLIM_INFINITY) {
1717 if (pos >= limit) {
1718 send_sig(SIGXFSZ, current, 0);
1719 return -EFBIG;
1720 }
1721 *count = min(*count, limit - pos);
1722 }
1723
1724 if (!(file->f_flags & O_LARGEFILE))
1725 max_size = MAX_NON_LFS;
1726
1727 if (unlikely(pos >= max_size))
1728 return -EFBIG;
1729
1730 *count = min(*count, max_size - pos);
1731
1732 return 0;
1733}
1734EXPORT_SYMBOL_GPL(generic_write_check_limits);
1735
1736/* Like generic_write_checks(), but takes size of write instead of iter. */
1737int generic_write_checks_count(struct kiocb *iocb, loff_t *count)
1738{
1739 struct file *file = iocb->ki_filp;
1740 struct inode *inode = file->f_mapping->host;
1741
1742 if (IS_SWAPFILE(inode))
1743 return -ETXTBSY;
1744
1745 if (!*count)
1746 return 0;
1747
1748 if (iocb->ki_flags & IOCB_APPEND)
1749 iocb->ki_pos = i_size_read(inode);
1750
1751 if ((iocb->ki_flags & IOCB_NOWAIT) &&
1752 !((iocb->ki_flags & IOCB_DIRECT) ||
1753 (file->f_op->fop_flags & FOP_BUFFER_WASYNC)))
1754 return -EINVAL;
1755
1756 return generic_write_check_limits(iocb->ki_filp, iocb->ki_pos, count);
1757}
1758EXPORT_SYMBOL(generic_write_checks_count);
1759
1760/*
1761 * Performs necessary checks before doing a write
1762 *
1763 * Can adjust writing position or amount of bytes to write.
1764 * Returns the number of bytes that may be written on success (which
1765 * may be less than requested if truncated), or a negative error code.
1766 */
1767ssize_t generic_write_checks(struct kiocb *iocb, struct iov_iter *from)
1768{
1769 loff_t count = iov_iter_count(from);
1770 int ret;
1771
1772 ret = generic_write_checks_count(iocb, &count);
1773 if (ret)
1774 return ret;
1775
1776 iov_iter_truncate(from, count);
1777 return iov_iter_count(from);
1778}
1779EXPORT_SYMBOL(generic_write_checks);
1780
1781/*
1782 * Performs common checks before doing a file copy/clone
1783 * from @file_in to @file_out.
1784 */
1785int generic_file_rw_checks(struct file *file_in, struct file *file_out)
1786{
1787 struct inode *inode_in = file_inode(file_in);
1788 struct inode *inode_out = file_inode(file_out);
1789
1790 /* Don't copy dirs, pipes, sockets... */
1791 if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
1792 return -EISDIR;
1793 if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
1794 return -EINVAL;
1795
1796 if (!(file_in->f_mode & FMODE_READ) ||
1797 !(file_out->f_mode & FMODE_WRITE) ||
1798 (file_out->f_flags & O_APPEND))
1799 return -EBADF;
1800
1801 return 0;
1802}
1803
1804int generic_atomic_write_valid(struct kiocb *iocb, struct iov_iter *iter)
1805{
1806 size_t len = iov_iter_count(iter);
1807
1808 if (!iter_is_ubuf(iter))
1809 return -EINVAL;
1810
1811 if (!is_power_of_2(len))
1812 return -EINVAL;
1813
1814 if (!IS_ALIGNED(iocb->ki_pos, len))
1815 return -EINVAL;
1816
1817 if (!(iocb->ki_flags & IOCB_DIRECT))
1818 return -EOPNOTSUPP;
1819
1820 return 0;
1821}
1822EXPORT_SYMBOL_GPL(generic_atomic_write_valid);