projects/linux·blob master · refs

1068 lines · 28.8 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * linux/fs/ext4/file.c
4 *
5 * Copyright (C) 1992, 1993, 1994, 1995
6 * Remy Card ([email protected])
7 * Laboratoire MASI - Institut Blaise Pascal
8 * Universite Pierre et Marie Curie (Paris VI)
9 *
10 * from
11 *
12 * linux/fs/minix/file.c
13 *
14 * Copyright (C) 1991, 1992 Linus Torvalds
15 *
16 * ext4 fs regular file handling primitives
17 *
18 * 64-bit file support on 64-bit platforms by Jakub Jelinek
19 * ([email protected])
20 */
21
22#include <linux/time.h>
23#include <linux/fs.h>
24#include <linux/iomap.h>
25#include <linux/mount.h>
26#include <linux/path.h>
27#include <linux/dax.h>
28#include <linux/filelock.h>
29#include <linux/quotaops.h>
30#include <linux/uio.h>
31#include <linux/mman.h>
32#include <linux/backing-dev.h>
33#include "ext4.h"
34#include "ext4_jbd2.h"
35#include "xattr.h"
36#include "acl.h"
37#include "truncate.h"
38
39/*
40 * Returns %true if the given DIO request should be attempted with DIO, or
41 * %false if it should fall back to buffered I/O.
42 *
43 * DIO isn't well specified; when it's unsupported (either due to the request
44 * being misaligned, or due to the file not supporting DIO at all), filesystems
45 * either fall back to buffered I/O or return EINVAL. For files that don't use
46 * any special features like encryption or verity, ext4 has traditionally
47 * returned EINVAL for misaligned DIO. iomap_dio_rw() uses this convention too.
48 * In this case, we should attempt the DIO, *not* fall back to buffered I/O.
49 *
50 * In contrast, in cases where DIO is unsupported due to ext4 features, ext4
51 * traditionally falls back to buffered I/O.
52 *
53 * This function implements the traditional ext4 behavior in all these cases.
54 */
55static bool ext4_should_use_dio(struct kiocb *iocb, struct iov_iter *iter)
56{
57 struct inode *inode = file_inode(iocb->ki_filp);
58 u32 dio_align = ext4_dio_alignment(inode);
59
60 if (dio_align == 0)
61 return false;
62
63 if (dio_align == 1)
64 return true;
65
66 return IS_ALIGNED(iocb->ki_pos | iov_iter_alignment(iter), dio_align);
67}
68
69static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to)
70{
71 ssize_t ret;
72 struct inode *inode = file_inode(iocb->ki_filp);
73
74 if (iocb->ki_flags & IOCB_NOWAIT) {
75 if (!inode_trylock_shared(inode))
76 return -EAGAIN;
77 } else {
78 inode_lock_shared(inode);
79 }
80
81 if (!ext4_should_use_dio(iocb, to)) {
82 inode_unlock_shared(inode);
83 /*
84 * Fallback to buffered I/O if the operation being performed on
85 * the inode is not supported by direct I/O. The IOCB_DIRECT
86 * flag needs to be cleared here in order to ensure that the
87 * direct I/O path within generic_file_read_iter() is not
88 * taken.
89 */
90 iocb->ki_flags &= ~IOCB_DIRECT;
91 return generic_file_read_iter(iocb, to);
92 }
93
94 ret = iomap_dio_read_simple(iocb, to, ext4_iomap_begin);
95 if (ret == -ENOTBLK)
96 ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0, NULL, 0);
97 inode_unlock_shared(inode);
98
99 file_accessed(iocb->ki_filp);
100 return ret;
101}
102
103#ifdef CONFIG_FS_DAX
104static ssize_t ext4_dax_read_iter(struct kiocb *iocb, struct iov_iter *to)
105{
106 struct inode *inode = file_inode(iocb->ki_filp);
107 ssize_t ret;
108
109 if (iocb->ki_flags & IOCB_NOWAIT) {
110 if (!inode_trylock_shared(inode))
111 return -EAGAIN;
112 } else {
113 inode_lock_shared(inode);
114 }
115 /*
116 * Recheck under inode lock - at this point we are sure it cannot
117 * change anymore
118 */
119 if (!IS_DAX(inode)) {
120 inode_unlock_shared(inode);
121 /* Fallback to buffered IO in case we cannot support DAX */
122 return generic_file_read_iter(iocb, to);
123 }
124 ret = dax_iomap_rw(iocb, to, &ext4_iomap_ops);
125 inode_unlock_shared(inode);
126
127 file_accessed(iocb->ki_filp);
128 return ret;
129}
130#endif
131
132static ssize_t ext4_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
133{
134 struct inode *inode = file_inode(iocb->ki_filp);
135
136 if (unlikely(ext4_forced_shutdown(inode->i_sb)))
137 return -EIO;
138
139 if (!iov_iter_count(to))
140 return 0; /* skip atime */
141
142#ifdef CONFIG_FS_DAX
143 if (IS_DAX(inode))
144 return ext4_dax_read_iter(iocb, to);
145#endif
146 if (iocb->ki_flags & IOCB_DIRECT)
147 return ext4_dio_read_iter(iocb, to);
148
149 return generic_file_read_iter(iocb, to);
150}
151
152static ssize_t ext4_file_splice_read(struct file *in, loff_t *ppos,
153 struct pipe_inode_info *pipe,
154 size_t len, unsigned int flags)
155{
156 struct inode *inode = file_inode(in);
157
158 if (unlikely(ext4_forced_shutdown(inode->i_sb)))
159 return -EIO;
160 return filemap_splice_read(in, ppos, pipe, len, flags);
161}
162
163/*
164 * Called when an inode is released. Note that this is different
165 * from ext4_file_open: open gets called at every open, but release
166 * gets called only when /all/ the files are closed.
167 */
168static int ext4_release_file(struct inode *inode, struct file *filp)
169{
170 if (ext4_test_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE)) {
171 ext4_alloc_da_blocks(inode);
172 ext4_clear_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE);
173 }
174 /* if we are the last writer on the inode, drop the block reservation */
175 if ((filp->f_mode & FMODE_WRITE) &&
176 (atomic_read(&inode->i_writecount) == 1) &&
177 !EXT4_I(inode)->i_reserved_data_blocks) {
178 down_write(&EXT4_I(inode)->i_data_sem);
179 ext4_discard_preallocations(inode);
180 up_write(&EXT4_I(inode)->i_data_sem);
181 }
182 if (is_dx(inode) && filp->private_data)
183 ext4_htree_free_dir_info(filp->private_data);
184
185 return 0;
186}
187
188/*
189 * This tests whether the IO in question is block-aligned or not.
190 * Ext4 utilizes unwritten extents when hole-filling during direct IO, and they
191 * are converted to written only after the IO is complete. Until they are
192 * mapped, these blocks appear as holes, so dio_zero_block() will assume that
193 * it needs to zero out portions of the start and/or end block. If 2 AIO
194 * threads are at work on the same unwritten block, they must be synchronized
195 * or one thread will zero the other's data, causing corruption.
196 */
197static bool
198ext4_unaligned_io(struct inode *inode, struct iov_iter *from, loff_t pos)
199{
200 struct super_block *sb = inode->i_sb;
201 unsigned long blockmask = sb->s_blocksize - 1;
202
203 if ((pos | iov_iter_alignment(from)) & blockmask)
204 return true;
205
206 return false;
207}
208
209static bool
210ext4_extending_io(struct inode *inode, loff_t offset, size_t len)
211{
212 if (offset + len > i_size_read(inode) ||
213 offset + len > EXT4_I(inode)->i_disksize)
214 return true;
215 return false;
216}
217
218/*
219 * Does an unaligned DIO write require partial block zeroing?
220 *
221 * Partial block zeroing is performed only for the head and tail blocks
222 * when they are partially covered by the write and the underlying extent
223 * is a hole or unwritten. Middle blocks (fully covered by the write)
224 * are written as whole blocks without zeroing.
225 *
226 * When zeroing is required, two concurrent unaligned DIO writes to the
227 * same partial block can race and corrupt each other's data, so the
228 * caller must take the exclusive i_rwsem and drain in-flight DIO. When
229 * zeroing is not required, shared lock is safe -- block allocation and
230 * unwritten conversion for middle blocks are protected by i_data_sem
231 * and inode_dio_begin().
232 */
233static bool ext4_dio_needs_zeroing(struct inode *inode, loff_t pos, loff_t len)
234{
235 struct ext4_map_blocks map;
236 unsigned int blkbits = inode->i_blkbits;
237 unsigned long blockmask = inode->i_sb->s_blocksize - 1;
238 bool head_partial, tail_partial;
239 ext4_lblk_t head_lblk, tail_lblk;
240 int err;
241
242 if (pos + len > i_size_read(inode))
243 return true;
244
245 head_partial = (pos & blockmask) != 0;
246 tail_partial = ((pos + len) & blockmask) != 0;
247 head_lblk = pos >> blkbits;
248 tail_lblk = (pos + len - 1) >> blkbits;
249
250 /* Check the head partial block. */
251 if (head_partial) {
252 map.m_lblk = head_lblk;
253 map.m_len = tail_lblk - head_lblk + 1;
254 err = ext4_map_blocks(NULL, inode, &map, 0);
255 if (err <= 0 || !(map.m_flags & EXT4_MAP_MAPPED))
256 return true;
257 /* If this mapping already covers the tail block, we're done. */
258 if (!tail_partial || map.m_lblk + err > tail_lblk)
259 return false;
260 }
261
262 /* Check the tail partial block. */
263 if (tail_partial) {
264 map.m_lblk = tail_lblk;
265 map.m_len = 1;
266 err = ext4_map_blocks(NULL, inode, &map, 0);
267 if (err <= 0 || !(map.m_flags & EXT4_MAP_MAPPED))
268 return true;
269 }
270
271 return false;
272}
273
274static ssize_t ext4_generic_write_checks(struct kiocb *iocb,
275 struct iov_iter *from)
276{
277 struct inode *inode = file_inode(iocb->ki_filp);
278 ssize_t ret;
279
280 if (unlikely(IS_IMMUTABLE(inode)))
281 return -EPERM;
282
283 ret = generic_write_checks(iocb, from);
284 if (ret <= 0)
285 return ret;
286
287 /*
288 * If we have encountered a bitmap-format file, the size limit
289 * is smaller than s_maxbytes, which is for extent-mapped files.
290 */
291 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
292 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
293
294 if (iocb->ki_pos >= sbi->s_bitmap_maxbytes)
295 return -EFBIG;
296 iov_iter_truncate(from, sbi->s_bitmap_maxbytes - iocb->ki_pos);
297 }
298
299 return iov_iter_count(from);
300}
301
302static ssize_t ext4_write_checks(struct kiocb *iocb, struct iov_iter *from)
303{
304 struct inode *inode = file_inode(iocb->ki_filp);
305 loff_t old_size = i_size_read(inode);
306 ssize_t ret, count;
307
308 count = ext4_generic_write_checks(iocb, from);
309 if (count <= 0)
310 return count;
311
312 ret = kiocb_modified(iocb);
313 if (ret)
314 return ret;
315
316 /*
317 * If the position is beyond the EOF, it is necessary to zero out the
318 * partial block that beyond the existing EOF, as it may contains
319 * stale data written through mmap.
320 */
321 if (iocb->ki_pos > old_size && !ext4_verity_in_progress(inode)) {
322 if (iocb->ki_flags & IOCB_NOWAIT)
323 return -EAGAIN;
324
325 ret = ext4_block_zero_eof(inode, old_size, iocb->ki_pos);
326 if (ret)
327 return ret;
328 }
329
330 return count;
331}
332
333static ssize_t ext4_buffered_write_iter(struct kiocb *iocb,
334 struct iov_iter *from)
335{
336 ssize_t ret;
337 struct inode *inode = file_inode(iocb->ki_filp);
338
339 if (iocb->ki_flags & IOCB_NOWAIT)
340 return -EOPNOTSUPP;
341
342 inode_lock(inode);
343
344 /*
345 * Prevent concurrent direct I/O and buffered I/O to the same file
346 * range. Wait for in-flight DIO to finish before dirtying pages.
347 */
348 inode_dio_wait(inode);
349
350 ret = ext4_write_checks(iocb, from);
351 if (ret <= 0)
352 goto out;
353
354 ret = generic_perform_write(iocb, from);
355
356out:
357 inode_unlock(inode);
358 if (unlikely(ret <= 0))
359 return ret;
360 return generic_write_sync(iocb, ret);
361}
362
363static ssize_t ext4_handle_inode_extension(struct inode *inode, loff_t offset,
364 ssize_t written, ssize_t count)
365{
366 handle_t *handle;
367
368 lockdep_assert_held_write(&inode->i_rwsem);
369 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
370 if (IS_ERR(handle))
371 return PTR_ERR(handle);
372
373 if (ext4_update_inode_size(inode, offset + written)) {
374 int ret = ext4_mark_inode_dirty(handle, inode);
375 if (unlikely(ret)) {
376 ext4_journal_stop(handle);
377 return ret;
378 }
379 }
380
381 if ((written == count) && inode->i_nlink)
382 ext4_orphan_del(handle, inode);
383 ext4_journal_stop(handle);
384
385 return written;
386}
387
388/*
389 * Clean up the inode after DIO or DAX extending write has completed and the
390 * inode size has been updated using ext4_handle_inode_extension().
391 */
392static void ext4_inode_extension_cleanup(struct inode *inode, bool need_trunc)
393{
394 lockdep_assert_held_write(&inode->i_rwsem);
395 if (need_trunc) {
396 ext4_truncate_failed_write(inode);
397 /*
398 * If the truncate operation failed early, then the inode may
399 * still be on the orphan list. In that case, we need to try
400 * remove the inode from the in-memory linked list.
401 */
402 if (inode->i_nlink)
403 ext4_orphan_del(NULL, inode);
404 return;
405 }
406 /*
407 * If i_disksize got extended either due to writeback of delalloc
408 * blocks or extending truncate while the DIO was running we could fail
409 * to cleanup the orphan list in ext4_handle_inode_extension(). Do it
410 * now.
411 */
412 if (ext4_inode_orphan_tracked(inode) && inode->i_nlink) {
413 handle_t *handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
414
415 if (IS_ERR(handle)) {
416 /*
417 * The write has successfully completed. Not much to
418 * do with the error here so just cleanup the orphan
419 * list and hope for the best.
420 */
421 ext4_orphan_del(NULL, inode);
422 return;
423 }
424 ext4_orphan_del(handle, inode);
425 ext4_journal_stop(handle);
426 }
427}
428
429static int ext4_dio_write_end_io(struct kiocb *iocb, ssize_t size,
430 int error, unsigned int flags)
431{
432 loff_t pos = iocb->ki_pos;
433 struct inode *inode = file_inode(iocb->ki_filp);
434
435
436 if (!error && size && (flags & IOMAP_DIO_UNWRITTEN) &&
437 (iocb->ki_flags & IOCB_ATOMIC))
438 error = ext4_convert_unwritten_extents_atomic(NULL, inode, pos,
439 size);
440 else if (!error && size && flags & IOMAP_DIO_UNWRITTEN)
441 error = ext4_convert_unwritten_extents(NULL, inode, pos, size,
442 NULL);
443 if (error)
444 return error;
445 /*
446 * Note that EXT4_I(inode)->i_disksize can get extended up to
447 * inode->i_size while the I/O was running due to writeback of delalloc
448 * blocks. But the code in ext4_iomap_alloc() is careful to use
449 * zeroed/unwritten extents if this is possible; thus we won't leave
450 * uninitialized blocks in a file even if we didn't succeed in writing
451 * as much as we intended. Also we can race with truncate or write
452 * expanding the file so we have to be a bit careful here.
453 */
454 if (pos + size <= READ_ONCE(EXT4_I(inode)->i_disksize) &&
455 pos + size <= i_size_read(inode))
456 return 0;
457 error = ext4_handle_inode_extension(inode, pos, size, size);
458 return error < 0 ? error : 0;
459}
460
461static const struct iomap_dio_ops ext4_dio_write_ops = {
462 .end_io = ext4_dio_write_end_io,
463};
464
465/*
466 * The intention here is to start with shared lock acquired then see if any
467 * condition requires an exclusive inode lock. If yes, then we restart the
468 * whole operation by releasing the shared lock and acquiring exclusive lock.
469 *
470 * The decision is layered, evaluated in this order:
471 *
472 * 1. If kiocb_modified() needs to update security info (!IS_NOSEC), upgrade
473 * to the exclusive lock -- the security update itself requires it,
474 * regardless of whether the write extends the file or is aligned.
475 *
476 * 2. If the write extends i_size or i_disksize, upgrade to the exclusive
477 * lock to safely update i_disksize and the orphan list, regardless of
478 * alignment.
479 *
480 * 3. Otherwise, for aligned non-extending writes, shared lock is always
481 * sufficient regardless of extent state (written, unwritten, or hole).
482 * truncate/punch_hole cannot run while we hold the shared i_rwsem
483 * (they need it exclusively); after we release it, inode_dio_begin()
484 * keeps their inode_dio_wait() blocked until in-flight bios complete.
485 * i_data_sem serializes concurrent extent tree modifications.
486 *
487 * 4. Otherwise, the write is unaligned and non-extending. Shared lock is
488 * safe unless the DIO layer needs to perform partial block zeroing --
489 * i.e. the head or tail partial block sits on a hole or unwritten
490 * extent. In that case upgrade to the exclusive lock and drain
491 * in-flight DIO to avoid races with concurrent partial block zeroing.
492 */
493static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from,
494 bool *ilock_shared, bool *extend,
495 int *dio_flags)
496{
497 struct file *file = iocb->ki_filp;
498 struct inode *inode = file_inode(file);
499 loff_t offset;
500 size_t count;
501 ssize_t ret;
502 bool needs_zeroing = false;
503
504restart:
505 ret = ext4_generic_write_checks(iocb, from);
506 if (ret <= 0)
507 goto out;
508
509 offset = iocb->ki_pos;
510 count = ret;
511
512 *extend = ext4_extending_io(inode, offset, count);
513
514 /*
515 * For unaligned writes, check whether partial block zeroing will be
516 * needed. If so, exclusive lock is required to serialize against
517 * concurrent DIO that could race with the zeroing.
518 *
519 * For aligned writes we skip this check entirely since allocation
520 * under shared lock is safe.
521 */
522 if (ext4_unaligned_io(inode, from, offset))
523 needs_zeroing = ext4_dio_needs_zeroing(inode, offset, count);
524
525 /* Determine whether we need to upgrade to an exclusive lock. */
526 if (*ilock_shared &&
527 (!IS_NOSEC(inode) || *extend || needs_zeroing)) {
528 if (iocb->ki_flags & IOCB_NOWAIT) {
529 ret = -EAGAIN;
530 goto out;
531 }
532 inode_unlock_shared(inode);
533 *ilock_shared = false;
534 inode_lock(inode);
535 goto restart;
536 }
537
538 /*
539 * Now that locking is settled, determine dio flags and exclusivity
540 * requirements. We don't use DIO_OVERWRITE_ONLY because we enforce
541 * behavior already. When holding the exclusive lock for a write that
542 * needs partial block zeroing or is extending the file, we must wait
543 * for the I/O to complete synchronously:
544 *
545 * - needs_zeroing: drain in-flight DIO whose end_io could race with
546 * our partial block zeroing, and force synchronous completion so we
547 * don't leave in-flight zeroing bios for the next writer to drain.
548 *
549 * - extend: the caller must update i_disksize after I/O completion,
550 * which requires the data to be on disk first.
551 */
552 if (!*ilock_shared && (needs_zeroing || *extend)) {
553 if (iocb->ki_flags & IOCB_NOWAIT) {
554 ret = -EAGAIN;
555 goto out;
556 }
557 if (needs_zeroing)
558 inode_dio_wait(inode);
559 *dio_flags = IOMAP_DIO_FORCE_WAIT;
560 }
561
562 ret = kiocb_modified(iocb);
563 if (ret < 0)
564 goto out;
565
566 return count;
567out:
568 if (*ilock_shared)
569 inode_unlock_shared(inode);
570 else
571 inode_unlock(inode);
572 return ret;
573}
574
575static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from)
576{
577 ssize_t ret;
578 handle_t *handle;
579 struct inode *inode = file_inode(iocb->ki_filp);
580 loff_t offset = iocb->ki_pos;
581 size_t count = iov_iter_count(from);
582 bool extend = false;
583 bool ilock_shared = true;
584 int dio_flags = 0;
585
586 /*
587 * Quick check here without any i_rwsem lock to see if it is extending
588 * IO. A more reliable check is done in ext4_dio_write_checks() with
589 * proper locking in place.
590 */
591 if (offset + count > i_size_read(inode))
592 ilock_shared = false;
593
594 if (iocb->ki_flags & IOCB_NOWAIT) {
595 if (ilock_shared) {
596 if (!inode_trylock_shared(inode))
597 return -EAGAIN;
598 } else {
599 if (!inode_trylock(inode))
600 return -EAGAIN;
601 }
602 } else {
603 if (ilock_shared)
604 inode_lock_shared(inode);
605 else
606 inode_lock(inode);
607 }
608
609 /* Fallback to buffered I/O if the inode does not support direct I/O. */
610 if (!ext4_should_use_dio(iocb, from)) {
611 if (ilock_shared)
612 inode_unlock_shared(inode);
613 else
614 inode_unlock(inode);
615 return ext4_buffered_write_iter(iocb, from);
616 }
617
618 /*
619 * Prevent inline data from being created since we are going to allocate
620 * blocks for DIO. We know the inode does not currently have inline data
621 * because ext4_should_use_dio() checked for it, but we have to clear
622 * the state flag before the write checks because a lock cycle could
623 * introduce races with other writers.
624 */
625 ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
626
627 ret = ext4_dio_write_checks(iocb, from, &ilock_shared, &extend,
628 &dio_flags);
629 if (ret <= 0)
630 return ret;
631
632 offset = iocb->ki_pos;
633 count = ret;
634
635 if (extend) {
636 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
637 if (IS_ERR(handle)) {
638 ret = PTR_ERR(handle);
639 goto out;
640 }
641
642 ret = ext4_orphan_add(handle, inode);
643 ext4_journal_stop(handle);
644 if (ret)
645 goto out;
646 }
647
648 ret = iomap_dio_rw(iocb, from, &ext4_iomap_ops, &ext4_dio_write_ops,
649 dio_flags, NULL, 0);
650 if (ret == -ENOTBLK)
651 ret = 0;
652 if (extend) {
653 /*
654 * We always perform extending DIO write synchronously so by
655 * now the IO is completed and ext4_handle_inode_extension()
656 * was called. Cleanup the inode in case of error or race with
657 * writeback of delalloc blocks.
658 */
659 WARN_ON_ONCE(ret == -EIOCBQUEUED);
660 ext4_inode_extension_cleanup(inode, ret < 0);
661 }
662
663out:
664 if (ilock_shared)
665 inode_unlock_shared(inode);
666 else
667 inode_unlock(inode);
668
669 if (ret >= 0 && iov_iter_count(from)) {
670 ssize_t err;
671 loff_t endbyte;
672
673 /*
674 * There is no support for atomic writes on buffered-io yet,
675 * we should never fallback to buffered-io for DIO atomic
676 * writes.
677 */
678 WARN_ON_ONCE(iocb->ki_flags & IOCB_ATOMIC);
679
680 offset = iocb->ki_pos;
681 err = ext4_buffered_write_iter(iocb, from);
682 if (err < 0)
683 return err;
684
685 /*
686 * We need to ensure that the pages within the page cache for
687 * the range covered by this I/O are written to disk and
688 * invalidated. This is in attempt to preserve the expected
689 * direct I/O semantics in the case we fallback to buffered I/O
690 * to complete off the I/O request.
691 */
692 ret += err;
693 endbyte = offset + err - 1;
694 err = filemap_write_and_wait_range(iocb->ki_filp->f_mapping,
695 offset, endbyte);
696 if (!err)
697 invalidate_mapping_pages(iocb->ki_filp->f_mapping,
698 offset >> PAGE_SHIFT,
699 endbyte >> PAGE_SHIFT);
700 }
701
702 return ret;
703}
704
705#ifdef CONFIG_FS_DAX
706static ssize_t
707ext4_dax_write_iter(struct kiocb *iocb, struct iov_iter *from)
708{
709 ssize_t ret;
710 size_t count;
711 loff_t offset;
712 handle_t *handle;
713 bool extend = false;
714 struct inode *inode = file_inode(iocb->ki_filp);
715
716 if (iocb->ki_flags & IOCB_NOWAIT) {
717 if (!inode_trylock(inode))
718 return -EAGAIN;
719 } else {
720 inode_lock(inode);
721 }
722
723 ret = ext4_write_checks(iocb, from);
724 if (ret <= 0)
725 goto out;
726
727 offset = iocb->ki_pos;
728 count = iov_iter_count(from);
729
730 if (offset + count > EXT4_I(inode)->i_disksize) {
731 if (iocb->ki_flags & IOCB_NOWAIT) {
732 ret = -EAGAIN;
733 goto out;
734 }
735
736 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
737 if (IS_ERR(handle)) {
738 ret = PTR_ERR(handle);
739 goto out;
740 }
741
742 ret = ext4_orphan_add(handle, inode);
743 if (ret) {
744 ext4_journal_stop(handle);
745 goto out;
746 }
747
748 extend = true;
749 ext4_journal_stop(handle);
750 }
751
752 ret = dax_iomap_rw(iocb, from, &ext4_iomap_ops);
753
754 if (extend) {
755 ret = ext4_handle_inode_extension(inode, offset, ret, count);
756 ext4_inode_extension_cleanup(inode, ret < (ssize_t)count);
757 }
758out:
759 inode_unlock(inode);
760 if (ret > 0)
761 ret = generic_write_sync(iocb, ret);
762 return ret;
763}
764#endif
765
766static ssize_t
767ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
768{
769 int ret;
770 struct inode *inode = file_inode(iocb->ki_filp);
771
772 ret = ext4_emergency_state(inode->i_sb);
773 if (unlikely(ret))
774 return ret;
775
776#ifdef CONFIG_FS_DAX
777 if (IS_DAX(inode))
778 return ext4_dax_write_iter(iocb, from);
779#endif
780
781 if (iocb->ki_flags & IOCB_ATOMIC) {
782 size_t len = iov_iter_count(from);
783
784 if (len < EXT4_SB(inode->i_sb)->s_awu_min ||
785 len > EXT4_SB(inode->i_sb)->s_awu_max)
786 return -EINVAL;
787
788 ret = generic_atomic_write_valid(iocb, from);
789 if (ret)
790 return ret;
791 }
792
793 if (iocb->ki_flags & IOCB_DIRECT)
794 return ext4_dio_write_iter(iocb, from);
795 else
796 return ext4_buffered_write_iter(iocb, from);
797}
798
799#ifdef CONFIG_FS_DAX
800static vm_fault_t ext4_dax_huge_fault(struct vm_fault *vmf, unsigned int order)
801{
802 int error = 0;
803 vm_fault_t result;
804 int retries = 0;
805 handle_t *handle = NULL;
806 struct inode *inode = file_inode(vmf->vma->vm_file);
807 struct super_block *sb = inode->i_sb;
808
809 /*
810 * We have to distinguish real writes from writes which will result in a
811 * COW page; COW writes should *not* poke the journal (the file will not
812 * be changed). Doing so would cause unintended failures when mounted
813 * read-only.
814 *
815 * We check for VM_SHARED rather than vmf->cow_page since the latter is
816 * unset for order != 0 (i.e. only in do_cow_fault); for
817 * other sizes, dax_iomap_fault will handle splitting / fallback so that
818 * we eventually come back with a COW page.
819 */
820 bool write = (vmf->flags & FAULT_FLAG_WRITE) &&
821 (vmf->vma->vm_flags & VM_SHARED);
822 struct address_space *mapping = vmf->vma->vm_file->f_mapping;
823 unsigned long pfn;
824
825 if (write) {
826 sb_start_pagefault(sb);
827 file_update_time(vmf->vma->vm_file);
828 filemap_invalidate_lock_shared(mapping);
829retry:
830 handle = ext4_journal_start_sb(sb, EXT4_HT_WRITE_PAGE,
831 EXT4_DATA_TRANS_BLOCKS(sb));
832 if (IS_ERR(handle)) {
833 filemap_invalidate_unlock_shared(mapping);
834 sb_end_pagefault(sb);
835 return VM_FAULT_SIGBUS;
836 }
837 } else {
838 filemap_invalidate_lock_shared(mapping);
839 }
840 result = dax_iomap_fault(vmf, order, &pfn, &error, &ext4_iomap_ops);
841 if (write) {
842 ext4_journal_stop(handle);
843
844 if ((result & VM_FAULT_ERROR) && error == -ENOSPC &&
845 ext4_should_retry_alloc(sb, &retries))
846 goto retry;
847 /* Handling synchronous page fault? */
848 if (result & VM_FAULT_NEEDDSYNC)
849 result = dax_finish_sync_fault(vmf, order, pfn);
850 filemap_invalidate_unlock_shared(mapping);
851 sb_end_pagefault(sb);
852 } else {
853 filemap_invalidate_unlock_shared(mapping);
854 }
855
856 return result;
857}
858
859static vm_fault_t ext4_dax_fault(struct vm_fault *vmf)
860{
861 return ext4_dax_huge_fault(vmf, 0);
862}
863
864static const struct vm_operations_struct ext4_dax_vm_ops = {
865 .fault = ext4_dax_fault,
866 .huge_fault = ext4_dax_huge_fault,
867 .page_mkwrite = ext4_dax_fault,
868 .pfn_mkwrite = ext4_dax_fault,
869};
870#else
871#define ext4_dax_vm_ops ext4_file_vm_ops
872#endif
873
874static const struct vm_operations_struct ext4_file_vm_ops = {
875 .fault = filemap_fault,
876 .map_pages = filemap_map_pages,
877 .page_mkwrite = ext4_page_mkwrite,
878};
879
880static int ext4_file_mmap_prepare(struct vm_area_desc *desc)
881{
882 int ret;
883 struct file *file = desc->file;
884 struct inode *inode = file->f_mapping->host;
885 struct dax_device *dax_dev = EXT4_SB(inode->i_sb)->s_daxdev;
886
887 if (file->f_mode & FMODE_WRITE)
888 ret = ext4_emergency_state(inode->i_sb);
889 else
890 ret = ext4_forced_shutdown(inode->i_sb) ? -EIO : 0;
891 if (unlikely(ret))
892 return ret;
893
894 /*
895 * We don't support synchronous mappings for non-DAX files and
896 * for DAX files if underneath dax_device is not synchronous.
897 */
898 if (!daxdev_mapping_supported(desc, file_inode(file), dax_dev))
899 return -EOPNOTSUPP;
900
901 file_accessed(file);
902 if (IS_DAX(file_inode(file))) {
903 desc->vm_ops = &ext4_dax_vm_ops;
904 vma_desc_set_flags(desc, VMA_HUGEPAGE_BIT);
905 } else {
906 desc->vm_ops = &ext4_file_vm_ops;
907 }
908 return 0;
909}
910
911static int ext4_sample_last_mounted(struct super_block *sb,
912 struct vfsmount *mnt)
913{
914 struct ext4_sb_info *sbi = EXT4_SB(sb);
915 struct path path;
916 char buf[64], *cp;
917 handle_t *handle;
918 int err;
919
920 if (likely(ext4_test_mount_flag(sb, EXT4_MF_MNTDIR_SAMPLED)))
921 return 0;
922
923 if (ext4_emergency_state(sb) || sb_rdonly(sb) ||
924 !sb_start_intwrite_trylock(sb))
925 return 0;
926
927 ext4_set_mount_flag(sb, EXT4_MF_MNTDIR_SAMPLED);
928 /*
929 * Sample where the filesystem has been mounted and
930 * store it in the superblock for sysadmin convenience
931 * when trying to sort through large numbers of block
932 * devices or filesystem images.
933 */
934 path.mnt = mnt;
935 path.dentry = mnt->mnt_root;
936 cp = d_path(&path, buf, sizeof(buf));
937 err = 0;
938 if (IS_ERR(cp))
939 goto out;
940
941 handle = ext4_journal_start_sb(sb, EXT4_HT_MISC, 1);
942 err = PTR_ERR(handle);
943 if (IS_ERR(handle))
944 goto out;
945 BUFFER_TRACE(sbi->s_sbh, "get_write_access");
946 err = ext4_journal_get_write_access(handle, sb, sbi->s_sbh,
947 EXT4_JTR_NONE);
948 if (err)
949 goto out_journal;
950 lock_buffer(sbi->s_sbh);
951 strtomem_pad(sbi->s_es->s_last_mounted, cp, 0);
952 ext4_superblock_csum_set(sb);
953 unlock_buffer(sbi->s_sbh);
954 ext4_handle_dirty_metadata(handle, NULL, sbi->s_sbh);
955out_journal:
956 ext4_journal_stop(handle);
957out:
958 sb_end_intwrite(sb);
959 return err;
960}
961
962static int ext4_file_open(struct inode *inode, struct file *filp)
963{
964 int ret;
965
966 if (filp->f_mode & FMODE_WRITE)
967 ret = ext4_emergency_state(inode->i_sb);
968 else
969 ret = ext4_forced_shutdown(inode->i_sb) ? -EIO : 0;
970 if (unlikely(ret))
971 return ret;
972
973 ret = ext4_sample_last_mounted(inode->i_sb, filp->f_path.mnt);
974 if (ret)
975 return ret;
976
977 ret = fscrypt_file_open(inode, filp);
978 if (ret)
979 return ret;
980
981 ret = fsverity_file_open(inode, filp);
982 if (ret)
983 return ret;
984
985 /*
986 * Set up the jbd2_inode if we are opening the inode for
987 * writing and the journal is present
988 */
989 if (filp->f_mode & FMODE_WRITE) {
990 ret = ext4_inode_attach_jinode(inode);
991 if (ret < 0)
992 return ret;
993 }
994
995 if (ext4_inode_can_atomic_write(inode))
996 filp->f_mode |= FMODE_CAN_ATOMIC_WRITE;
997
998 filp->f_mode |= FMODE_NOWAIT | FMODE_CAN_ODIRECT;
999 return dquot_file_open(inode, filp);
1000}
1001
1002/*
1003 * ext4_llseek() handles both block-mapped and extent-mapped maxbytes values
1004 * by calling generic_file_llseek_size() with the appropriate maxbytes
1005 * value for each.
1006 */
1007loff_t ext4_llseek(struct file *file, loff_t offset, int whence)
1008{
1009 struct inode *inode = file->f_mapping->host;
1010 loff_t maxbytes = ext4_get_maxbytes(inode);
1011
1012 switch (whence) {
1013 default:
1014 return generic_file_llseek_size(file, offset, whence,
1015 maxbytes, i_size_read(inode));
1016 case SEEK_HOLE:
1017 inode_lock_shared(inode);
1018 offset = iomap_seek_hole(inode, offset,
1019 &ext4_iomap_report_ops);
1020 inode_unlock_shared(inode);
1021 break;
1022 case SEEK_DATA:
1023 inode_lock_shared(inode);
1024 offset = iomap_seek_data(inode, offset,
1025 &ext4_iomap_report_ops);
1026 inode_unlock_shared(inode);
1027 break;
1028 }
1029
1030 if (offset < 0)
1031 return offset;
1032 return vfs_setpos(file, offset, maxbytes);
1033}
1034
1035const struct file_operations ext4_file_operations = {
1036 .llseek = ext4_llseek,
1037 .read_iter = ext4_file_read_iter,
1038 .write_iter = ext4_file_write_iter,
1039 .iopoll = iocb_bio_iopoll,
1040 .unlocked_ioctl = ext4_ioctl,
1041#ifdef CONFIG_COMPAT
1042 .compat_ioctl = ext4_compat_ioctl,
1043#endif
1044 .mmap_prepare = ext4_file_mmap_prepare,
1045 .open = ext4_file_open,
1046 .release = ext4_release_file,
1047 .fsync = ext4_sync_file,
1048 .get_unmapped_area = thp_get_unmapped_area,
1049 .splice_read = ext4_file_splice_read,
1050 .splice_write = iter_file_splice_write,
1051 .fallocate = ext4_fallocate,
1052 .fop_flags = FOP_MMAP_SYNC | FOP_BUFFER_RASYNC |
1053 FOP_DIO_PARALLEL_WRITE |
1054 FOP_DONTCACHE,
1055 .setlease = generic_setlease,
1056};
1057
1058const struct inode_operations ext4_file_inode_operations = {
1059 .setattr = ext4_setattr,
1060 .getattr = ext4_file_getattr,
1061 .listxattr = ext4_listxattr,
1062 .get_inode_acl = ext4_get_acl,
1063 .set_acl = ext4_set_acl,
1064 .fiemap = ext4_fiemap,
1065 .fileattr_get = ext4_fileattr_get,
1066 .fileattr_set = ext4_fileattr_set,
1067};
1068