projects/linux·blob master · refs

2018 lines · 50.2 KiB

1// SPDX-License-Identifier: LGPL-2.1
2/*
3 * Copyright (c) 2012 Taobao.
4 * Written by Tao Ma <[email protected]>
5 */
6
7#include <linux/iomap.h>
8#include <linux/fiemap.h>
9#include <linux/namei.h>
10#include <linux/iversion.h>
11#include <linux/sched/mm.h>
12
13#include "ext4_jbd2.h"
14#include "ext4.h"
15#include "xattr.h"
16#include "truncate.h"
17
18#define EXT4_XATTR_SYSTEM_DATA "data"
19#define EXT4_MIN_INLINE_DATA_SIZE ((sizeof(__le32) * EXT4_N_BLOCKS))
20#define EXT4_INLINE_DOTDOT_OFFSET 2
21#define EXT4_INLINE_DOTDOT_SIZE 4
22
23
24static int ext4_da_convert_inline_data_to_extent(struct address_space *mapping,
25 struct inode *inode);
26
27static int ext4_get_inline_size(struct inode *inode)
28{
29 if (EXT4_I(inode)->i_inline_off)
30 return EXT4_I(inode)->i_inline_size;
31
32 return 0;
33}
34
35static int get_max_inline_xattr_value_size(struct inode *inode,
36 struct ext4_iloc *iloc)
37{
38 struct ext4_xattr_ibody_header *header;
39 struct ext4_xattr_entry *entry;
40 struct ext4_inode *raw_inode;
41 void *end;
42 int free, min_offs;
43
44 if (!EXT4_INODE_HAS_XATTR_SPACE(inode))
45 return 0;
46
47 min_offs = EXT4_SB(inode->i_sb)->s_inode_size -
48 EXT4_GOOD_OLD_INODE_SIZE -
49 EXT4_I(inode)->i_extra_isize -
50 sizeof(struct ext4_xattr_ibody_header);
51
52 /*
53 * We need to subtract another sizeof(__u32) since an in-inode xattr
54 * needs an empty 4 bytes to indicate the gap between the xattr entry
55 * and the name/value pair.
56 */
57 if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR))
58 return EXT4_XATTR_SIZE(min_offs -
59 EXT4_XATTR_LEN(strlen(EXT4_XATTR_SYSTEM_DATA)) -
60 EXT4_XATTR_ROUND - sizeof(__u32));
61
62 raw_inode = ext4_raw_inode(iloc);
63 header = IHDR(inode, raw_inode);
64 entry = IFIRST(header);
65 end = (void *)raw_inode + EXT4_SB(inode->i_sb)->s_inode_size;
66
67 /* Compute min_offs. */
68 while (!IS_LAST_ENTRY(entry)) {
69 void *next = EXT4_XATTR_NEXT(entry);
70
71 if (next >= end) {
72 EXT4_ERROR_INODE(inode,
73 "corrupt xattr in inline inode");
74 return 0;
75 }
76 if (!entry->e_value_inum && entry->e_value_size) {
77 size_t offs = le16_to_cpu(entry->e_value_offs);
78 if (offs < min_offs)
79 min_offs = offs;
80 }
81 entry = next;
82 }
83 free = min_offs -
84 ((void *)entry - (void *)IFIRST(header)) - sizeof(__u32);
85
86 if (EXT4_I(inode)->i_inline_off) {
87 entry = (struct ext4_xattr_entry *)
88 ((void *)raw_inode + EXT4_I(inode)->i_inline_off);
89
90 free += EXT4_XATTR_SIZE(le32_to_cpu(entry->e_value_size));
91 goto out;
92 }
93
94 free -= EXT4_XATTR_LEN(strlen(EXT4_XATTR_SYSTEM_DATA));
95
96 if (free > EXT4_XATTR_ROUND)
97 free = EXT4_XATTR_SIZE(free - EXT4_XATTR_ROUND);
98 else
99 free = 0;
100
101out:
102 return free;
103}
104
105/*
106 * Get the maximum size we now can store in an inode.
107 * If we can't find the space for a xattr entry, don't use the space
108 * of the extents since we have no space to indicate the inline data.
109 */
110int ext4_get_max_inline_size(struct inode *inode)
111{
112 int error, max_inline_size;
113 struct ext4_iloc iloc;
114
115 if (EXT4_I(inode)->i_extra_isize == 0)
116 return 0;
117
118 error = ext4_get_inode_loc(inode, &iloc);
119 if (error) {
120 ext4_error_inode_err(inode, __func__, __LINE__, 0, -error,
121 "can't get inode location %llu",
122 inode->i_ino);
123 return 0;
124 }
125
126 down_read(&EXT4_I(inode)->xattr_sem);
127 max_inline_size = get_max_inline_xattr_value_size(inode, &iloc);
128 up_read(&EXT4_I(inode)->xattr_sem);
129
130 brelse(iloc.bh);
131
132 if (!max_inline_size)
133 return 0;
134
135 return max_inline_size + EXT4_MIN_INLINE_DATA_SIZE;
136}
137
138/*
139 * this function does not take xattr_sem, which is OK because it is
140 * currently only used in a code path coming form ext4_iget, before
141 * the new inode has been unlocked
142 */
143int ext4_find_inline_data_nolock(struct inode *inode)
144{
145 struct ext4_xattr_ibody_find is = {
146 .s = { .not_found = -ENODATA, },
147 };
148 struct ext4_xattr_info i = {
149 .name_index = EXT4_XATTR_INDEX_SYSTEM,
150 .name = EXT4_XATTR_SYSTEM_DATA,
151 };
152 int error;
153
154 if (EXT4_I(inode)->i_extra_isize == 0)
155 return 0;
156
157 error = ext4_get_inode_loc(inode, &is.iloc);
158 if (error)
159 return error;
160
161 error = ext4_xattr_ibody_find(inode, &i, &is);
162 if (error)
163 goto out;
164
165 if (!is.s.not_found) {
166 if (is.s.here->e_value_inum) {
167 EXT4_ERROR_INODE(inode, "inline data xattr refers "
168 "to an external xattr inode");
169 error = -EFSCORRUPTED;
170 goto out;
171 }
172 EXT4_I(inode)->i_inline_off = (u16)((void *)is.s.here -
173 (void *)ext4_raw_inode(&is.iloc));
174 EXT4_I(inode)->i_inline_size = EXT4_MIN_INLINE_DATA_SIZE +
175 le32_to_cpu(is.s.here->e_value_size);
176 }
177out:
178 brelse(is.iloc.bh);
179 return error;
180}
181
182static int ext4_read_inline_data(struct inode *inode, void *buffer,
183 unsigned int len,
184 struct ext4_iloc *iloc)
185{
186 struct ext4_xattr_entry *entry;
187 struct ext4_xattr_ibody_header *header;
188 int cp_len = 0;
189 struct ext4_inode *raw_inode;
190
191 if (!len)
192 return 0;
193
194 BUG_ON(len > EXT4_I(inode)->i_inline_size);
195
196 cp_len = min_t(unsigned int, len, EXT4_MIN_INLINE_DATA_SIZE);
197
198 raw_inode = ext4_raw_inode(iloc);
199 memcpy(buffer, (void *)(raw_inode->i_block), cp_len);
200
201 len -= cp_len;
202 buffer += cp_len;
203
204 if (!len)
205 goto out;
206
207 header = IHDR(inode, raw_inode);
208 entry = (struct ext4_xattr_entry *)((void *)raw_inode +
209 EXT4_I(inode)->i_inline_off);
210 len = min_t(unsigned int, len,
211 (unsigned int)le32_to_cpu(entry->e_value_size));
212
213 memcpy(buffer,
214 (void *)IFIRST(header) + le16_to_cpu(entry->e_value_offs), len);
215 cp_len += len;
216
217out:
218 return cp_len;
219}
220
221/*
222 * write the buffer to the inline inode.
223 * If 'create' is set, we don't need to do the extra copy in the xattr
224 * value since it is already handled by ext4_xattr_ibody_set.
225 * That saves us one memcpy.
226 */
227static void ext4_write_inline_data(struct inode *inode, struct ext4_iloc *iloc,
228 void *buffer, loff_t pos, unsigned int len)
229{
230 struct ext4_xattr_entry *entry;
231 struct ext4_xattr_ibody_header *header;
232 struct ext4_inode *raw_inode;
233 int cp_len = 0;
234
235 if (unlikely(ext4_emergency_state(inode->i_sb)))
236 return;
237
238 BUG_ON(!EXT4_I(inode)->i_inline_off);
239 BUG_ON(pos + len > EXT4_I(inode)->i_inline_size);
240
241 raw_inode = ext4_raw_inode(iloc);
242 buffer += pos;
243
244 if (pos < EXT4_MIN_INLINE_DATA_SIZE) {
245 cp_len = pos + len > EXT4_MIN_INLINE_DATA_SIZE ?
246 EXT4_MIN_INLINE_DATA_SIZE - pos : len;
247 memcpy((void *)raw_inode->i_block + pos, buffer, cp_len);
248
249 len -= cp_len;
250 buffer += cp_len;
251 pos += cp_len;
252 }
253
254 if (!len)
255 return;
256
257 pos -= EXT4_MIN_INLINE_DATA_SIZE;
258 header = IHDR(inode, raw_inode);
259 entry = (struct ext4_xattr_entry *)((void *)raw_inode +
260 EXT4_I(inode)->i_inline_off);
261
262 memcpy((void *)IFIRST(header) + le16_to_cpu(entry->e_value_offs) + pos,
263 buffer, len);
264}
265
266static int ext4_create_inline_data(handle_t *handle,
267 struct inode *inode, unsigned len)
268{
269 int error;
270 void *value = NULL;
271 struct ext4_xattr_ibody_find is = {
272 .s = { .not_found = -ENODATA, },
273 };
274 struct ext4_xattr_info i = {
275 .name_index = EXT4_XATTR_INDEX_SYSTEM,
276 .name = EXT4_XATTR_SYSTEM_DATA,
277 };
278
279 error = ext4_get_inode_loc(inode, &is.iloc);
280 if (error)
281 return error;
282
283 BUFFER_TRACE(is.iloc.bh, "get_write_access");
284 error = ext4_journal_get_write_access(handle, inode->i_sb, is.iloc.bh,
285 EXT4_JTR_NONE);
286 if (error)
287 goto out;
288
289 if (len > EXT4_MIN_INLINE_DATA_SIZE) {
290 value = EXT4_ZERO_XATTR_VALUE;
291 len -= EXT4_MIN_INLINE_DATA_SIZE;
292 } else {
293 value = "";
294 len = 0;
295 }
296
297 /* Insert the xttr entry. */
298 i.value = value;
299 i.value_len = len;
300
301 error = ext4_xattr_ibody_find(inode, &i, &is);
302 if (error)
303 goto out;
304
305 if (!is.s.not_found) {
306 EXT4_ERROR_INODE(inode, "unexpected inline data xattr");
307 error = -EFSCORRUPTED;
308 goto out;
309 }
310
311 error = ext4_xattr_ibody_set(handle, inode, &i, &is);
312 if (error) {
313 if (error == -ENOSPC)
314 ext4_clear_inode_state(inode,
315 EXT4_STATE_MAY_INLINE_DATA);
316 goto out;
317 }
318
319 memset((void *)ext4_raw_inode(&is.iloc)->i_block,
320 0, EXT4_MIN_INLINE_DATA_SIZE);
321
322 EXT4_I(inode)->i_inline_off = (u16)((void *)is.s.here -
323 (void *)ext4_raw_inode(&is.iloc));
324 EXT4_I(inode)->i_inline_size = len + EXT4_MIN_INLINE_DATA_SIZE;
325 ext4_clear_inode_flag(inode, EXT4_INODE_EXTENTS);
326 ext4_set_inode_flag(inode, EXT4_INODE_INLINE_DATA);
327 get_bh(is.iloc.bh);
328 error = ext4_mark_iloc_dirty(handle, inode, &is.iloc);
329
330out:
331 brelse(is.iloc.bh);
332 return error;
333}
334
335static int ext4_update_inline_data(handle_t *handle, struct inode *inode,
336 unsigned int len)
337{
338 int error;
339 void *value = NULL;
340 struct ext4_xattr_ibody_find is = {
341 .s = { .not_found = -ENODATA, },
342 };
343 struct ext4_xattr_info i = {
344 .name_index = EXT4_XATTR_INDEX_SYSTEM,
345 .name = EXT4_XATTR_SYSTEM_DATA,
346 };
347
348 /* If the old space is ok, write the data directly. */
349 if (len <= EXT4_I(inode)->i_inline_size)
350 return 0;
351
352 error = ext4_get_inode_loc(inode, &is.iloc);
353 if (error)
354 return error;
355
356 error = ext4_xattr_ibody_find(inode, &i, &is);
357 if (error)
358 goto out;
359
360 if (is.s.not_found) {
361 EXT4_ERROR_INODE(inode, "missing inline data xattr");
362 error = -EFSCORRUPTED;
363 goto out;
364 }
365
366 len -= EXT4_MIN_INLINE_DATA_SIZE;
367 value = kzalloc(len, GFP_NOFS);
368 if (!value) {
369 error = -ENOMEM;
370 goto out;
371 }
372
373 error = ext4_xattr_ibody_get(inode, i.name_index, i.name,
374 value, len);
375 if (error < 0)
376 goto out;
377
378 BUFFER_TRACE(is.iloc.bh, "get_write_access");
379 error = ext4_journal_get_write_access(handle, inode->i_sb, is.iloc.bh,
380 EXT4_JTR_NONE);
381 if (error)
382 goto out;
383
384 /* Update the xattr entry. */
385 i.value = value;
386 i.value_len = len;
387
388 error = ext4_xattr_ibody_set(handle, inode, &i, &is);
389 if (error)
390 goto out;
391
392 EXT4_I(inode)->i_inline_off = (u16)((void *)is.s.here -
393 (void *)ext4_raw_inode(&is.iloc));
394 EXT4_I(inode)->i_inline_size = EXT4_MIN_INLINE_DATA_SIZE +
395 le32_to_cpu(is.s.here->e_value_size);
396 ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
397 get_bh(is.iloc.bh);
398 error = ext4_mark_iloc_dirty(handle, inode, &is.iloc);
399
400out:
401 kfree(value);
402 brelse(is.iloc.bh);
403 return error;
404}
405
406static int ext4_prepare_inline_data(handle_t *handle, struct inode *inode,
407 loff_t len)
408{
409 int ret, size, no_expand;
410 struct ext4_inode_info *ei = EXT4_I(inode);
411
412 if (!ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA))
413 return -ENOSPC;
414
415 size = ext4_get_max_inline_size(inode);
416 if (size < len)
417 return -ENOSPC;
418
419 ext4_write_lock_xattr(inode, &no_expand);
420 /*
421 * ei->i_inline_size may have changed since the initial check
422 * if other xattrs were added. Recalculate to ensure
423 * ext4_update_inline_data() validates against current capacity.
424 */
425 (void) ext4_find_inline_data_nolock(inode);
426 if (ei->i_inline_off)
427 ret = ext4_update_inline_data(handle, inode, len);
428 else
429 ret = ext4_create_inline_data(handle, inode, len);
430
431 ext4_write_unlock_xattr(inode, &no_expand);
432 return ret;
433}
434
435static int ext4_destroy_inline_data_nolock(handle_t *handle,
436 struct inode *inode)
437{
438 struct ext4_inode_info *ei = EXT4_I(inode);
439 struct ext4_xattr_ibody_find is = {
440 .s = { .not_found = 0, },
441 };
442 struct ext4_xattr_info i = {
443 .name_index = EXT4_XATTR_INDEX_SYSTEM,
444 .name = EXT4_XATTR_SYSTEM_DATA,
445 .value = NULL,
446 .value_len = 0,
447 };
448 int error;
449
450 if (!ei->i_inline_off)
451 return 0;
452
453 down_write(&ei->i_data_sem);
454
455 error = ext4_get_inode_loc(inode, &is.iloc);
456 if (error) {
457 up_write(&ei->i_data_sem);
458 return error;
459 }
460
461 error = ext4_xattr_ibody_find(inode, &i, &is);
462 if (error)
463 goto out;
464
465 BUFFER_TRACE(is.iloc.bh, "get_write_access");
466 error = ext4_journal_get_write_access(handle, inode->i_sb, is.iloc.bh,
467 EXT4_JTR_NONE);
468 if (error)
469 goto out;
470
471 error = ext4_xattr_ibody_set(handle, inode, &i, &is);
472 if (error)
473 goto out;
474
475 memset((void *)ext4_raw_inode(&is.iloc)->i_block,
476 0, EXT4_MIN_INLINE_DATA_SIZE);
477 memset(ei->i_data, 0, EXT4_MIN_INLINE_DATA_SIZE);
478
479 if (ext4_has_feature_extents(inode->i_sb)) {
480 if (S_ISDIR(inode->i_mode) ||
481 S_ISREG(inode->i_mode) || S_ISLNK(inode->i_mode)) {
482 ext4_set_inode_flag(inode, EXT4_INODE_EXTENTS);
483 ext4_ext_tree_init(handle, inode);
484 }
485 }
486 ext4_clear_inode_flag(inode, EXT4_INODE_INLINE_DATA);
487
488 get_bh(is.iloc.bh);
489 error = ext4_mark_iloc_dirty(handle, inode, &is.iloc);
490
491 EXT4_I(inode)->i_inline_off = 0;
492 EXT4_I(inode)->i_inline_size = 0;
493 ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
494out:
495 brelse(is.iloc.bh);
496 if (error == -ENODATA)
497 error = 0;
498 up_write(&ei->i_data_sem);
499 return error;
500}
501
502static int ext4_read_inline_folio(struct inode *inode, struct folio *folio)
503{
504 void *kaddr;
505 int ret = 0;
506 size_t len;
507 struct ext4_iloc iloc;
508
509 BUG_ON(!folio_test_locked(folio));
510 BUG_ON(!ext4_has_inline_data(inode));
511 BUG_ON(folio->index);
512
513 if (!EXT4_I(inode)->i_inline_off) {
514 ext4_warning(inode->i_sb, "inode %llu doesn't have inline data.",
515 inode->i_ino);
516 goto out;
517 }
518
519 ret = ext4_get_inode_loc(inode, &iloc);
520 if (ret)
521 goto out;
522
523 len = min_t(size_t, ext4_get_inline_size(inode), i_size_read(inode));
524
525 if (len > PAGE_SIZE) {
526 ext4_error_inode(inode, __func__, __LINE__, 0,
527 "inline size %zu exceeds PAGE_SIZE", len);
528 ret = -EFSCORRUPTED;
529 brelse(iloc.bh);
530 goto out;
531 }
532
533 kaddr = kmap_local_folio(folio, 0);
534 ret = ext4_read_inline_data(inode, kaddr, len, &iloc);
535 kaddr = folio_zero_tail(folio, len, kaddr + len);
536 kunmap_local(kaddr);
537 folio_mark_uptodate(folio);
538 brelse(iloc.bh);
539
540out:
541 return ret;
542}
543
544int ext4_readpage_inline(struct inode *inode, struct folio *folio)
545{
546 int ret = 0;
547
548 down_read(&EXT4_I(inode)->xattr_sem);
549 if (!ext4_has_inline_data(inode)) {
550 up_read(&EXT4_I(inode)->xattr_sem);
551 return -EAGAIN;
552 }
553
554 /*
555 * Current inline data can only exist in the 1st page,
556 * So for all the other pages, just set them uptodate.
557 */
558 if (!folio->index)
559 ret = ext4_read_inline_folio(inode, folio);
560 else if (!folio_test_uptodate(folio)) {
561 folio_zero_segment(folio, 0, folio_size(folio));
562 folio_mark_uptodate(folio);
563 }
564
565 up_read(&EXT4_I(inode)->xattr_sem);
566
567 folio_unlock(folio);
568 return ret >= 0 ? 0 : ret;
569}
570
571static int ext4_convert_inline_data_to_extent(struct address_space *mapping,
572 struct inode *inode)
573{
574 int ret, needed_blocks, no_expand;
575 handle_t *handle = NULL;
576 int retries = 0, sem_held = 0;
577 struct folio *folio = NULL;
578 unsigned from, to;
579 struct ext4_iloc iloc;
580
581 if (!ext4_has_inline_data(inode)) {
582 /*
583 * clear the flag so that no new write
584 * will trap here again.
585 */
586 ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
587 return 0;
588 }
589
590 needed_blocks = ext4_chunk_trans_extent(inode, 1);
591
592 ret = ext4_get_inode_loc(inode, &iloc);
593 if (ret)
594 return ret;
595
596retry:
597 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks);
598 if (IS_ERR(handle)) {
599 ret = PTR_ERR(handle);
600 handle = NULL;
601 goto out;
602 }
603
604 /* We cannot recurse into the filesystem as the transaction is already
605 * started */
606 folio = __filemap_get_folio(mapping, 0, FGP_WRITEBEGIN | FGP_NOFS,
607 mapping_gfp_mask(mapping));
608 if (IS_ERR(folio)) {
609 ret = PTR_ERR(folio);
610 goto out_nofolio;
611 }
612
613 ext4_write_lock_xattr(inode, &no_expand);
614 sem_held = 1;
615 /* If some one has already done this for us, just exit. */
616 if (!ext4_has_inline_data(inode)) {
617 ret = 0;
618 goto out;
619 }
620
621 from = 0;
622 to = ext4_get_inline_size(inode);
623 if (!folio_test_uptodate(folio)) {
624 ret = ext4_read_inline_folio(inode, folio);
625 if (ret < 0)
626 goto out;
627 }
628
629 ext4_fc_track_inode(handle, inode);
630 ret = ext4_destroy_inline_data_nolock(handle, inode);
631 if (ret)
632 goto out;
633
634 if (ext4_should_dioread_nolock(inode)) {
635 ret = ext4_block_write_begin(handle, folio, from, to,
636 ext4_get_block_unwritten);
637 } else
638 ret = ext4_block_write_begin(handle, folio, from, to,
639 ext4_get_block);
640 clear_buffer_new(folio_buffers(folio));
641
642 if (!ret && ext4_should_journal_data(inode)) {
643 ret = ext4_walk_page_buffers(handle, inode,
644 folio_buffers(folio), from, to,
645 NULL, do_journal_get_write_access);
646 }
647
648 if (ret) {
649 folio_unlock(folio);
650 folio_put(folio);
651 folio = NULL;
652 ext4_orphan_add(handle, inode);
653 ext4_write_unlock_xattr(inode, &no_expand);
654 sem_held = 0;
655 ext4_journal_stop(handle);
656 handle = NULL;
657 ext4_truncate_failed_write(inode);
658 /*
659 * If truncate failed early the inode might
660 * still be on the orphan list; we need to
661 * make sure the inode is removed from the
662 * orphan list in that case.
663 */
664 if (inode->i_nlink)
665 ext4_orphan_del(NULL, inode);
666 }
667
668 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
669 goto retry;
670
671 if (folio)
672 block_commit_write(folio, from, to);
673out:
674 if (folio) {
675 folio_unlock(folio);
676 folio_put(folio);
677 }
678out_nofolio:
679 if (sem_held)
680 ext4_write_unlock_xattr(inode, &no_expand);
681 if (handle)
682 ext4_journal_stop(handle);
683 brelse(iloc.bh);
684 return ret;
685}
686
687/*
688 * Prepare the write for the inline data.
689 * If the data can be written into the inode, we just read
690 * the page and make it uptodate, and start the journal.
691 * Otherwise read the page, makes it dirty so that it can be
692 * handle in writepages(the i_disksize update is left to the
693 * normal ext4_da_write_end).
694 */
695int ext4_generic_write_inline_data(struct address_space *mapping,
696 struct inode *inode,
697 loff_t pos, unsigned len,
698 struct folio **foliop,
699 bool da)
700{
701 int ret;
702 handle_t *handle;
703 struct folio *folio;
704 struct ext4_iloc iloc;
705 int retries = 0;
706
707 ret = ext4_get_inode_loc(inode, &iloc);
708 if (ret)
709 return ret;
710
711retry_journal:
712 handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
713 if (IS_ERR(handle)) {
714 ret = PTR_ERR(handle);
715 goto out_release_bh;
716 }
717
718 ret = ext4_prepare_inline_data(handle, inode, pos + len);
719 if (ret && ret != -ENOSPC)
720 goto out_stop_journal;
721
722 if (ret == -ENOSPC) {
723 ext4_journal_stop(handle);
724 if (!da) {
725 brelse(iloc.bh);
726 /* Retry inside */
727 return ext4_convert_inline_data_to_extent(mapping, inode);
728 }
729
730 ret = ext4_da_convert_inline_data_to_extent(mapping, inode);
731 if (ret == -ENOSPC &&
732 ext4_should_retry_alloc(inode->i_sb, &retries))
733 goto retry_journal;
734 goto out_release_bh;
735 }
736
737 folio = __filemap_get_folio(mapping, 0, FGP_WRITEBEGIN | FGP_NOFS,
738 mapping_gfp_mask(mapping));
739 if (IS_ERR(folio)) {
740 ret = PTR_ERR(folio);
741 goto out_stop_journal;
742 }
743
744 down_read(&EXT4_I(inode)->xattr_sem);
745 /* Someone else had converted it to extent */
746 if (!ext4_has_inline_data(inode)) {
747 ret = 0;
748 goto out_release_folio;
749 }
750
751 if (!folio_test_uptodate(folio)) {
752 ret = ext4_read_inline_folio(inode, folio);
753 if (ret < 0)
754 goto out_release_folio;
755 }
756
757 ret = ext4_journal_get_write_access(handle, inode->i_sb, iloc.bh, EXT4_JTR_NONE);
758 if (ret)
759 goto out_release_folio;
760 *foliop = folio;
761 up_read(&EXT4_I(inode)->xattr_sem);
762 brelse(iloc.bh);
763 return 1;
764
765out_release_folio:
766 up_read(&EXT4_I(inode)->xattr_sem);
767 folio_unlock(folio);
768 folio_put(folio);
769out_stop_journal:
770 ext4_journal_stop(handle);
771out_release_bh:
772 brelse(iloc.bh);
773 return ret;
774}
775
776/*
777 * Try to write data in the inode.
778 * If the inode has inline data, check whether the new write can be
779 * in the inode also. If not, create the page the handle, move the data
780 * to the page make it update and let the later codes create extent for it.
781 */
782int ext4_try_to_write_inline_data(struct address_space *mapping,
783 struct inode *inode,
784 loff_t pos, unsigned len,
785 struct folio **foliop)
786{
787 if (pos + len > ext4_get_max_inline_size(inode))
788 return ext4_convert_inline_data_to_extent(mapping, inode);
789 return ext4_generic_write_inline_data(mapping, inode, pos, len,
790 foliop, false);
791}
792
793int ext4_write_inline_data_end(struct inode *inode, loff_t pos, unsigned len,
794 unsigned copied, struct folio *folio)
795{
796 handle_t *handle = ext4_journal_current_handle();
797 int no_expand;
798 void *kaddr;
799 struct ext4_iloc iloc;
800 int ret = 0, ret2;
801
802 if (unlikely(copied < len) && !folio_test_uptodate(folio))
803 copied = 0;
804
805 if (likely(copied)) {
806 ret = ext4_get_inode_loc(inode, &iloc);
807 if (ret) {
808 folio_unlock(folio);
809 folio_put(folio);
810 ext4_std_error(inode->i_sb, ret);
811 goto out;
812 }
813 ext4_write_lock_xattr(inode, &no_expand);
814 /*
815 * We could have raced with ext4_page_mkwrite() converting
816 * the inode and clearing the inline data flag, so we just
817 * release resources and retry the whole write.
818 */
819 if (unlikely(!ext4_has_inline_data(inode))) {
820 ext4_write_unlock_xattr(inode, &no_expand);
821 brelse(iloc.bh);
822 folio_unlock(folio);
823 folio_put(folio);
824 ext4_journal_stop(handle);
825 return 0;
826 }
827
828 /*
829 * ei->i_inline_off may have changed since
830 * ext4_write_begin() called
831 * ext4_try_to_write_inline_data()
832 */
833 (void) ext4_find_inline_data_nolock(inode);
834
835 kaddr = kmap_local_folio(folio, 0);
836 ext4_write_inline_data(inode, &iloc, kaddr, pos, copied);
837 kunmap_local(kaddr);
838 folio_mark_uptodate(folio);
839 /* clear dirty flag so that writepages wouldn't work for us. */
840 folio_clear_dirty(folio);
841
842 ext4_write_unlock_xattr(inode, &no_expand);
843 brelse(iloc.bh);
844
845 /*
846 * It's important to update i_size while still holding folio
847 * lock: page writeout could otherwise come in and zero
848 * beyond i_size.
849 */
850 ext4_update_inode_size(inode, pos + copied);
851 }
852 folio_unlock(folio);
853 folio_put(folio);
854
855 /*
856 * Don't mark the inode dirty under folio lock. First, it unnecessarily
857 * makes the holding time of folio lock longer. Second, it forces lock
858 * ordering of folio lock and transaction start for journaling
859 * filesystems.
860 */
861 if (likely(copied))
862 mark_inode_dirty(inode);
863out:
864 /*
865 * If we didn't copy as much data as expected, we need to trim back
866 * size of xattr containing inline data.
867 */
868 if (pos + len > inode->i_size && ext4_can_truncate(inode))
869 ext4_orphan_add(handle, inode);
870
871 ret2 = ext4_journal_stop(handle);
872 if (!ret)
873 ret = ret2;
874 if (pos + len > inode->i_size) {
875 ext4_truncate_failed_write(inode);
876 /*
877 * If truncate failed early the inode might still be
878 * on the orphan list; we need to make sure the inode
879 * is removed from the orphan list in that case.
880 */
881 if (inode->i_nlink)
882 ext4_orphan_del(NULL, inode);
883 }
884 return ret ? ret : copied;
885}
886
887/*
888 * Try to make the page cache and handle ready for the inline data case.
889 * We can call this function in 2 cases:
890 * 1. The inode is created and the first write exceeds inline size. We can
891 * clear the inode state safely.
892 * 2. The inode has inline data, then we need to read the data, make it
893 * update and dirty so that ext4_da_writepages can handle it. We don't
894 * need to start the journal since the file's metadata isn't changed now.
895 */
896static int ext4_da_convert_inline_data_to_extent(struct address_space *mapping,
897 struct inode *inode)
898{
899 int ret = 0, inline_size;
900 struct folio *folio;
901
902 folio = __filemap_get_folio(mapping, 0, FGP_WRITEBEGIN,
903 mapping_gfp_mask(mapping));
904 if (IS_ERR(folio))
905 return PTR_ERR(folio);
906
907 down_read(&EXT4_I(inode)->xattr_sem);
908 if (!ext4_has_inline_data(inode)) {
909 ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
910 goto out;
911 }
912
913 inline_size = ext4_get_inline_size(inode);
914
915 if (!folio_test_uptodate(folio)) {
916 ret = ext4_read_inline_folio(inode, folio);
917 if (ret < 0)
918 goto out;
919 }
920
921 ret = ext4_block_write_begin(NULL, folio, 0, inline_size,
922 ext4_da_get_block_prep);
923 if (ret) {
924 up_read(&EXT4_I(inode)->xattr_sem);
925 folio_unlock(folio);
926 folio_put(folio);
927 ext4_truncate_failed_write(inode);
928 return ret;
929 }
930
931 clear_buffer_new(folio_buffers(folio));
932 folio_mark_dirty(folio);
933 folio_mark_uptodate(folio);
934 ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
935
936out:
937 up_read(&EXT4_I(inode)->xattr_sem);
938 if (folio) {
939 folio_unlock(folio);
940 folio_put(folio);
941 }
942 return ret;
943}
944
945#ifdef INLINE_DIR_DEBUG
946void ext4_show_inline_dir(struct inode *dir, struct buffer_head *bh,
947 void *inline_start, int inline_size)
948{
949 int offset;
950 unsigned short de_len;
951 struct ext4_dir_entry_2 *de = inline_start;
952 void *dlimit = inline_start + inline_size;
953
954 trace_printk("inode %llu\n", dir->i_ino);
955 offset = 0;
956 while ((void *)de < dlimit) {
957 de_len = ext4_rec_len_from_disk(de->rec_len, inline_size);
958 trace_printk("de: off %u rlen %u name %.*s nlen %u ino %u\n",
959 offset, de_len, de->name_len, de->name,
960 de->name_len, le32_to_cpu(de->inode));
961 if (ext4_check_dir_entry(dir, NULL, de, bh,
962 inline_start, inline_size, offset))
963 BUG();
964
965 offset += de_len;
966 de = (struct ext4_dir_entry_2 *) ((char *) de + de_len);
967 }
968}
969#else
970#define ext4_show_inline_dir(dir, bh, inline_start, inline_size)
971#endif
972
973/*
974 * Add a new entry into a inline dir.
975 * It will return -ENOSPC if no space is available, and -EIO
976 * and -EEXIST if directory entry already exists.
977 */
978static int ext4_add_dirent_to_inline(handle_t *handle,
979 struct ext4_filename *fname,
980 struct inode *dir,
981 struct inode *inode,
982 struct ext4_iloc *iloc,
983 void *inline_start, int inline_size)
984{
985 int err;
986 struct ext4_dir_entry_2 *de;
987
988 err = ext4_find_dest_de(dir, iloc->bh, inline_start,
989 inline_size, fname, &de);
990 if (err)
991 return err;
992
993 BUFFER_TRACE(iloc->bh, "get_write_access");
994 err = ext4_journal_get_write_access(handle, dir->i_sb, iloc->bh,
995 EXT4_JTR_NONE);
996 if (err)
997 return err;
998 ext4_insert_dentry(dir, inode, de, inline_size, fname);
999
1000 ext4_show_inline_dir(dir, iloc->bh, inline_start, inline_size);
1001
1002 /*
1003 * XXX shouldn't update any times until successful
1004 * completion of syscall, but too many callers depend
1005 * on this.
1006 *
1007 * XXX similarly, too many callers depend on
1008 * ext4_new_inode() setting the times, but error
1009 * recovery deletes the inode, so the worst that can
1010 * happen is that the times are slightly out of date
1011 * and/or different from the directory change time.
1012 */
1013 inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
1014 ext4_update_dx_flag(dir);
1015 inode_inc_iversion(dir);
1016 return 1;
1017}
1018
1019static void *ext4_get_inline_xattr_pos(struct inode *inode,
1020 struct ext4_iloc *iloc)
1021{
1022 struct ext4_xattr_entry *entry;
1023 struct ext4_xattr_ibody_header *header;
1024
1025 BUG_ON(!EXT4_I(inode)->i_inline_off);
1026
1027 header = IHDR(inode, ext4_raw_inode(iloc));
1028 entry = (struct ext4_xattr_entry *)((void *)ext4_raw_inode(iloc) +
1029 EXT4_I(inode)->i_inline_off);
1030
1031 return (void *)IFIRST(header) + le16_to_cpu(entry->e_value_offs);
1032}
1033
1034/* Set the final de to cover the whole block. */
1035void ext4_update_final_de(void *de_buf, int old_size, int new_size)
1036{
1037 struct ext4_dir_entry_2 *de, *prev_de;
1038 void *limit;
1039 int de_len;
1040
1041 de = de_buf;
1042 if (old_size) {
1043 limit = de_buf + old_size;
1044 do {
1045 prev_de = de;
1046 de_len = ext4_rec_len_from_disk(de->rec_len, old_size);
1047 de_buf += de_len;
1048 de = de_buf;
1049 } while (de_buf < limit);
1050
1051 prev_de->rec_len = ext4_rec_len_to_disk(de_len + new_size -
1052 old_size, new_size);
1053 } else {
1054 /* this is just created, so create an empty entry. */
1055 de->inode = 0;
1056 de->rec_len = ext4_rec_len_to_disk(new_size, new_size);
1057 }
1058}
1059
1060static int ext4_update_inline_dir(handle_t *handle, struct inode *dir,
1061 struct ext4_iloc *iloc)
1062{
1063 int ret;
1064 int old_size = EXT4_I(dir)->i_inline_size - EXT4_MIN_INLINE_DATA_SIZE;
1065 int new_size = get_max_inline_xattr_value_size(dir, iloc);
1066
1067 if (new_size - old_size <= ext4_dir_rec_len(1, NULL))
1068 return -ENOSPC;
1069
1070 ret = ext4_update_inline_data(handle, dir,
1071 new_size + EXT4_MIN_INLINE_DATA_SIZE);
1072 if (ret)
1073 return ret;
1074
1075 ext4_update_final_de(ext4_get_inline_xattr_pos(dir, iloc), old_size,
1076 EXT4_I(dir)->i_inline_size -
1077 EXT4_MIN_INLINE_DATA_SIZE);
1078 dir->i_size = EXT4_I(dir)->i_disksize = EXT4_I(dir)->i_inline_size;
1079 return 0;
1080}
1081
1082static void ext4_restore_inline_data(handle_t *handle, struct inode *inode,
1083 struct ext4_iloc *iloc,
1084 void *buf, int inline_size)
1085{
1086 int ret;
1087
1088 ret = ext4_create_inline_data(handle, inode, inline_size);
1089 if (ret) {
1090 ext4_msg(inode->i_sb, KERN_EMERG,
1091 "error restoring inline_data for inode -- potential data loss! (inode %llu, error %d)",
1092 inode->i_ino, ret);
1093 return;
1094 }
1095 ext4_write_inline_data(inode, iloc, buf, 0, inline_size);
1096 ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
1097}
1098
1099static int ext4_convert_inline_data_nolock(handle_t *handle,
1100 struct inode *inode,
1101 struct ext4_iloc *iloc)
1102{
1103 int error;
1104 void *buf = NULL;
1105 struct buffer_head *data_bh = NULL;
1106 struct ext4_map_blocks map;
1107 int inline_size;
1108
1109 inline_size = ext4_get_inline_size(inode);
1110 buf = kmalloc(inline_size, GFP_NOFS);
1111 if (!buf) {
1112 error = -ENOMEM;
1113 goto out;
1114 }
1115
1116 error = ext4_read_inline_data(inode, buf, inline_size, iloc);
1117 if (error < 0)
1118 goto out;
1119
1120 /*
1121 * Make sure the inline directory entries pass checks before we try to
1122 * convert them, so that we avoid touching stuff that needs fsck.
1123 */
1124 if (S_ISDIR(inode->i_mode)) {
1125 error = ext4_check_all_de(inode, iloc->bh,
1126 buf + EXT4_INLINE_DOTDOT_SIZE,
1127 inline_size - EXT4_INLINE_DOTDOT_SIZE);
1128 if (error)
1129 goto out;
1130 }
1131
1132 error = ext4_destroy_inline_data_nolock(handle, inode);
1133 if (error)
1134 goto out;
1135
1136 map.m_lblk = 0;
1137 map.m_len = 1;
1138 map.m_flags = 0;
1139 error = ext4_map_blocks(handle, inode, &map, EXT4_GET_BLOCKS_CREATE);
1140 if (error < 0)
1141 goto out_restore;
1142 if (!(map.m_flags & EXT4_MAP_MAPPED)) {
1143 error = -EIO;
1144 goto out_restore;
1145 }
1146
1147 data_bh = sb_getblk(inode->i_sb, map.m_pblk);
1148 if (!data_bh) {
1149 error = -ENOMEM;
1150 goto out_restore;
1151 }
1152
1153 lock_buffer(data_bh);
1154 error = ext4_journal_get_create_access(handle, inode->i_sb, data_bh,
1155 EXT4_JTR_NONE);
1156 if (error) {
1157 unlock_buffer(data_bh);
1158 error = -EIO;
1159 goto out_restore;
1160 }
1161 memset(data_bh->b_data, 0, inode->i_sb->s_blocksize);
1162
1163 if (!S_ISDIR(inode->i_mode)) {
1164 memcpy(data_bh->b_data, buf, inline_size);
1165 set_buffer_uptodate(data_bh);
1166 unlock_buffer(data_bh);
1167 error = ext4_handle_dirty_metadata(handle,
1168 inode, data_bh);
1169 } else {
1170 unlock_buffer(data_bh);
1171 inode->i_size = inode->i_sb->s_blocksize;
1172 i_size_write(inode, inode->i_sb->s_blocksize);
1173 EXT4_I(inode)->i_disksize = inode->i_sb->s_blocksize;
1174
1175 error = ext4_init_dirblock(handle, inode, data_bh,
1176 le32_to_cpu(((struct ext4_dir_entry_2 *)buf)->inode),
1177 buf + EXT4_INLINE_DOTDOT_SIZE,
1178 inline_size - EXT4_INLINE_DOTDOT_SIZE);
1179 if (!error)
1180 error = ext4_mark_inode_dirty(handle, inode);
1181 }
1182
1183out_restore:
1184 if (error)
1185 ext4_restore_inline_data(handle, inode, iloc, buf, inline_size);
1186
1187out:
1188 brelse(data_bh);
1189 kfree(buf);
1190 return error;
1191}
1192
1193/*
1194 * Try to add the new entry to the inline data.
1195 * If succeeds, return 0. If not, extended the inline dir and copied data to
1196 * the new created block.
1197 */
1198int ext4_try_add_inline_entry(handle_t *handle, struct ext4_filename *fname,
1199 struct inode *dir, struct inode *inode)
1200{
1201 int ret, ret2, inline_size, no_expand;
1202 void *inline_start;
1203 struct ext4_iloc iloc;
1204
1205 ret = ext4_get_inode_loc(dir, &iloc);
1206 if (ret)
1207 return ret;
1208
1209 ext4_write_lock_xattr(dir, &no_expand);
1210 if (!ext4_has_inline_data(dir))
1211 goto out;
1212
1213 inline_start = (void *)ext4_raw_inode(&iloc)->i_block +
1214 EXT4_INLINE_DOTDOT_SIZE;
1215 inline_size = EXT4_MIN_INLINE_DATA_SIZE - EXT4_INLINE_DOTDOT_SIZE;
1216
1217 ret = ext4_add_dirent_to_inline(handle, fname, dir, inode, &iloc,
1218 inline_start, inline_size);
1219 if (ret != -ENOSPC)
1220 goto out;
1221
1222 /* check whether it can be inserted to inline xattr space. */
1223 inline_size = EXT4_I(dir)->i_inline_size -
1224 EXT4_MIN_INLINE_DATA_SIZE;
1225 if (!inline_size) {
1226 /* Try to use the xattr space.*/
1227 ret = ext4_update_inline_dir(handle, dir, &iloc);
1228 if (ret && ret != -ENOSPC)
1229 goto out;
1230
1231 inline_size = EXT4_I(dir)->i_inline_size -
1232 EXT4_MIN_INLINE_DATA_SIZE;
1233 }
1234
1235 if (inline_size) {
1236 inline_start = ext4_get_inline_xattr_pos(dir, &iloc);
1237
1238 ret = ext4_add_dirent_to_inline(handle, fname, dir,
1239 inode, &iloc, inline_start,
1240 inline_size);
1241
1242 if (ret != -ENOSPC)
1243 goto out;
1244 }
1245
1246 /*
1247 * The inline space is filled up, so create a new block for it.
1248 * As the extent tree will be created, we have to save the inline
1249 * dir first.
1250 */
1251 ret = ext4_convert_inline_data_nolock(handle, dir, &iloc);
1252
1253out:
1254 ext4_write_unlock_xattr(dir, &no_expand);
1255 ret2 = ext4_mark_inode_dirty(handle, dir);
1256 if (unlikely(ret2 && !ret))
1257 ret = ret2;
1258 brelse(iloc.bh);
1259 return ret;
1260}
1261
1262/*
1263 * This function fills a red-black tree with information from an
1264 * inlined dir. It returns the number directory entries loaded
1265 * into the tree. If there is an error it is returned in err.
1266 */
1267int ext4_inlinedir_to_tree(struct file *dir_file,
1268 struct inode *dir, ext4_lblk_t block,
1269 struct dx_hash_info *hinfo,
1270 __u32 start_hash, __u32 start_minor_hash,
1271 int *has_inline_data)
1272{
1273 int err = 0, count = 0;
1274 unsigned int parent_ino;
1275 int pos;
1276 struct ext4_dir_entry_2 *de;
1277 struct inode *inode = file_inode(dir_file);
1278 int ret, inline_size = 0;
1279 struct ext4_iloc iloc;
1280 void *dir_buf = NULL;
1281 struct ext4_dir_entry_2 fake;
1282 struct fscrypt_str tmp_str;
1283
1284 ret = ext4_get_inode_loc(inode, &iloc);
1285 if (ret)
1286 return ret;
1287
1288 down_read(&EXT4_I(inode)->xattr_sem);
1289 if (!ext4_has_inline_data(inode)) {
1290 up_read(&EXT4_I(inode)->xattr_sem);
1291 *has_inline_data = 0;
1292 goto out;
1293 }
1294
1295 inline_size = ext4_get_inline_size(inode);
1296 dir_buf = kmalloc(inline_size, GFP_NOFS);
1297 if (!dir_buf) {
1298 ret = -ENOMEM;
1299 up_read(&EXT4_I(inode)->xattr_sem);
1300 goto out;
1301 }
1302
1303 ret = ext4_read_inline_data(inode, dir_buf, inline_size, &iloc);
1304 up_read(&EXT4_I(inode)->xattr_sem);
1305 if (ret < 0)
1306 goto out;
1307
1308 pos = 0;
1309 parent_ino = le32_to_cpu(((struct ext4_dir_entry_2 *)dir_buf)->inode);
1310 while (pos < inline_size) {
1311 /*
1312 * As inlined dir doesn't store any information about '.' and
1313 * only the inode number of '..' is stored, we have to handle
1314 * them differently.
1315 */
1316 if (pos == 0) {
1317 fake.inode = cpu_to_le32(inode->i_ino);
1318 fake.name_len = 1;
1319 memcpy(fake.name, ".", 2);
1320 fake.rec_len = ext4_rec_len_to_disk(
1321 ext4_dir_rec_len(fake.name_len, NULL),
1322 inline_size);
1323 ext4_set_de_type(inode->i_sb, &fake, S_IFDIR);
1324 de = &fake;
1325 pos = EXT4_INLINE_DOTDOT_OFFSET;
1326 } else if (pos == EXT4_INLINE_DOTDOT_OFFSET) {
1327 fake.inode = cpu_to_le32(parent_ino);
1328 fake.name_len = 2;
1329 memcpy(fake.name, "..", 3);
1330 fake.rec_len = ext4_rec_len_to_disk(
1331 ext4_dir_rec_len(fake.name_len, NULL),
1332 inline_size);
1333 ext4_set_de_type(inode->i_sb, &fake, S_IFDIR);
1334 de = &fake;
1335 pos = EXT4_INLINE_DOTDOT_SIZE;
1336 } else {
1337 de = (struct ext4_dir_entry_2 *)(dir_buf + pos);
1338 pos += ext4_rec_len_from_disk(de->rec_len, inline_size);
1339 if (ext4_check_dir_entry(inode, dir_file, de,
1340 iloc.bh, dir_buf,
1341 inline_size, pos)) {
1342 ret = count;
1343 goto out;
1344 }
1345 }
1346
1347 if (ext4_hash_in_dirent(dir)) {
1348 hinfo->hash = EXT4_DIRENT_HASH(de);
1349 hinfo->minor_hash = EXT4_DIRENT_MINOR_HASH(de);
1350 } else {
1351 err = ext4fs_dirhash(dir, de->name, de->name_len, hinfo);
1352 if (err) {
1353 ret = err;
1354 goto out;
1355 }
1356 }
1357 if ((hinfo->hash < start_hash) ||
1358 ((hinfo->hash == start_hash) &&
1359 (hinfo->minor_hash < start_minor_hash)))
1360 continue;
1361 if (de->inode == 0)
1362 continue;
1363 tmp_str.name = de->name;
1364 tmp_str.len = de->name_len;
1365 err = ext4_htree_store_dirent(dir_file, hinfo->hash,
1366 hinfo->minor_hash, de, &tmp_str);
1367 if (err) {
1368 ret = err;
1369 goto out;
1370 }
1371 count++;
1372 }
1373 ret = count;
1374out:
1375 kfree(dir_buf);
1376 brelse(iloc.bh);
1377 return ret;
1378}
1379
1380/*
1381 * So this function is called when the volume is mkfsed with
1382 * dir_index disabled. In order to keep f_pos persistent
1383 * after we convert from an inlined dir to a blocked based,
1384 * we just pretend that we are a normal dir and return the
1385 * offset as if '.' and '..' really take place.
1386 *
1387 */
1388int ext4_read_inline_dir(struct file *file,
1389 struct dir_context *ctx,
1390 int *has_inline_data)
1391{
1392 unsigned int offset, parent_ino;
1393 int i;
1394 struct ext4_dir_entry_2 *de;
1395 struct super_block *sb;
1396 struct inode *inode = file_inode(file);
1397 int ret, inline_size = 0;
1398 struct ext4_iloc iloc;
1399 void *dir_buf = NULL;
1400 int dotdot_offset, dotdot_size, extra_offset, extra_size;
1401 struct dir_private_info *info = file->private_data;
1402
1403 ret = ext4_get_inode_loc(inode, &iloc);
1404 if (ret)
1405 return ret;
1406
1407 down_read(&EXT4_I(inode)->xattr_sem);
1408 if (!ext4_has_inline_data(inode)) {
1409 up_read(&EXT4_I(inode)->xattr_sem);
1410 *has_inline_data = 0;
1411 goto out;
1412 }
1413
1414 inline_size = ext4_get_inline_size(inode);
1415 dir_buf = kmalloc(inline_size, GFP_NOFS);
1416 if (!dir_buf) {
1417 ret = -ENOMEM;
1418 up_read(&EXT4_I(inode)->xattr_sem);
1419 goto out;
1420 }
1421
1422 ret = ext4_read_inline_data(inode, dir_buf, inline_size, &iloc);
1423 up_read(&EXT4_I(inode)->xattr_sem);
1424 if (ret < 0)
1425 goto out;
1426
1427 ret = 0;
1428 sb = inode->i_sb;
1429 parent_ino = le32_to_cpu(((struct ext4_dir_entry_2 *)dir_buf)->inode);
1430 offset = ctx->pos;
1431
1432 /*
1433 * dotdot_offset and dotdot_size is the real offset and
1434 * size for ".." and "." if the dir is block based while
1435 * the real size for them are only EXT4_INLINE_DOTDOT_SIZE.
1436 * So we will use extra_offset and extra_size to indicate them
1437 * during the inline dir iteration.
1438 */
1439 dotdot_offset = ext4_dir_rec_len(1, NULL);
1440 dotdot_size = dotdot_offset + ext4_dir_rec_len(2, NULL);
1441 extra_offset = dotdot_size - EXT4_INLINE_DOTDOT_SIZE;
1442 extra_size = extra_offset + inline_size;
1443
1444 /*
1445 * If the cookie has changed since the last call to
1446 * readdir(2), then we might be pointing to an invalid
1447 * dirent right now. Scan from the start of the inline
1448 * dir to make sure.
1449 */
1450 if (!inode_eq_iversion(inode, info->cookie)) {
1451 for (i = 0; i < extra_size && i < offset;) {
1452 /*
1453 * "." is with offset 0 and
1454 * ".." is dotdot_offset.
1455 */
1456 if (!i) {
1457 i = dotdot_offset;
1458 continue;
1459 } else if (i == dotdot_offset) {
1460 i = dotdot_size;
1461 continue;
1462 }
1463 /* for other entry, the real offset in
1464 * the buf has to be tuned accordingly.
1465 */
1466 if (i + ext4_dir_rec_len(1, NULL) > extra_size)
1467 break;
1468 de = (struct ext4_dir_entry_2 *)
1469 (dir_buf + i - extra_offset);
1470 /* It's too expensive to do a full
1471 * dirent test each time round this
1472 * loop, but we do have to test at
1473 * least that it is non-zero. A
1474 * failure will be detected in the
1475 * dirent test below. */
1476 if (ext4_rec_len_from_disk(de->rec_len, extra_size)
1477 < ext4_dir_rec_len(1, NULL))
1478 break;
1479 i += ext4_rec_len_from_disk(de->rec_len,
1480 extra_size);
1481 }
1482 offset = i;
1483 ctx->pos = offset;
1484 info->cookie = inode_query_iversion(inode);
1485 }
1486
1487 while (ctx->pos < extra_size) {
1488 if (ctx->pos == 0) {
1489 if (!dir_emit(ctx, ".", 1, inode->i_ino, DT_DIR))
1490 goto out;
1491 ctx->pos = dotdot_offset;
1492 continue;
1493 }
1494
1495 if (ctx->pos == dotdot_offset) {
1496 if (!dir_emit(ctx, "..", 2, parent_ino, DT_DIR))
1497 goto out;
1498 ctx->pos = dotdot_size;
1499 continue;
1500 }
1501
1502 /*
1503 * de lives at dir_buf + ctx->pos - extra_offset, within the
1504 * kmalloc(inline_size) buffer. Make sure its header fits before
1505 * ext4_check_dir_entry() dereferences de->rec_len.
1506 */
1507 if (ctx->pos + ext4_dir_rec_len(1, NULL) > extra_size)
1508 goto out;
1509 de = (struct ext4_dir_entry_2 *)
1510 (dir_buf + ctx->pos - extra_offset);
1511 if (ext4_check_dir_entry(inode, file, de, iloc.bh, dir_buf,
1512 inline_size, ctx->pos))
1513 goto out;
1514 if (le32_to_cpu(de->inode)) {
1515 if (!dir_emit(ctx, de->name, de->name_len,
1516 le32_to_cpu(de->inode),
1517 get_dtype(sb, de->file_type)))
1518 goto out;
1519 }
1520 ctx->pos += ext4_rec_len_from_disk(de->rec_len, extra_size);
1521 }
1522out:
1523 kfree(dir_buf);
1524 brelse(iloc.bh);
1525 return ret;
1526}
1527
1528void *ext4_read_inline_link(struct inode *inode)
1529{
1530 struct ext4_iloc iloc;
1531 int ret, inline_size;
1532 void *link;
1533
1534 ret = ext4_get_inode_loc(inode, &iloc);
1535 if (ret)
1536 return ERR_PTR(ret);
1537
1538 ret = -ENOMEM;
1539 inline_size = ext4_get_inline_size(inode);
1540 link = kmalloc(inline_size + 1, GFP_NOFS);
1541 if (!link)
1542 goto out;
1543
1544 ret = ext4_read_inline_data(inode, link, inline_size, &iloc);
1545 if (ret < 0) {
1546 kfree(link);
1547 goto out;
1548 }
1549 nd_terminate_link(link, inode->i_size, ret);
1550out:
1551 if (ret < 0)
1552 link = ERR_PTR(ret);
1553 brelse(iloc.bh);
1554 return link;
1555}
1556
1557struct buffer_head *ext4_get_first_inline_block(struct inode *inode,
1558 struct ext4_dir_entry_2 **parent_de,
1559 int *retval)
1560{
1561 struct ext4_iloc iloc;
1562
1563 *retval = ext4_get_inode_loc(inode, &iloc);
1564 if (*retval)
1565 return NULL;
1566
1567 *parent_de = (struct ext4_dir_entry_2 *)ext4_raw_inode(&iloc)->i_block;
1568
1569 return iloc.bh;
1570}
1571
1572/*
1573 * Try to create the inline data for the new dir.
1574 * If it succeeds, return 0, otherwise return the error.
1575 * In case of ENOSPC, the caller should create the normal disk layout dir.
1576 */
1577int ext4_try_create_inline_dir(handle_t *handle, struct inode *parent,
1578 struct inode *inode)
1579{
1580 int ret, inline_size = EXT4_MIN_INLINE_DATA_SIZE;
1581 struct ext4_iloc iloc;
1582 struct ext4_dir_entry_2 *de;
1583
1584 ret = ext4_get_inode_loc(inode, &iloc);
1585 if (ret)
1586 return ret;
1587
1588 ret = ext4_prepare_inline_data(handle, inode, inline_size);
1589 if (ret)
1590 goto out;
1591
1592 /*
1593 * For inline dir, we only save the inode information for the ".."
1594 * and create a fake dentry to cover the left space.
1595 */
1596 de = (struct ext4_dir_entry_2 *)ext4_raw_inode(&iloc)->i_block;
1597 de->inode = cpu_to_le32(parent->i_ino);
1598 de = (struct ext4_dir_entry_2 *)((void *)de + EXT4_INLINE_DOTDOT_SIZE);
1599 de->inode = 0;
1600 de->rec_len = ext4_rec_len_to_disk(
1601 inline_size - EXT4_INLINE_DOTDOT_SIZE,
1602 inline_size);
1603 set_nlink(inode, 2);
1604 inode->i_size = EXT4_I(inode)->i_disksize = inline_size;
1605out:
1606 brelse(iloc.bh);
1607 return ret;
1608}
1609
1610struct buffer_head *ext4_find_inline_entry(struct inode *dir,
1611 struct ext4_filename *fname,
1612 struct ext4_dir_entry_2 **res_dir,
1613 int *has_inline_data)
1614{
1615 struct ext4_xattr_ibody_find is = {
1616 .s = { .not_found = -ENODATA, },
1617 };
1618 struct ext4_xattr_info i = {
1619 .name_index = EXT4_XATTR_INDEX_SYSTEM,
1620 .name = EXT4_XATTR_SYSTEM_DATA,
1621 };
1622 int ret;
1623 void *inline_start;
1624 int inline_size;
1625
1626 ret = ext4_get_inode_loc(dir, &is.iloc);
1627 if (ret)
1628 return ERR_PTR(ret);
1629
1630 down_read(&EXT4_I(dir)->xattr_sem);
1631
1632 ret = ext4_xattr_ibody_find(dir, &i, &is);
1633 if (ret)
1634 goto out;
1635
1636 if (!ext4_has_inline_data(dir)) {
1637 *has_inline_data = 0;
1638 goto out;
1639 }
1640
1641 inline_start = (void *)ext4_raw_inode(&is.iloc)->i_block +
1642 EXT4_INLINE_DOTDOT_SIZE;
1643 inline_size = EXT4_MIN_INLINE_DATA_SIZE - EXT4_INLINE_DOTDOT_SIZE;
1644 ret = ext4_search_dir(is.iloc.bh, inline_start, inline_size,
1645 dir, fname, 0, res_dir);
1646 if (ret == 1)
1647 goto out_find;
1648 if (ret < 0)
1649 goto out;
1650
1651 if (ext4_get_inline_size(dir) == EXT4_MIN_INLINE_DATA_SIZE)
1652 goto out;
1653
1654 inline_start = ext4_get_inline_xattr_pos(dir, &is.iloc);
1655 inline_size = ext4_get_inline_size(dir) - EXT4_MIN_INLINE_DATA_SIZE;
1656
1657 ret = ext4_search_dir(is.iloc.bh, inline_start, inline_size,
1658 dir, fname, 0, res_dir);
1659 if (ret == 1)
1660 goto out_find;
1661
1662out:
1663 brelse(is.iloc.bh);
1664 if (ret < 0)
1665 is.iloc.bh = ERR_PTR(ret);
1666 else
1667 is.iloc.bh = NULL;
1668out_find:
1669 up_read(&EXT4_I(dir)->xattr_sem);
1670 return is.iloc.bh;
1671}
1672
1673int ext4_delete_inline_entry(handle_t *handle,
1674 struct inode *dir,
1675 struct ext4_dir_entry_2 *de_del,
1676 struct buffer_head *bh,
1677 int *has_inline_data)
1678{
1679 int err, inline_size, no_expand;
1680 struct ext4_iloc iloc;
1681 void *inline_start;
1682
1683 err = ext4_get_inode_loc(dir, &iloc);
1684 if (err)
1685 return err;
1686
1687 ext4_write_lock_xattr(dir, &no_expand);
1688 if (!ext4_has_inline_data(dir)) {
1689 *has_inline_data = 0;
1690 goto out;
1691 }
1692
1693 if ((void *)de_del - ((void *)ext4_raw_inode(&iloc)->i_block) <
1694 EXT4_MIN_INLINE_DATA_SIZE) {
1695 inline_start = (void *)ext4_raw_inode(&iloc)->i_block +
1696 EXT4_INLINE_DOTDOT_SIZE;
1697 inline_size = EXT4_MIN_INLINE_DATA_SIZE -
1698 EXT4_INLINE_DOTDOT_SIZE;
1699 } else {
1700 inline_start = ext4_get_inline_xattr_pos(dir, &iloc);
1701 inline_size = ext4_get_inline_size(dir) -
1702 EXT4_MIN_INLINE_DATA_SIZE;
1703 }
1704
1705 BUFFER_TRACE(bh, "get_write_access");
1706 err = ext4_journal_get_write_access(handle, dir->i_sb, bh,
1707 EXT4_JTR_NONE);
1708 if (err)
1709 goto out;
1710
1711 err = ext4_generic_delete_entry(dir, de_del, bh,
1712 inline_start, inline_size, 0);
1713 if (err)
1714 goto out;
1715
1716 ext4_show_inline_dir(dir, iloc.bh, inline_start, inline_size);
1717out:
1718 ext4_write_unlock_xattr(dir, &no_expand);
1719 if (likely(err == 0))
1720 err = ext4_mark_inode_dirty(handle, dir);
1721 brelse(iloc.bh);
1722 if (err != -ENOENT)
1723 ext4_std_error(dir->i_sb, err);
1724 return err;
1725}
1726
1727/*
1728 * Get the inline dentry at offset.
1729 */
1730static inline struct ext4_dir_entry_2 *
1731ext4_get_inline_entry(struct inode *inode,
1732 struct ext4_iloc *iloc,
1733 unsigned int offset,
1734 void **inline_start,
1735 int *inline_size)
1736{
1737 void *inline_pos;
1738
1739 BUG_ON(offset > ext4_get_inline_size(inode));
1740
1741 if (offset < EXT4_MIN_INLINE_DATA_SIZE) {
1742 inline_pos = (void *)ext4_raw_inode(iloc)->i_block;
1743 *inline_size = EXT4_MIN_INLINE_DATA_SIZE;
1744 } else {
1745 inline_pos = ext4_get_inline_xattr_pos(inode, iloc);
1746 offset -= EXT4_MIN_INLINE_DATA_SIZE;
1747 *inline_size = ext4_get_inline_size(inode) -
1748 EXT4_MIN_INLINE_DATA_SIZE;
1749 }
1750
1751 if (inline_start)
1752 *inline_start = inline_pos;
1753 return (struct ext4_dir_entry_2 *)(inline_pos + offset);
1754}
1755
1756bool empty_inline_dir(struct inode *dir, int *has_inline_data)
1757{
1758 int err, inline_size;
1759 struct ext4_iloc iloc;
1760 size_t inline_len;
1761 void *inline_pos;
1762 unsigned int offset;
1763 struct ext4_dir_entry_2 *de;
1764 bool ret = false;
1765
1766 err = ext4_get_inode_loc(dir, &iloc);
1767 if (err) {
1768 EXT4_ERROR_INODE_ERR(dir, -err,
1769 "error %d getting inode %llu block",
1770 err, dir->i_ino);
1771 return false;
1772 }
1773
1774 down_read(&EXT4_I(dir)->xattr_sem);
1775 if (!ext4_has_inline_data(dir)) {
1776 *has_inline_data = 0;
1777 ret = true;
1778 goto out;
1779 }
1780
1781 de = (struct ext4_dir_entry_2 *)ext4_raw_inode(&iloc)->i_block;
1782 if (!le32_to_cpu(de->inode)) {
1783 ext4_warning(dir->i_sb,
1784 "bad inline directory (dir #%llu) - no `..'",
1785 dir->i_ino);
1786 goto out;
1787 }
1788
1789 inline_len = ext4_get_inline_size(dir);
1790 offset = EXT4_INLINE_DOTDOT_SIZE;
1791 while (offset < inline_len) {
1792 de = ext4_get_inline_entry(dir, &iloc, offset,
1793 &inline_pos, &inline_size);
1794 if (ext4_check_dir_entry(dir, NULL, de,
1795 iloc.bh, inline_pos,
1796 inline_size, offset)) {
1797 ext4_warning(dir->i_sb,
1798 "bad inline directory (dir #%llu) - "
1799 "inode %u, rec_len %u, name_len %d"
1800 "inline size %d",
1801 dir->i_ino, le32_to_cpu(de->inode),
1802 le16_to_cpu(de->rec_len), de->name_len,
1803 inline_size);
1804 goto out;
1805 }
1806 if (le32_to_cpu(de->inode)) {
1807 goto out;
1808 }
1809 offset += ext4_rec_len_from_disk(de->rec_len, inline_size);
1810 }
1811
1812 ret = true;
1813out:
1814 up_read(&EXT4_I(dir)->xattr_sem);
1815 brelse(iloc.bh);
1816 return ret;
1817}
1818
1819int ext4_destroy_inline_data(handle_t *handle, struct inode *inode)
1820{
1821 int ret, no_expand;
1822
1823 ext4_write_lock_xattr(inode, &no_expand);
1824 ret = ext4_destroy_inline_data_nolock(handle, inode);
1825 ext4_write_unlock_xattr(inode, &no_expand);
1826
1827 return ret;
1828}
1829
1830int ext4_inline_data_iomap(struct inode *inode, struct iomap *iomap)
1831{
1832 __u64 addr;
1833 int error = -EAGAIN;
1834 struct ext4_iloc iloc;
1835
1836 down_read(&EXT4_I(inode)->xattr_sem);
1837 if (!ext4_has_inline_data(inode))
1838 goto out;
1839
1840 error = ext4_get_inode_loc(inode, &iloc);
1841 if (error)
1842 goto out;
1843
1844 addr = (__u64)iloc.bh->b_blocknr << inode->i_sb->s_blocksize_bits;
1845 addr += (char *)ext4_raw_inode(&iloc) - iloc.bh->b_data;
1846 addr += offsetof(struct ext4_inode, i_block);
1847
1848 brelse(iloc.bh);
1849
1850 iomap->addr = addr;
1851 iomap->offset = 0;
1852 iomap->length = min_t(loff_t, ext4_get_inline_size(inode),
1853 i_size_read(inode));
1854 iomap->type = IOMAP_INLINE;
1855 iomap->flags = 0;
1856
1857out:
1858 up_read(&EXT4_I(inode)->xattr_sem);
1859 return error;
1860}
1861
1862int ext4_inline_data_truncate(struct inode *inode, int *has_inline)
1863{
1864 handle_t *handle;
1865 int inline_size, value_len, needed_blocks, no_expand, err = 0;
1866 size_t i_size;
1867 void *value = NULL;
1868 struct ext4_xattr_ibody_find is = {
1869 .s = { .not_found = -ENODATA, },
1870 };
1871 struct ext4_xattr_info i = {
1872 .name_index = EXT4_XATTR_INDEX_SYSTEM,
1873 .name = EXT4_XATTR_SYSTEM_DATA,
1874 };
1875
1876
1877 needed_blocks = ext4_chunk_trans_extent(inode, 1);
1878 handle = ext4_journal_start(inode, EXT4_HT_INODE, needed_blocks);
1879 if (IS_ERR(handle))
1880 return PTR_ERR(handle);
1881
1882 ext4_write_lock_xattr(inode, &no_expand);
1883 if (!ext4_has_inline_data(inode)) {
1884 ext4_write_unlock_xattr(inode, &no_expand);
1885 *has_inline = 0;
1886 ext4_journal_stop(handle);
1887 return 0;
1888 }
1889
1890 if ((err = ext4_orphan_add(handle, inode)) != 0)
1891 goto out;
1892
1893 if ((err = ext4_get_inode_loc(inode, &is.iloc)) != 0)
1894 goto out;
1895
1896 down_write(&EXT4_I(inode)->i_data_sem);
1897 i_size = inode->i_size;
1898 inline_size = ext4_get_inline_size(inode);
1899 EXT4_I(inode)->i_disksize = i_size;
1900
1901 if (i_size < inline_size) {
1902 /*
1903 * if there's inline data to truncate and this file was
1904 * converted to extents after that inline data was written,
1905 * the extent status cache must be cleared to avoid leaving
1906 * behind stale delayed allocated extent entries
1907 */
1908 if (!ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA))
1909 ext4_es_remove_extent(inode, 0, EXT_MAX_BLOCKS);
1910
1911 /* Clear the content in the xattr space. */
1912 if (inline_size > EXT4_MIN_INLINE_DATA_SIZE) {
1913 if ((err = ext4_xattr_ibody_find(inode, &i, &is)) != 0)
1914 goto out_error;
1915
1916 if (is.s.not_found) {
1917 EXT4_ERROR_INODE(inode,
1918 "missing inline data xattr");
1919 err = -EFSCORRUPTED;
1920 goto out_error;
1921 }
1922
1923 value_len = le32_to_cpu(is.s.here->e_value_size);
1924 value = kmalloc(value_len, GFP_NOFS);
1925 if (!value) {
1926 err = -ENOMEM;
1927 goto out_error;
1928 }
1929
1930 err = ext4_xattr_ibody_get(inode, i.name_index,
1931 i.name, value, value_len);
1932 if (err <= 0)
1933 goto out_error;
1934
1935 i.value = value;
1936 i.value_len = i_size > EXT4_MIN_INLINE_DATA_SIZE ?
1937 i_size - EXT4_MIN_INLINE_DATA_SIZE : 0;
1938 err = ext4_xattr_ibody_set(handle, inode, &i, &is);
1939 if (err)
1940 goto out_error;
1941 }
1942
1943 /* Clear the content within i_blocks. */
1944 if (i_size < EXT4_MIN_INLINE_DATA_SIZE) {
1945 void *p = (void *) ext4_raw_inode(&is.iloc)->i_block;
1946 memset(p + i_size, 0,
1947 EXT4_MIN_INLINE_DATA_SIZE - i_size);
1948 }
1949
1950 EXT4_I(inode)->i_inline_size = i_size <
1951 EXT4_MIN_INLINE_DATA_SIZE ?
1952 EXT4_MIN_INLINE_DATA_SIZE : i_size;
1953 }
1954
1955out_error:
1956 up_write(&EXT4_I(inode)->i_data_sem);
1957out:
1958 brelse(is.iloc.bh);
1959 ext4_write_unlock_xattr(inode, &no_expand);
1960 kfree(value);
1961 if (inode->i_nlink)
1962 ext4_orphan_del(handle, inode);
1963
1964 if (err == 0) {
1965 inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode));
1966 err = ext4_mark_inode_dirty(handle, inode);
1967 if (IS_SYNC(inode))
1968 ext4_handle_sync(handle);
1969 }
1970 ext4_journal_stop(handle);
1971 return err;
1972}
1973
1974int ext4_convert_inline_data(struct inode *inode)
1975{
1976 int error, needed_blocks, no_expand;
1977 handle_t *handle;
1978 struct ext4_iloc iloc;
1979
1980 if (!ext4_has_inline_data(inode)) {
1981 ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
1982 return 0;
1983 } else if (!ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
1984 /*
1985 * Inode has inline data but EXT4_STATE_MAY_INLINE_DATA is
1986 * cleared. This means we are in the middle of moving of
1987 * inline data to delay allocated block. Just force writeout
1988 * here to finish conversion.
1989 */
1990 error = filemap_flush(inode->i_mapping);
1991 if (error)
1992 return error;
1993 if (!ext4_has_inline_data(inode))
1994 return 0;
1995 }
1996
1997 needed_blocks = ext4_chunk_trans_extent(inode, 1);
1998
1999 iloc.bh = NULL;
2000 error = ext4_get_inode_loc(inode, &iloc);
2001 if (error)
2002 return error;
2003
2004 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks);
2005 if (IS_ERR(handle)) {
2006 error = PTR_ERR(handle);
2007 goto out_free;
2008 }
2009
2010 ext4_write_lock_xattr(inode, &no_expand);
2011 if (ext4_has_inline_data(inode))
2012 error = ext4_convert_inline_data_nolock(handle, inode, &iloc);
2013 ext4_write_unlock_xattr(inode, &no_expand);
2014 ext4_journal_stop(handle);
2015out_free:
2016 brelse(iloc.bh);
2017 return error;
2018}