projects/linux·blob master · refs

6018 lines · 153.2 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * fs/f2fs/segment.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
7 */
8#include <linux/fs.h>
9#include <linux/f2fs_fs.h>
10#include <linux/bio.h>
11#include <linux/blkdev.h>
12#include <linux/sched/mm.h>
13#include <linux/prefetch.h>
14#include <linux/kthread.h>
15#include <linux/swap.h>
16#include <linux/timer.h>
17#include <linux/freezer.h>
18#include <linux/sched/signal.h>
19#include <linux/random.h>
20#include <linux/fserror.h>
21
22#include "f2fs.h"
23#include "segment.h"
24#include "node.h"
25#include "gc.h"
26#include "iostat.h"
27#include <trace/events/f2fs.h>
28
29#define __reverse_ffz(x) __reverse_ffs(~(x))
30
31static struct kmem_cache *discard_entry_slab;
32static struct kmem_cache *discard_cmd_slab;
33static struct kmem_cache *sit_entry_set_slab;
34static struct kmem_cache *revoke_entry_slab;
35
36static unsigned long __reverse_ulong(unsigned char *str)
37{
38 unsigned long tmp = 0;
39 int shift = 24, idx = 0;
40
41#if BITS_PER_LONG == 64
42 shift = 56;
43#endif
44 while (shift >= 0) {
45 tmp |= (unsigned long)str[idx++] << shift;
46 shift -= BITS_PER_BYTE;
47 }
48 return tmp;
49}
50
51/*
52 * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
53 * MSB and LSB are reversed in a byte by f2fs_set_bit.
54 */
55static inline unsigned long __reverse_ffs(unsigned long word)
56{
57 int num = 0;
58
59#if BITS_PER_LONG == 64
60 if ((word & 0xffffffff00000000UL) == 0)
61 num += 32;
62 else
63 word >>= 32;
64#endif
65 if ((word & 0xffff0000) == 0)
66 num += 16;
67 else
68 word >>= 16;
69
70 if ((word & 0xff00) == 0)
71 num += 8;
72 else
73 word >>= 8;
74
75 if ((word & 0xf0) == 0)
76 num += 4;
77 else
78 word >>= 4;
79
80 if ((word & 0xc) == 0)
81 num += 2;
82 else
83 word >>= 2;
84
85 if ((word & 0x2) == 0)
86 num += 1;
87 return num;
88}
89
90/*
91 * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because
92 * f2fs_set_bit makes MSB and LSB reversed in a byte.
93 * @size must be integral times of unsigned long.
94 * Example:
95 * MSB <--> LSB
96 * f2fs_set_bit(0, bitmap) => 1000 0000
97 * f2fs_set_bit(7, bitmap) => 0000 0001
98 */
99static unsigned long __find_rev_next_bit(const unsigned long *addr,
100 unsigned long size, unsigned long offset)
101{
102 const unsigned long *p = addr + BIT_WORD(offset);
103 unsigned long result = size;
104 unsigned long tmp;
105
106 if (offset >= size)
107 return size;
108
109 size -= (offset & ~(BITS_PER_LONG - 1));
110 offset %= BITS_PER_LONG;
111
112 while (1) {
113 if (*p == 0)
114 goto pass;
115
116 tmp = __reverse_ulong((unsigned char *)p);
117
118 tmp &= ~0UL >> offset;
119 if (size < BITS_PER_LONG)
120 tmp &= (~0UL << (BITS_PER_LONG - size));
121 if (tmp)
122 goto found;
123pass:
124 if (size <= BITS_PER_LONG)
125 break;
126 size -= BITS_PER_LONG;
127 offset = 0;
128 p++;
129 }
130 return result;
131found:
132 return result - size + __reverse_ffs(tmp);
133}
134
135static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
136 unsigned long size, unsigned long offset)
137{
138 const unsigned long *p = addr + BIT_WORD(offset);
139 unsigned long result = size;
140 unsigned long tmp;
141
142 if (offset >= size)
143 return size;
144
145 size -= (offset & ~(BITS_PER_LONG - 1));
146 offset %= BITS_PER_LONG;
147
148 while (1) {
149 if (*p == ~0UL)
150 goto pass;
151
152 tmp = __reverse_ulong((unsigned char *)p);
153
154 if (offset)
155 tmp |= ~0UL << (BITS_PER_LONG - offset);
156 if (size < BITS_PER_LONG)
157 tmp |= ~0UL >> size;
158 if (tmp != ~0UL)
159 goto found;
160pass:
161 if (size <= BITS_PER_LONG)
162 break;
163 size -= BITS_PER_LONG;
164 offset = 0;
165 p++;
166 }
167 return result;
168found:
169 return result - size + __reverse_ffz(tmp);
170}
171
172bool f2fs_need_SSR(struct f2fs_sb_info *sbi)
173{
174 int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);
175 int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);
176 int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA);
177
178 if (f2fs_lfs_mode(sbi))
179 return false;
180 if (sbi->gc_mode == GC_URGENT_HIGH)
181 return true;
182 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
183 return true;
184
185 return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs +
186 SM_I(sbi)->min_ssr_sections + reserved_sections(sbi));
187}
188
189void f2fs_abort_atomic_write(struct inode *inode, bool clean)
190{
191 struct f2fs_inode_info *fi = F2FS_I(inode);
192
193 if (!f2fs_is_atomic_file(inode))
194 return;
195
196 if (clean)
197 truncate_inode_pages_final(inode->i_mapping);
198
199 release_atomic_write_cnt(inode);
200 clear_inode_flag(inode, FI_ATOMIC_COMMITTED);
201 clear_inode_flag(inode, FI_ATOMIC_REPLACE);
202 clear_inode_flag(inode, FI_ATOMIC_FILE);
203 if (is_inode_flag_set(inode, FI_ATOMIC_DIRTIED)) {
204 clear_inode_flag(inode, FI_ATOMIC_DIRTIED);
205 /*
206 * The vfs inode keeps clean during commit, but the f2fs inode
207 * doesn't. So clear the dirty state after commit and let
208 * f2fs_mark_inode_dirty_sync ensure a consistent dirty state.
209 */
210 f2fs_inode_synced(inode);
211 f2fs_mark_inode_dirty_sync(inode, true);
212 }
213 stat_dec_atomic_inode(inode);
214
215 F2FS_I(inode)->atomic_write_task = NULL;
216
217 if (clean) {
218 f2fs_i_size_write(inode, fi->original_i_size);
219 fi->original_i_size = 0;
220 }
221 /* avoid stale dirty inode during eviction */
222 sync_inode_metadata(inode, 0);
223}
224
225static int __replace_atomic_write_block(struct inode *inode, pgoff_t index,
226 block_t new_addr, block_t *old_addr, bool recover)
227{
228 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
229 struct dnode_of_data dn;
230 struct node_info ni;
231 int err;
232
233retry:
234 set_new_dnode(&dn, inode, NULL, NULL, 0);
235 err = f2fs_get_dnode_of_data(&dn, index, ALLOC_NODE);
236 if (err) {
237 if (err == -ENOMEM) {
238 memalloc_retry_wait(GFP_NOFS);
239 goto retry;
240 }
241 return err;
242 }
243
244 err = f2fs_get_node_info(sbi, dn.nid, &ni, false);
245 if (err) {
246 f2fs_put_dnode(&dn);
247 return err;
248 }
249
250 if (recover) {
251 /* dn.data_blkaddr is always valid */
252 if (!__is_valid_data_blkaddr(new_addr)) {
253 if (new_addr == NULL_ADDR)
254 dec_valid_block_count(sbi, inode, 1);
255 f2fs_invalidate_blocks(sbi, dn.data_blkaddr, 1);
256 f2fs_update_data_blkaddr(&dn, new_addr);
257 } else {
258 f2fs_replace_block(sbi, &dn, dn.data_blkaddr,
259 new_addr, ni.version, true, true);
260 }
261 } else {
262 blkcnt_t count = 1;
263
264 err = inc_valid_block_count(sbi, inode, &count, true, false);
265 if (err) {
266 f2fs_put_dnode(&dn);
267 return err;
268 }
269
270 *old_addr = dn.data_blkaddr;
271 f2fs_truncate_data_blocks_range(&dn, 1);
272 dec_valid_block_count(sbi, F2FS_I(inode)->cow_inode, count);
273
274 f2fs_replace_block(sbi, &dn, dn.data_blkaddr, new_addr,
275 ni.version, true, false);
276 }
277
278 f2fs_put_dnode(&dn);
279
280 trace_f2fs_replace_atomic_write_block(inode, F2FS_I(inode)->cow_inode,
281 index, old_addr ? *old_addr : 0, new_addr, recover);
282 return 0;
283}
284
285static void __complete_revoke_list(struct inode *inode, struct list_head *head,
286 bool revoke)
287{
288 struct revoke_entry *cur, *tmp;
289 pgoff_t start_index = 0;
290 bool truncate = is_inode_flag_set(inode, FI_ATOMIC_REPLACE);
291
292 list_for_each_entry_safe(cur, tmp, head, list) {
293 if (revoke) {
294 __replace_atomic_write_block(inode, cur->index,
295 cur->old_addr, NULL, true);
296 } else if (truncate) {
297 f2fs_truncate_hole(inode, start_index, cur->index);
298 start_index = cur->index + 1;
299 }
300
301 list_del(&cur->list);
302 kmem_cache_free(revoke_entry_slab, cur);
303 }
304
305 if (!revoke && truncate)
306 f2fs_do_truncate_blocks(inode, start_index * PAGE_SIZE, false);
307}
308
309static int __f2fs_commit_atomic_write(struct inode *inode)
310{
311 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
312 struct f2fs_inode_info *fi = F2FS_I(inode);
313 struct inode *cow_inode = fi->cow_inode;
314 struct revoke_entry *new;
315 struct list_head revoke_list;
316 block_t blkaddr;
317 struct dnode_of_data dn;
318 pgoff_t len = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
319 pgoff_t off = 0, blen, index;
320 int ret = 0, i;
321
322 INIT_LIST_HEAD(&revoke_list);
323
324 while (len) {
325 blen = min_t(pgoff_t, ADDRS_PER_BLOCK(cow_inode), len);
326
327 set_new_dnode(&dn, cow_inode, NULL, NULL, 0);
328 ret = f2fs_get_dnode_of_data(&dn, off, LOOKUP_NODE_RA);
329 if (ret && ret != -ENOENT) {
330 goto out;
331 } else if (ret == -ENOENT) {
332 ret = 0;
333 if (dn.max_level == 0)
334 goto out;
335 goto next;
336 }
337
338 blen = min((pgoff_t)ADDRS_PER_PAGE(dn.node_folio, cow_inode),
339 len);
340 index = off;
341 for (i = 0; i < blen; i++, dn.ofs_in_node++, index++) {
342 blkaddr = f2fs_data_blkaddr(&dn);
343
344 if (!__is_valid_data_blkaddr(blkaddr)) {
345 continue;
346 } else if (!f2fs_is_valid_blkaddr(sbi, blkaddr,
347 DATA_GENERIC_ENHANCE)) {
348 f2fs_put_dnode(&dn);
349 ret = -EFSCORRUPTED;
350 goto out;
351 }
352
353 new = f2fs_kmem_cache_alloc(revoke_entry_slab, GFP_NOFS,
354 true, NULL);
355
356 ret = __replace_atomic_write_block(inode, index, blkaddr,
357 &new->old_addr, false);
358 if (ret) {
359 f2fs_put_dnode(&dn);
360 kmem_cache_free(revoke_entry_slab, new);
361 goto out;
362 }
363
364 f2fs_update_data_blkaddr(&dn, NULL_ADDR);
365 new->index = index;
366 list_add_tail(&new->list, &revoke_list);
367 }
368 f2fs_put_dnode(&dn);
369next:
370 off += blen;
371 len -= blen;
372 }
373
374out:
375 if (time_to_inject(sbi, FAULT_ATOMIC_TIMEOUT))
376 f2fs_schedule_timeout_killable(DEFAULT_FAULT_TIMEOUT, true);
377
378 if (ret) {
379 sbi->revoked_atomic_block += fi->atomic_write_cnt;
380 } else {
381 sbi->committed_atomic_block += fi->atomic_write_cnt;
382 set_inode_flag(inode, FI_ATOMIC_COMMITTED);
383
384 /*
385 * inode may has no FI_ATOMIC_DIRTIED flag due to no write
386 * before commit.
387 */
388 if (is_inode_flag_set(inode, FI_ATOMIC_DIRTIED)) {
389 /* clear atomic dirty status and set vfs dirty status */
390 clear_inode_flag(inode, FI_ATOMIC_DIRTIED);
391 f2fs_mark_inode_dirty_sync(inode, true);
392 }
393 }
394
395 __complete_revoke_list(inode, &revoke_list, ret ? true : false);
396
397 return ret;
398}
399
400int f2fs_commit_atomic_write(struct inode *inode)
401{
402 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
403 struct f2fs_inode_info *fi = F2FS_I(inode);
404 struct f2fs_lock_context lc;
405 int err;
406
407 err = filemap_write_and_wait_range(inode->i_mapping, 0, LLONG_MAX);
408 if (err)
409 return err;
410
411 f2fs_down_write(&fi->i_gc_rwsem[WRITE]);
412 f2fs_lock_op(sbi, &lc);
413
414 err = __f2fs_commit_atomic_write(inode);
415
416 f2fs_unlock_op(sbi, &lc);
417 f2fs_up_write(&fi->i_gc_rwsem[WRITE]);
418
419 return err;
420}
421
422/*
423 * This function balances dirty node and dentry pages.
424 * In addition, it controls garbage collection.
425 */
426void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
427{
428 if (f2fs_cp_error(sbi))
429 return;
430
431 if (time_to_inject(sbi, FAULT_CHECKPOINT))
432 f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_FAULT_INJECT);
433
434 /* balance_fs_bg is able to be pending */
435 if (need && excess_cached_nats(sbi))
436 f2fs_balance_fs_bg(sbi, false);
437
438 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
439 return;
440
441 /*
442 * We should do GC or end up with checkpoint, if there are so many dirty
443 * dir/node pages without enough free segments.
444 */
445 if (has_enough_free_secs(sbi, 0, 0))
446 return;
447
448 /*
449 * Submit all cached OPU/IPU DATA bios before triggering
450 * foreground GC to avoid potential deadlocks.
451 */
452 f2fs_submit_merged_write(sbi, DATA);
453 f2fs_submit_all_merged_ipu_writes(sbi);
454
455 if (test_opt(sbi, GC_MERGE) && sbi->gc_thread.f2fs_gc_task) {
456 DEFINE_WAIT(wait);
457
458 prepare_to_wait(&sbi->gc_thread.fggc_wq, &wait,
459 TASK_UNINTERRUPTIBLE);
460 wake_up(&sbi->gc_thread.gc_wait_queue_head);
461 io_schedule();
462 finish_wait(&sbi->gc_thread.fggc_wq, &wait);
463 } else {
464 struct f2fs_gc_control gc_control = {
465 .victim_segno = NULL_SEGNO,
466 .init_gc_type = f2fs_sb_has_blkzoned(sbi) ?
467 FG_GC : BG_GC,
468 .no_bg_gc = true,
469 .should_migrate_blocks = false,
470 .err_gc_skipped = false,
471 .nr_free_secs = 1 };
472
473 f2fs_down_write_trace(&sbi->gc_lock, &gc_control.lc);
474 stat_inc_gc_call_count(sbi, FOREGROUND);
475 f2fs_gc(sbi, &gc_control);
476 }
477}
478
479static inline bool excess_dirty_threshold(struct f2fs_sb_info *sbi)
480{
481 int factor = f2fs_rwsem_is_locked(&sbi->cp_rwsem) ? 3 : 2;
482 unsigned int dents = get_pages(sbi, F2FS_DIRTY_DENTS);
483 unsigned int qdata = get_pages(sbi, F2FS_DIRTY_QDATA);
484 unsigned int nodes = get_pages(sbi, F2FS_DIRTY_NODES);
485 unsigned int meta = get_pages(sbi, F2FS_DIRTY_META);
486 unsigned int imeta = get_pages(sbi, F2FS_DIRTY_IMETA);
487 unsigned int threshold =
488 SEGS_TO_BLKS(sbi, (factor * DEFAULT_DIRTY_THRESHOLD));
489 unsigned int global_threshold = threshold * 3 / 2;
490
491 if (dents >= threshold || qdata >= threshold ||
492 nodes >= threshold || meta >= threshold ||
493 imeta >= threshold)
494 return true;
495 return dents + qdata + nodes + meta + imeta > global_threshold;
496}
497
498void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg)
499{
500 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
501 return;
502
503 /* try to shrink extent cache when there is no enough memory */
504 if (!f2fs_available_free_memory(sbi, READ_EXTENT_CACHE))
505 f2fs_shrink_read_extent_tree(sbi,
506 READ_EXTENT_CACHE_SHRINK_NUMBER);
507
508 /* try to shrink age extent cache when there is no enough memory */
509 if (!f2fs_available_free_memory(sbi, AGE_EXTENT_CACHE))
510 f2fs_shrink_age_extent_tree(sbi,
511 AGE_EXTENT_CACHE_SHRINK_NUMBER);
512
513 /* check the # of cached NAT entries */
514 if (!f2fs_available_free_memory(sbi, NAT_ENTRIES))
515 f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
516
517 if (!f2fs_available_free_memory(sbi, FREE_NIDS))
518 f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS);
519 else
520 f2fs_build_free_nids(sbi, false, false);
521
522 if (excess_dirty_nats(sbi) || excess_dirty_threshold(sbi) ||
523 excess_prefree_segs(sbi) || !f2fs_space_for_roll_forward(sbi))
524 goto do_sync;
525
526 /* there is background inflight IO or foreground operation recently */
527 if (is_inflight_io(sbi, REQ_TIME) ||
528 (!f2fs_time_over(sbi, REQ_TIME) && f2fs_rwsem_is_locked(&sbi->cp_rwsem)))
529 return;
530
531 /* exceed periodical checkpoint timeout threshold */
532 if (f2fs_time_over(sbi, CP_TIME))
533 goto do_sync;
534
535 /* checkpoint is the only way to shrink partial cached entries */
536 if (f2fs_available_free_memory(sbi, NAT_ENTRIES) &&
537 f2fs_available_free_memory(sbi, INO_ENTRIES))
538 return;
539
540do_sync:
541 if (test_opt(sbi, DATA_FLUSH) && from_bg) {
542 struct blk_plug plug;
543
544 mutex_lock(&sbi->flush_lock);
545
546 blk_start_plug(&plug);
547 f2fs_sync_dirty_inodes(sbi, FILE_INODE, false);
548 blk_finish_plug(&plug);
549
550 mutex_unlock(&sbi->flush_lock);
551 }
552 stat_inc_cp_call_count(sbi, BACKGROUND);
553 f2fs_sync_fs(sbi->sb, 1);
554}
555
556static int __submit_flush_wait(struct f2fs_sb_info *sbi,
557 struct block_device *bdev)
558{
559 int ret = blkdev_issue_flush(bdev);
560
561 trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER),
562 test_opt(sbi, FLUSH_MERGE), ret);
563 if (!ret)
564 f2fs_update_iostat(sbi, NULL, FS_FLUSH_IO, 0);
565 return ret;
566}
567
568static void f2fs_flush_end_io(struct bio *bio)
569{
570 complete(bio->bi_private);
571}
572
573struct f2fs_flush_bio {
574 struct bio bio;
575 struct completion wait;
576};
577
578static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino)
579{
580 struct f2fs_flush_bio *flush_bio;
581 unsigned long devices = 0;
582 int ret = 0;
583 int i;
584
585 if (!f2fs_is_multi_device(sbi))
586 return __submit_flush_wait(sbi, sbi->sb->s_bdev);
587
588 flush_bio = kmalloc(array_size(sbi->s_ndevs, sizeof(*flush_bio)),
589 GFP_NOFS | __GFP_NOFAIL);
590
591 for (i = 0; i < sbi->s_ndevs; i++) {
592 if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO))
593 continue;
594
595 bio_init(&flush_bio[i].bio, FDEV(i).bdev, NULL, 0,
596 REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH);
597 init_completion(&flush_bio[i].wait);
598 flush_bio[i].bio.bi_private = &flush_bio[i].wait;
599 flush_bio[i].bio.bi_end_io = f2fs_flush_end_io;
600 devices |= BIT(i);
601 submit_bio(&flush_bio[i].bio);
602 }
603
604 for (i = 0; i < sbi->s_ndevs; i++) {
605 int err;
606
607 if (!(devices & BIT(i)))
608 continue;
609
610 wait_for_completion(&flush_bio[i].wait);
611 err = blk_status_to_errno(flush_bio[i].bio.bi_status);
612 trace_f2fs_issue_flush(FDEV(i).bdev, test_opt(sbi, NOBARRIER),
613 test_opt(sbi, FLUSH_MERGE), err);
614 if (!err)
615 f2fs_update_iostat(sbi, NULL, FS_FLUSH_IO, 0);
616 else if (!ret)
617 ret = err;
618 bio_uninit(&flush_bio[i].bio);
619 }
620 kfree(flush_bio);
621 return ret;
622}
623
624static int issue_flush_thread(void *data)
625{
626 struct f2fs_sb_info *sbi = data;
627 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
628 wait_queue_head_t *q = &fcc->flush_wait_queue;
629repeat:
630 if (kthread_should_stop())
631 return 0;
632
633 if (!llist_empty(&fcc->issue_list)) {
634 struct flush_cmd *cmd, *next;
635 int ret;
636
637 fcc->dispatch_list = llist_del_all(&fcc->issue_list);
638 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
639
640 cmd = llist_entry(fcc->dispatch_list, struct flush_cmd, llnode);
641
642 ret = submit_flush_wait(sbi, cmd->ino);
643 atomic_inc(&fcc->issued_flush);
644
645 llist_for_each_entry_safe(cmd, next,
646 fcc->dispatch_list, llnode) {
647 cmd->ret = ret;
648 complete(&cmd->wait);
649 }
650 fcc->dispatch_list = NULL;
651 }
652
653 wait_event_interruptible(*q,
654 kthread_should_stop() || !llist_empty(&fcc->issue_list));
655 goto repeat;
656}
657
658int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino)
659{
660 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
661 struct flush_cmd cmd;
662 int ret;
663
664 if (test_opt(sbi, NOBARRIER))
665 return 0;
666
667 if (!test_opt(sbi, FLUSH_MERGE)) {
668 atomic_inc(&fcc->queued_flush);
669 ret = submit_flush_wait(sbi, ino);
670 atomic_dec(&fcc->queued_flush);
671 atomic_inc(&fcc->issued_flush);
672 return ret;
673 }
674
675 if (atomic_inc_return(&fcc->queued_flush) == 1 ||
676 f2fs_is_multi_device(sbi)) {
677 ret = submit_flush_wait(sbi, ino);
678 atomic_dec(&fcc->queued_flush);
679
680 atomic_inc(&fcc->issued_flush);
681 return ret;
682 }
683
684 cmd.ino = ino;
685 init_completion(&cmd.wait);
686
687 llist_add(&cmd.llnode, &fcc->issue_list);
688
689 /*
690 * update issue_list before we wake up issue_flush thread, this
691 * smp_mb() pairs with another barrier in ___wait_event(), see
692 * more details in comments of waitqueue_active().
693 */
694 smp_mb();
695
696 if (waitqueue_active(&fcc->flush_wait_queue))
697 wake_up(&fcc->flush_wait_queue);
698
699 if (fcc->f2fs_issue_flush) {
700 wait_for_completion(&cmd.wait);
701 atomic_dec(&fcc->queued_flush);
702 } else {
703 struct llist_node *list;
704
705 list = llist_del_all(&fcc->issue_list);
706 if (!list) {
707 wait_for_completion(&cmd.wait);
708 atomic_dec(&fcc->queued_flush);
709 } else {
710 struct flush_cmd *tmp, *next;
711
712 ret = submit_flush_wait(sbi, ino);
713
714 llist_for_each_entry_safe(tmp, next, list, llnode) {
715 if (tmp == &cmd) {
716 cmd.ret = ret;
717 atomic_dec(&fcc->queued_flush);
718 continue;
719 }
720 tmp->ret = ret;
721 complete(&tmp->wait);
722 }
723 }
724 }
725
726 return cmd.ret;
727}
728
729int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi)
730{
731 dev_t dev = sbi->sb->s_bdev->bd_dev;
732 struct flush_cmd_control *fcc;
733
734 if (SM_I(sbi)->fcc_info) {
735 fcc = SM_I(sbi)->fcc_info;
736 if (fcc->f2fs_issue_flush)
737 return 0;
738 goto init_thread;
739 }
740
741 fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL);
742 if (!fcc)
743 return -ENOMEM;
744 atomic_set(&fcc->issued_flush, 0);
745 atomic_set(&fcc->queued_flush, 0);
746 init_waitqueue_head(&fcc->flush_wait_queue);
747 init_llist_head(&fcc->issue_list);
748 SM_I(sbi)->fcc_info = fcc;
749 if (!test_opt(sbi, FLUSH_MERGE))
750 return 0;
751
752init_thread:
753 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
754 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
755 if (IS_ERR(fcc->f2fs_issue_flush)) {
756 int err = PTR_ERR(fcc->f2fs_issue_flush);
757
758 fcc->f2fs_issue_flush = NULL;
759 return err;
760 }
761
762 return 0;
763}
764
765void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free)
766{
767 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
768
769 if (fcc && fcc->f2fs_issue_flush) {
770 struct task_struct *flush_thread = fcc->f2fs_issue_flush;
771
772 fcc->f2fs_issue_flush = NULL;
773 kthread_stop(flush_thread);
774 }
775 if (free) {
776 kfree(fcc);
777 SM_I(sbi)->fcc_info = NULL;
778 }
779}
780
781int f2fs_flush_device_cache(struct f2fs_sb_info *sbi)
782{
783 int ret = 0, i;
784
785 if (!f2fs_is_multi_device(sbi))
786 return 0;
787
788 if (test_opt(sbi, NOBARRIER))
789 return 0;
790
791 for (i = 1; i < sbi->s_ndevs; i++) {
792 int count = DEFAULT_RETRY_IO_COUNT;
793
794 if (!f2fs_test_bit(i, (char *)&sbi->dirty_device))
795 continue;
796
797 do {
798 ret = __submit_flush_wait(sbi, FDEV(i).bdev);
799 if (ret)
800 f2fs_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT);
801 } while (ret && --count);
802
803 if (ret) {
804 f2fs_stop_checkpoint(sbi, false,
805 STOP_CP_REASON_FLUSH_FAIL);
806 break;
807 }
808
809 spin_lock(&sbi->dev_lock);
810 f2fs_clear_bit(i, (char *)&sbi->dirty_device);
811 spin_unlock(&sbi->dev_lock);
812 }
813
814 return ret;
815}
816
817static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
818 enum dirty_type dirty_type)
819{
820 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
821
822 /* need not be added */
823 if (is_curseg(sbi, segno))
824 return;
825
826 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
827 dirty_i->nr_dirty[dirty_type]++;
828
829 if (dirty_type == DIRTY) {
830 struct seg_entry *sentry = get_seg_entry(sbi, segno);
831 enum dirty_type t = sentry->type;
832
833 if (unlikely(t >= DIRTY)) {
834 f2fs_bug_on(sbi, 1);
835 return;
836 }
837 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
838 dirty_i->nr_dirty[t]++;
839
840 if (__is_large_section(sbi)) {
841 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
842 block_t valid_blocks =
843 get_valid_blocks(sbi, segno, true);
844
845 f2fs_bug_on(sbi,
846 (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
847 !valid_blocks) ||
848 valid_blocks == CAP_BLKS_PER_SEC(sbi));
849
850 if (!is_cursec(sbi, secno))
851 set_bit(secno, dirty_i->dirty_secmap);
852 }
853 }
854}
855
856static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
857 enum dirty_type dirty_type)
858{
859 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
860 block_t valid_blocks;
861
862 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
863 dirty_i->nr_dirty[dirty_type]--;
864
865 if (dirty_type == DIRTY) {
866 struct seg_entry *sentry = get_seg_entry(sbi, segno);
867 enum dirty_type t = sentry->type;
868
869 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
870 dirty_i->nr_dirty[t]--;
871
872 valid_blocks = get_valid_blocks(sbi, segno, true);
873 if (valid_blocks == 0) {
874 clear_bit(GET_SEC_FROM_SEG(sbi, segno),
875 dirty_i->victim_secmap);
876#ifdef CONFIG_F2FS_CHECK_FS
877 clear_bit(segno, SIT_I(sbi)->invalid_segmap);
878#endif
879 }
880 if (__is_large_section(sbi)) {
881 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
882
883 if (!valid_blocks ||
884 valid_blocks == CAP_BLKS_PER_SEC(sbi)) {
885 clear_bit(secno, dirty_i->dirty_secmap);
886 return;
887 }
888
889 if (!is_cursec(sbi, secno))
890 set_bit(secno, dirty_i->dirty_secmap);
891 }
892 }
893}
894
895/*
896 * Should not occur error such as -ENOMEM.
897 * Adding dirty entry into seglist is not critical operation.
898 * If a given segment is one of current working segments, it won't be added.
899 */
900static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
901{
902 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
903 unsigned short valid_blocks, ckpt_valid_blocks;
904 unsigned int usable_blocks;
905
906 if (segno == NULL_SEGNO || is_curseg(sbi, segno))
907 return;
908
909 usable_blocks = f2fs_usable_blks_in_seg(sbi, segno);
910 mutex_lock(&dirty_i->seglist_lock);
911
912 valid_blocks = get_valid_blocks(sbi, segno, false);
913 ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false);
914
915 if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) ||
916 ckpt_valid_blocks == usable_blocks)) {
917 __locate_dirty_segment(sbi, segno, PRE);
918 __remove_dirty_segment(sbi, segno, DIRTY);
919 } else if (valid_blocks < usable_blocks) {
920 __locate_dirty_segment(sbi, segno, DIRTY);
921 } else {
922 /* Recovery routine with SSR needs this */
923 __remove_dirty_segment(sbi, segno, DIRTY);
924 }
925
926 mutex_unlock(&dirty_i->seglist_lock);
927}
928
929/* This moves currently empty dirty blocks to prefree. Must hold seglist_lock */
930void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi)
931{
932 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
933 unsigned int segno;
934
935 mutex_lock(&dirty_i->seglist_lock);
936 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
937 if (get_valid_blocks(sbi, segno, false))
938 continue;
939 if (is_curseg(sbi, segno))
940 continue;
941 __locate_dirty_segment(sbi, segno, PRE);
942 __remove_dirty_segment(sbi, segno, DIRTY);
943 }
944 mutex_unlock(&dirty_i->seglist_lock);
945}
946
947block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi)
948{
949 int ovp_hole_segs =
950 (overprovision_segments(sbi) - reserved_segments(sbi));
951 block_t ovp_holes = SEGS_TO_BLKS(sbi, ovp_hole_segs);
952 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
953 block_t holes[2] = {0, 0}; /* DATA and NODE */
954 block_t unusable;
955 struct seg_entry *se;
956 unsigned int segno;
957
958 mutex_lock(&dirty_i->seglist_lock);
959 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
960 se = get_seg_entry(sbi, segno);
961 if (IS_NODESEG(se->type))
962 holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) -
963 se->valid_blocks;
964 else
965 holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) -
966 se->valid_blocks;
967 }
968 mutex_unlock(&dirty_i->seglist_lock);
969
970 unusable = max(holes[DATA], holes[NODE]);
971 if (unusable > ovp_holes)
972 return unusable - ovp_holes;
973 return 0;
974}
975
976int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable)
977{
978 int ovp_hole_segs =
979 (overprovision_segments(sbi) - reserved_segments(sbi));
980
981 if (F2FS_OPTION(sbi).unusable_cap_perc == 100)
982 return 0;
983 if (unusable > F2FS_OPTION(sbi).unusable_cap)
984 return -EAGAIN;
985 if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) &&
986 dirty_segments(sbi) > ovp_hole_segs)
987 return -EAGAIN;
988 if (has_not_enough_free_secs(sbi, 0, 0))
989 return -EAGAIN;
990 return 0;
991}
992
993/* This is only used by SBI_CP_DISABLED */
994static unsigned int get_free_segment(struct f2fs_sb_info *sbi)
995{
996 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
997 unsigned int segno = 0;
998
999 mutex_lock(&dirty_i->seglist_lock);
1000 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
1001 if (get_valid_blocks(sbi, segno, false))
1002 continue;
1003 if (get_ckpt_valid_blocks(sbi, segno, false))
1004 continue;
1005 mutex_unlock(&dirty_i->seglist_lock);
1006 return segno;
1007 }
1008 mutex_unlock(&dirty_i->seglist_lock);
1009 return NULL_SEGNO;
1010}
1011
1012static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,
1013 struct block_device *bdev, block_t lstart,
1014 block_t start, block_t len)
1015{
1016 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1017 struct list_head *pend_list;
1018 struct discard_cmd *dc;
1019
1020 f2fs_bug_on(sbi, !len);
1021
1022 pend_list = &dcc->pend_list[plist_idx(len)];
1023
1024 dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS, true, NULL);
1025 INIT_LIST_HEAD(&dc->list);
1026 dc->bdev = bdev;
1027 dc->di.lstart = lstart;
1028 dc->di.start = start;
1029 dc->di.len = len;
1030 dc->ref = 0;
1031 dc->state = D_PREP;
1032 dc->queued = 0;
1033 dc->error = 0;
1034 init_completion(&dc->wait);
1035 list_add_tail(&dc->list, pend_list);
1036 spin_lock_init(&dc->lock);
1037 dc->bio_ref = 0;
1038 atomic_inc(&dcc->discard_cmd_cnt);
1039 dcc->undiscard_blks += len;
1040
1041 return dc;
1042}
1043
1044static bool f2fs_check_discard_tree(struct f2fs_sb_info *sbi)
1045{
1046#ifdef CONFIG_F2FS_CHECK_FS
1047 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1048 struct rb_node *cur = rb_first_cached(&dcc->root), *next;
1049 struct discard_cmd *cur_dc, *next_dc;
1050
1051 while (cur) {
1052 next = rb_next(cur);
1053 if (!next)
1054 return true;
1055
1056 cur_dc = rb_entry(cur, struct discard_cmd, rb_node);
1057 next_dc = rb_entry(next, struct discard_cmd, rb_node);
1058
1059 if (cur_dc->di.lstart + cur_dc->di.len > next_dc->di.lstart) {
1060 f2fs_info(sbi, "broken discard_rbtree, "
1061 "cur(%u, %u) next(%u, %u)",
1062 cur_dc->di.lstart, cur_dc->di.len,
1063 next_dc->di.lstart, next_dc->di.len);
1064 return false;
1065 }
1066 cur = next;
1067 }
1068#endif
1069 return true;
1070}
1071
1072static struct discard_cmd *__lookup_discard_cmd(struct f2fs_sb_info *sbi,
1073 block_t blkaddr)
1074{
1075 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1076 struct rb_node *node = dcc->root.rb_root.rb_node;
1077 struct discard_cmd *dc;
1078
1079 while (node) {
1080 dc = rb_entry(node, struct discard_cmd, rb_node);
1081
1082 if (blkaddr < dc->di.lstart)
1083 node = node->rb_left;
1084 else if (blkaddr >= dc->di.lstart + dc->di.len)
1085 node = node->rb_right;
1086 else
1087 return dc;
1088 }
1089 return NULL;
1090}
1091
1092static struct discard_cmd *__lookup_discard_cmd_ret(struct rb_root_cached *root,
1093 block_t blkaddr,
1094 struct discard_cmd **prev_entry,
1095 struct discard_cmd **next_entry,
1096 struct rb_node ***insert_p,
1097 struct rb_node **insert_parent)
1098{
1099 struct rb_node **pnode = &root->rb_root.rb_node;
1100 struct rb_node *parent = NULL, *tmp_node;
1101 struct discard_cmd *dc;
1102
1103 *insert_p = NULL;
1104 *insert_parent = NULL;
1105 *prev_entry = NULL;
1106 *next_entry = NULL;
1107
1108 if (RB_EMPTY_ROOT(&root->rb_root))
1109 return NULL;
1110
1111 while (*pnode) {
1112 parent = *pnode;
1113 dc = rb_entry(*pnode, struct discard_cmd, rb_node);
1114
1115 if (blkaddr < dc->di.lstart)
1116 pnode = &(*pnode)->rb_left;
1117 else if (blkaddr >= dc->di.lstart + dc->di.len)
1118 pnode = &(*pnode)->rb_right;
1119 else
1120 goto lookup_neighbors;
1121 }
1122
1123 *insert_p = pnode;
1124 *insert_parent = parent;
1125
1126 dc = rb_entry(parent, struct discard_cmd, rb_node);
1127 tmp_node = parent;
1128 if (parent && blkaddr > dc->di.lstart)
1129 tmp_node = rb_next(parent);
1130 *next_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node);
1131
1132 tmp_node = parent;
1133 if (parent && blkaddr < dc->di.lstart)
1134 tmp_node = rb_prev(parent);
1135 *prev_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node);
1136 return NULL;
1137
1138lookup_neighbors:
1139 /* lookup prev node for merging backward later */
1140 tmp_node = rb_prev(&dc->rb_node);
1141 *prev_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node);
1142
1143 /* lookup next node for merging frontward later */
1144 tmp_node = rb_next(&dc->rb_node);
1145 *next_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node);
1146 return dc;
1147}
1148
1149static void __detach_discard_cmd(struct discard_cmd_control *dcc,
1150 struct discard_cmd *dc)
1151{
1152 if (dc->state == D_DONE)
1153 atomic_sub(dc->queued, &dcc->queued_discard);
1154
1155 list_del(&dc->list);
1156 rb_erase_cached(&dc->rb_node, &dcc->root);
1157 dcc->undiscard_blks -= dc->di.len;
1158
1159 kmem_cache_free(discard_cmd_slab, dc);
1160
1161 atomic_dec(&dcc->discard_cmd_cnt);
1162}
1163
1164static void __remove_discard_cmd(struct f2fs_sb_info *sbi,
1165 struct discard_cmd *dc)
1166{
1167 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1168 unsigned long flags;
1169
1170 trace_f2fs_remove_discard(dc->bdev, dc->di.start, dc->di.len);
1171
1172 spin_lock_irqsave(&dc->lock, flags);
1173 if (dc->bio_ref) {
1174 spin_unlock_irqrestore(&dc->lock, flags);
1175 return;
1176 }
1177 spin_unlock_irqrestore(&dc->lock, flags);
1178
1179 f2fs_bug_on(sbi, dc->ref);
1180
1181 if (dc->error == -EOPNOTSUPP)
1182 dc->error = 0;
1183
1184 if (dc->error)
1185 f2fs_info_ratelimited(sbi,
1186 "Issue discard(%u, %u, %u) failed, ret: %d",
1187 dc->di.lstart, dc->di.start, dc->di.len, dc->error);
1188 __detach_discard_cmd(dcc, dc);
1189}
1190
1191static void f2fs_submit_discard_endio(struct bio *bio)
1192{
1193 struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private;
1194 unsigned long flags;
1195
1196 spin_lock_irqsave(&dc->lock, flags);
1197 if (!dc->error)
1198 dc->error = blk_status_to_errno(bio->bi_status);
1199 dc->bio_ref--;
1200 if (!dc->bio_ref && dc->state == D_SUBMIT) {
1201 dc->state = D_DONE;
1202 complete_all(&dc->wait);
1203 }
1204 spin_unlock_irqrestore(&dc->lock, flags);
1205 bio_put(bio);
1206}
1207
1208static void __check_sit_bitmap(struct f2fs_sb_info *sbi,
1209 block_t start, block_t end)
1210{
1211#ifdef CONFIG_F2FS_CHECK_FS
1212 struct seg_entry *sentry;
1213 unsigned int segno;
1214 block_t blk = start;
1215 unsigned long offset, size, *map;
1216
1217 while (blk < end) {
1218 segno = GET_SEGNO(sbi, blk);
1219 sentry = get_seg_entry(sbi, segno);
1220 offset = GET_BLKOFF_FROM_SEG0(sbi, blk);
1221
1222 if (end < START_BLOCK(sbi, segno + 1))
1223 size = GET_BLKOFF_FROM_SEG0(sbi, end);
1224 else
1225 size = BLKS_PER_SEG(sbi);
1226 map = (unsigned long *)(sentry->cur_valid_map);
1227 offset = __find_rev_next_bit(map, size, offset);
1228 f2fs_bug_on(sbi, offset != size);
1229 blk = START_BLOCK(sbi, segno + 1);
1230 }
1231#endif
1232}
1233
1234static void __init_discard_policy(struct f2fs_sb_info *sbi,
1235 struct discard_policy *dpolicy,
1236 int discard_type, unsigned int granularity)
1237{
1238 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1239
1240 /* common policy */
1241 dpolicy->type = discard_type;
1242 dpolicy->sync = true;
1243 dpolicy->ordered = false;
1244 dpolicy->granularity = granularity;
1245
1246 dpolicy->max_requests = dcc->max_discard_request;
1247 dpolicy->io_aware_gran = dcc->discard_io_aware_gran;
1248 dpolicy->timeout = false;
1249
1250 if (discard_type == DPOLICY_BG) {
1251 dpolicy->min_interval = dcc->min_discard_issue_time;
1252 dpolicy->mid_interval = dcc->mid_discard_issue_time;
1253 dpolicy->max_interval = dcc->max_discard_issue_time;
1254 if (dcc->discard_io_aware == DPOLICY_IO_AWARE_ENABLE)
1255 dpolicy->io_aware = true;
1256 else if (dcc->discard_io_aware == DPOLICY_IO_AWARE_DISABLE)
1257 dpolicy->io_aware = false;
1258 dpolicy->sync = false;
1259 dpolicy->ordered = true;
1260 if (utilization(sbi) > dcc->discard_urgent_util) {
1261 dpolicy->granularity = MIN_DISCARD_GRANULARITY;
1262 if (atomic_read(&dcc->discard_cmd_cnt))
1263 dpolicy->max_interval =
1264 dcc->min_discard_issue_time;
1265 }
1266 } else if (discard_type == DPOLICY_FORCE) {
1267 dpolicy->min_interval = dcc->min_discard_issue_time;
1268 dpolicy->mid_interval = dcc->mid_discard_issue_time;
1269 dpolicy->max_interval = dcc->max_discard_issue_time;
1270 dpolicy->io_aware = false;
1271 } else if (discard_type == DPOLICY_FSTRIM) {
1272 dpolicy->io_aware = false;
1273 } else if (discard_type == DPOLICY_UMOUNT) {
1274 dpolicy->io_aware = false;
1275 /* we need to issue all to keep CP_TRIMMED_FLAG */
1276 dpolicy->granularity = MIN_DISCARD_GRANULARITY;
1277 dpolicy->timeout = true;
1278 }
1279}
1280
1281static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1282 struct block_device *bdev, block_t lstart,
1283 block_t start, block_t len);
1284
1285#ifdef CONFIG_BLK_DEV_ZONED
1286static void __submit_zone_reset_cmd(struct f2fs_sb_info *sbi,
1287 struct discard_cmd *dc, blk_opf_t flag,
1288 struct list_head *wait_list,
1289 unsigned int *issued)
1290{
1291 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1292 struct block_device *bdev = dc->bdev;
1293 struct bio *bio = bio_alloc(bdev, 0, REQ_OP_ZONE_RESET | flag, GFP_NOFS);
1294 unsigned long flags;
1295
1296 trace_f2fs_issue_reset_zone(bdev, dc->di.start);
1297
1298 spin_lock_irqsave(&dc->lock, flags);
1299 dc->state = D_SUBMIT;
1300 dc->bio_ref++;
1301 spin_unlock_irqrestore(&dc->lock, flags);
1302
1303 if (issued)
1304 (*issued)++;
1305
1306 atomic_inc(&dcc->queued_discard);
1307 dc->queued++;
1308 list_move_tail(&dc->list, wait_list);
1309
1310 /* sanity check on discard range */
1311 __check_sit_bitmap(sbi, dc->di.lstart, dc->di.lstart + dc->di.len);
1312
1313 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(dc->di.start);
1314 bio->bi_private = dc;
1315 bio->bi_end_io = f2fs_submit_discard_endio;
1316 submit_bio(bio);
1317
1318 atomic_inc(&dcc->issued_discard);
1319 f2fs_update_iostat(sbi, NULL, FS_ZONE_RESET_IO, dc->di.len * F2FS_BLKSIZE);
1320}
1321#endif
1322
1323/* this function is copied from blkdev_issue_discard from block/blk-lib.c */
1324static int __submit_discard_cmd(struct f2fs_sb_info *sbi,
1325 struct discard_policy *dpolicy,
1326 struct discard_cmd *dc, int *issued)
1327{
1328 struct block_device *bdev = dc->bdev;
1329 unsigned int max_discard_blocks =
1330 SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev));
1331 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1332 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1333 &(dcc->fstrim_list) : &(dcc->wait_list);
1334 blk_opf_t flag = dpolicy->sync ? REQ_SYNC : 0;
1335 block_t lstart, start, len, total_len;
1336
1337 if (dc->state != D_PREP)
1338 return 0;
1339
1340 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))
1341 return 0;
1342
1343#ifdef CONFIG_BLK_DEV_ZONED
1344 if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev)) {
1345 int devi = f2fs_bdev_index(sbi, bdev);
1346
1347 if (devi < 0)
1348 return -EINVAL;
1349
1350 if (f2fs_blkz_is_seq(sbi, devi, dc->di.start)) {
1351 __submit_zone_reset_cmd(sbi, dc, flag,
1352 wait_list, issued);
1353 return 0;
1354 }
1355 }
1356#endif
1357
1358 /*
1359 * stop issuing discard for any of below cases:
1360 * 1. device is conventional zone, but it doesn't support discard.
1361 * 2. device is regulare device, after snapshot it doesn't support
1362 * discard.
1363 */
1364 if (!bdev_max_discard_sectors(bdev))
1365 return -EOPNOTSUPP;
1366
1367 trace_f2fs_issue_discard(bdev, dc->di.start, dc->di.len);
1368
1369 lstart = dc->di.lstart;
1370 start = dc->di.start;
1371 len = dc->di.len;
1372 total_len = len;
1373
1374 dc->di.len = 0;
1375
1376 while (total_len && *issued < dpolicy->max_requests) {
1377 struct bio *bio = NULL;
1378 unsigned long flags;
1379 bool last = true;
1380
1381 if (len > max_discard_blocks) {
1382 len = max_discard_blocks;
1383 last = false;
1384 }
1385
1386 (*issued)++;
1387 if (*issued == dpolicy->max_requests)
1388 last = true;
1389
1390 dc->di.len += len;
1391
1392 __blkdev_issue_discard(bdev, SECTOR_FROM_BLOCK(start),
1393 SECTOR_FROM_BLOCK(len), GFP_NOFS, &bio);
1394 f2fs_bug_on(sbi, !bio);
1395
1396 /*
1397 * should keep before submission to avoid D_DONE
1398 * right away
1399 */
1400 spin_lock_irqsave(&dc->lock, flags);
1401 if (last)
1402 dc->state = D_SUBMIT;
1403 else
1404 dc->state = D_PARTIAL;
1405 dc->bio_ref++;
1406 spin_unlock_irqrestore(&dc->lock, flags);
1407
1408 atomic_inc(&dcc->queued_discard);
1409 dc->queued++;
1410 list_move_tail(&dc->list, wait_list);
1411
1412 /* sanity check on discard range */
1413 __check_sit_bitmap(sbi, lstart, lstart + len);
1414
1415 bio->bi_private = dc;
1416 bio->bi_end_io = f2fs_submit_discard_endio;
1417 bio->bi_opf |= flag;
1418 submit_bio(bio);
1419
1420 atomic_inc(&dcc->issued_discard);
1421
1422 f2fs_update_iostat(sbi, NULL, FS_DISCARD_IO, len * F2FS_BLKSIZE);
1423
1424 lstart += len;
1425 start += len;
1426 total_len -= len;
1427 len = total_len;
1428 }
1429
1430 if (len) {
1431 dcc->undiscard_blks -= len;
1432 __update_discard_tree_range(sbi, bdev, lstart, start, len);
1433 }
1434 return 0;
1435}
1436
1437static void __insert_discard_cmd(struct f2fs_sb_info *sbi,
1438 struct block_device *bdev, block_t lstart,
1439 block_t start, block_t len)
1440{
1441 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1442 struct rb_node **p = &dcc->root.rb_root.rb_node;
1443 struct rb_node *parent = NULL;
1444 struct discard_cmd *dc;
1445 bool leftmost = true;
1446
1447 /* look up rb tree to find parent node */
1448 while (*p) {
1449 parent = *p;
1450 dc = rb_entry(parent, struct discard_cmd, rb_node);
1451
1452 if (lstart < dc->di.lstart) {
1453 p = &(*p)->rb_left;
1454 } else if (lstart >= dc->di.lstart + dc->di.len) {
1455 p = &(*p)->rb_right;
1456 leftmost = false;
1457 } else {
1458 /* Let's skip to add, if exists */
1459 return;
1460 }
1461 }
1462
1463 dc = __create_discard_cmd(sbi, bdev, lstart, start, len);
1464
1465 rb_link_node(&dc->rb_node, parent, p);
1466 rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost);
1467}
1468
1469static void __relocate_discard_cmd(struct discard_cmd_control *dcc,
1470 struct discard_cmd *dc)
1471{
1472 list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->di.len)]);
1473}
1474
1475static void __punch_discard_cmd(struct f2fs_sb_info *sbi,
1476 struct discard_cmd *dc, block_t blkaddr)
1477{
1478 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1479 struct discard_info di = dc->di;
1480 bool modified = false;
1481
1482 if (dc->state == D_DONE || dc->di.len == 1) {
1483 __remove_discard_cmd(sbi, dc);
1484 return;
1485 }
1486
1487 dcc->undiscard_blks -= di.len;
1488
1489 if (blkaddr > di.lstart) {
1490 dc->di.len = blkaddr - dc->di.lstart;
1491 dcc->undiscard_blks += dc->di.len;
1492 __relocate_discard_cmd(dcc, dc);
1493 modified = true;
1494 }
1495
1496 if (blkaddr < di.lstart + di.len - 1) {
1497 if (modified) {
1498 __insert_discard_cmd(sbi, dc->bdev, blkaddr + 1,
1499 di.start + blkaddr + 1 - di.lstart,
1500 di.lstart + di.len - 1 - blkaddr);
1501 } else {
1502 dc->di.lstart++;
1503 dc->di.len--;
1504 dc->di.start++;
1505 dcc->undiscard_blks += dc->di.len;
1506 __relocate_discard_cmd(dcc, dc);
1507 }
1508 }
1509}
1510
1511static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1512 struct block_device *bdev, block_t lstart,
1513 block_t start, block_t len)
1514{
1515 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1516 struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1517 struct discard_cmd *dc;
1518 struct discard_info di = {0};
1519 struct rb_node **insert_p = NULL, *insert_parent = NULL;
1520 unsigned int max_discard_blocks =
1521 SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev));
1522 block_t end = lstart + len;
1523
1524 dc = __lookup_discard_cmd_ret(&dcc->root, lstart,
1525 &prev_dc, &next_dc, &insert_p, &insert_parent);
1526 if (dc)
1527 prev_dc = dc;
1528
1529 if (!prev_dc) {
1530 di.lstart = lstart;
1531 di.len = next_dc ? next_dc->di.lstart - lstart : len;
1532 di.len = min(di.len, len);
1533 di.start = start;
1534 }
1535
1536 while (1) {
1537 struct rb_node *node;
1538 bool merged = false;
1539 struct discard_cmd *tdc = NULL;
1540
1541 if (prev_dc) {
1542 di.lstart = prev_dc->di.lstart + prev_dc->di.len;
1543 if (di.lstart < lstart)
1544 di.lstart = lstart;
1545 if (di.lstart >= end)
1546 break;
1547
1548 if (!next_dc || next_dc->di.lstart > end)
1549 di.len = end - di.lstart;
1550 else
1551 di.len = next_dc->di.lstart - di.lstart;
1552 di.start = start + di.lstart - lstart;
1553 }
1554
1555 if (!di.len)
1556 goto next;
1557
1558 if (prev_dc && prev_dc->state == D_PREP &&
1559 prev_dc->bdev == bdev &&
1560 __is_discard_back_mergeable(&di, &prev_dc->di,
1561 max_discard_blocks)) {
1562 prev_dc->di.len += di.len;
1563 dcc->undiscard_blks += di.len;
1564 __relocate_discard_cmd(dcc, prev_dc);
1565 di = prev_dc->di;
1566 tdc = prev_dc;
1567 merged = true;
1568 }
1569
1570 if (next_dc && next_dc->state == D_PREP &&
1571 next_dc->bdev == bdev &&
1572 __is_discard_front_mergeable(&di, &next_dc->di,
1573 max_discard_blocks)) {
1574 next_dc->di.lstart = di.lstart;
1575 next_dc->di.len += di.len;
1576 next_dc->di.start = di.start;
1577 dcc->undiscard_blks += di.len;
1578 __relocate_discard_cmd(dcc, next_dc);
1579 if (tdc)
1580 __remove_discard_cmd(sbi, tdc);
1581 merged = true;
1582 }
1583
1584 if (!merged)
1585 __insert_discard_cmd(sbi, bdev,
1586 di.lstart, di.start, di.len);
1587 next:
1588 prev_dc = next_dc;
1589 if (!prev_dc)
1590 break;
1591
1592 node = rb_next(&prev_dc->rb_node);
1593 next_dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1594 }
1595}
1596
1597#ifdef CONFIG_BLK_DEV_ZONED
1598static void __queue_zone_reset_cmd(struct f2fs_sb_info *sbi,
1599 struct block_device *bdev, block_t blkstart, block_t lblkstart,
1600 block_t blklen)
1601{
1602 trace_f2fs_queue_reset_zone(bdev, blkstart);
1603
1604 mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock);
1605 __insert_discard_cmd(sbi, bdev, lblkstart, blkstart, blklen);
1606 mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock);
1607}
1608#endif
1609
1610static void __queue_discard_cmd(struct f2fs_sb_info *sbi,
1611 struct block_device *bdev, block_t blkstart, block_t blklen)
1612{
1613 block_t lblkstart = blkstart;
1614
1615 if (!f2fs_bdev_support_discard(bdev))
1616 return;
1617
1618 trace_f2fs_queue_discard(bdev, blkstart, blklen);
1619
1620 if (f2fs_is_multi_device(sbi)) {
1621 int devi = f2fs_target_device_index(sbi, blkstart);
1622
1623 blkstart -= FDEV(devi).start_blk;
1624 }
1625 mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock);
1626 __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen);
1627 mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock);
1628}
1629
1630static void __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi,
1631 struct discard_policy *dpolicy, int *issued)
1632{
1633 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1634 struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1635 struct rb_node **insert_p = NULL, *insert_parent = NULL;
1636 struct discard_cmd *dc;
1637 struct blk_plug plug;
1638 bool io_interrupted = false;
1639
1640 mutex_lock(&dcc->cmd_lock);
1641 dc = __lookup_discard_cmd_ret(&dcc->root, dcc->next_pos,
1642 &prev_dc, &next_dc, &insert_p, &insert_parent);
1643 if (!dc)
1644 dc = next_dc;
1645
1646 blk_start_plug(&plug);
1647
1648 while (dc) {
1649 struct rb_node *node;
1650 int err = 0;
1651
1652 if (dc->state != D_PREP)
1653 goto next;
1654
1655 if (*issued > 0 && unlikely(freezing(current)))
1656 break;
1657
1658 if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) {
1659 io_interrupted = true;
1660 break;
1661 }
1662
1663 dcc->next_pos = dc->di.lstart + dc->di.len;
1664 err = __submit_discard_cmd(sbi, dpolicy, dc, issued);
1665
1666 if (*issued >= dpolicy->max_requests)
1667 break;
1668next:
1669 node = rb_next(&dc->rb_node);
1670 if (err)
1671 __remove_discard_cmd(sbi, dc);
1672 dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1673 }
1674
1675 blk_finish_plug(&plug);
1676
1677 if (!dc)
1678 dcc->next_pos = 0;
1679
1680 mutex_unlock(&dcc->cmd_lock);
1681
1682 if (!(*issued) && io_interrupted)
1683 *issued = -1;
1684}
1685static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1686 struct discard_policy *dpolicy);
1687
1688static int __issue_discard_cmd(struct f2fs_sb_info *sbi,
1689 struct discard_policy *dpolicy)
1690{
1691 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1692 struct list_head *pend_list;
1693 struct discard_cmd *dc, *tmp;
1694 struct blk_plug plug;
1695 int i, issued;
1696 bool io_interrupted = false;
1697 bool suspended = false;
1698
1699 if (dpolicy->timeout)
1700 f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT);
1701
1702retry:
1703 issued = 0;
1704 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1705 if (dpolicy->timeout &&
1706 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
1707 break;
1708
1709 if (i + 1 < dpolicy->granularity)
1710 break;
1711
1712 if (i + 1 < dcc->max_ordered_discard && dpolicy->ordered) {
1713 __issue_discard_cmd_orderly(sbi, dpolicy, &issued);
1714 return issued;
1715 }
1716
1717 pend_list = &dcc->pend_list[i];
1718
1719 mutex_lock(&dcc->cmd_lock);
1720 if (list_empty(pend_list))
1721 goto next;
1722 if (unlikely(dcc->rbtree_check))
1723 f2fs_bug_on(sbi, !f2fs_check_discard_tree(sbi));
1724 blk_start_plug(&plug);
1725 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1726 f2fs_bug_on(sbi, dc->state != D_PREP);
1727
1728 if (issued > 0 && unlikely(freezing(current))) {
1729 suspended = true;
1730 break;
1731 }
1732
1733 if (dpolicy->timeout &&
1734 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
1735 break;
1736
1737 if (dpolicy->io_aware && i < dpolicy->io_aware_gran &&
1738 !is_idle(sbi, DISCARD_TIME)) {
1739 io_interrupted = true;
1740 break;
1741 }
1742
1743 __submit_discard_cmd(sbi, dpolicy, dc, &issued);
1744
1745 if (issued >= dpolicy->max_requests)
1746 break;
1747 }
1748 blk_finish_plug(&plug);
1749next:
1750 mutex_unlock(&dcc->cmd_lock);
1751
1752 if (issued >= dpolicy->max_requests || io_interrupted ||
1753 suspended)
1754 break;
1755 }
1756
1757 if (dpolicy->type == DPOLICY_UMOUNT && issued) {
1758 __wait_all_discard_cmd(sbi, dpolicy);
1759 goto retry;
1760 }
1761
1762 if (!issued && io_interrupted)
1763 issued = -1;
1764
1765 return issued;
1766}
1767
1768static bool __drop_discard_cmd(struct f2fs_sb_info *sbi)
1769{
1770 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1771 struct list_head *pend_list;
1772 struct discard_cmd *dc, *tmp;
1773 int i;
1774 bool dropped = false;
1775
1776 mutex_lock(&dcc->cmd_lock);
1777 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1778 pend_list = &dcc->pend_list[i];
1779 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1780 f2fs_bug_on(sbi, dc->state != D_PREP);
1781 __remove_discard_cmd(sbi, dc);
1782 dropped = true;
1783 }
1784 }
1785 mutex_unlock(&dcc->cmd_lock);
1786
1787 return dropped;
1788}
1789
1790void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi)
1791{
1792 __drop_discard_cmd(sbi);
1793}
1794
1795static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi,
1796 struct discard_cmd *dc)
1797{
1798 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1799 unsigned int len = 0;
1800
1801 wait_for_completion_io(&dc->wait);
1802 mutex_lock(&dcc->cmd_lock);
1803 f2fs_bug_on(sbi, dc->state != D_DONE);
1804 dc->ref--;
1805 if (!dc->ref) {
1806 if (!dc->error)
1807 len = dc->di.len;
1808 __remove_discard_cmd(sbi, dc);
1809 }
1810 mutex_unlock(&dcc->cmd_lock);
1811
1812 return len;
1813}
1814
1815static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi,
1816 struct discard_policy *dpolicy,
1817 block_t start, block_t end)
1818{
1819 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1820 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1821 &(dcc->fstrim_list) : &(dcc->wait_list);
1822 struct discard_cmd *dc = NULL, *iter, *tmp;
1823 unsigned int trimmed = 0;
1824
1825next:
1826 dc = NULL;
1827
1828 mutex_lock(&dcc->cmd_lock);
1829 list_for_each_entry_safe(iter, tmp, wait_list, list) {
1830 if (iter->di.lstart + iter->di.len <= start ||
1831 end <= iter->di.lstart)
1832 continue;
1833 if (iter->di.len < dpolicy->granularity)
1834 continue;
1835 if (iter->state == D_DONE && !iter->ref) {
1836 wait_for_completion_io(&iter->wait);
1837 if (!iter->error)
1838 trimmed += iter->di.len;
1839 __remove_discard_cmd(sbi, iter);
1840 } else {
1841 iter->ref++;
1842 dc = iter;
1843 break;
1844 }
1845 }
1846 mutex_unlock(&dcc->cmd_lock);
1847
1848 if (dc) {
1849 trimmed += __wait_one_discard_bio(sbi, dc);
1850 goto next;
1851 }
1852
1853 return trimmed;
1854}
1855
1856static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1857 struct discard_policy *dpolicy)
1858{
1859 struct discard_policy dp;
1860 unsigned int discard_blks;
1861
1862 if (dpolicy)
1863 return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX);
1864
1865 /* wait all */
1866 __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, MIN_DISCARD_GRANULARITY);
1867 discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1868 __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, MIN_DISCARD_GRANULARITY);
1869 discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1870
1871 return discard_blks;
1872}
1873
1874/* This should be covered by global mutex, &sit_i->sentry_lock */
1875static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)
1876{
1877 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1878 struct discard_cmd *dc;
1879 bool need_wait = false;
1880
1881 mutex_lock(&dcc->cmd_lock);
1882 dc = __lookup_discard_cmd(sbi, blkaddr);
1883#ifdef CONFIG_BLK_DEV_ZONED
1884 if (dc && f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(dc->bdev)) {
1885 int devi = f2fs_bdev_index(sbi, dc->bdev);
1886
1887 if (devi < 0) {
1888 mutex_unlock(&dcc->cmd_lock);
1889 return;
1890 }
1891
1892 if (f2fs_blkz_is_seq(sbi, devi, dc->di.start)) {
1893 /* force submit zone reset */
1894 if (dc->state == D_PREP)
1895 __submit_zone_reset_cmd(sbi, dc, REQ_SYNC,
1896 &dcc->wait_list, NULL);
1897 dc->ref++;
1898 mutex_unlock(&dcc->cmd_lock);
1899 /* wait zone reset */
1900 __wait_one_discard_bio(sbi, dc);
1901 return;
1902 }
1903 }
1904#endif
1905 if (dc) {
1906 if (dc->state == D_PREP) {
1907 __punch_discard_cmd(sbi, dc, blkaddr);
1908 } else {
1909 dc->ref++;
1910 need_wait = true;
1911 }
1912 }
1913 mutex_unlock(&dcc->cmd_lock);
1914
1915 if (need_wait)
1916 __wait_one_discard_bio(sbi, dc);
1917}
1918
1919void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi)
1920{
1921 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1922
1923 if (dcc && dcc->f2fs_issue_discard) {
1924 struct task_struct *discard_thread = dcc->f2fs_issue_discard;
1925
1926 dcc->f2fs_issue_discard = NULL;
1927 kthread_stop(discard_thread);
1928 }
1929}
1930
1931/**
1932 * f2fs_issue_discard_timeout() - Issue all discard cmd within UMOUNT_DISCARD_TIMEOUT
1933 * @sbi: the f2fs_sb_info data for discard cmd to issue
1934 *
1935 * When UMOUNT_DISCARD_TIMEOUT is exceeded, all remaining discard commands will be dropped
1936 *
1937 * Return true if issued all discard cmd or no discard cmd need issue, otherwise return false.
1938 */
1939bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi, bool need_check)
1940{
1941 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1942 struct discard_policy dpolicy;
1943 bool dropped;
1944
1945 if (!atomic_read(&dcc->discard_cmd_cnt))
1946 return true;
1947
1948 __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT,
1949 dcc->discard_granularity);
1950 __issue_discard_cmd(sbi, &dpolicy);
1951 dropped = __drop_discard_cmd(sbi);
1952
1953 /* just to make sure there is no pending discard commands */
1954 __wait_all_discard_cmd(sbi, NULL);
1955
1956 f2fs_bug_on(sbi, need_check && atomic_read(&dcc->discard_cmd_cnt));
1957 return !dropped;
1958}
1959
1960static int issue_discard_thread(void *data)
1961{
1962 struct f2fs_sb_info *sbi = data;
1963 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1964 wait_queue_head_t *q = &dcc->discard_wait_queue;
1965 struct discard_policy dpolicy;
1966 unsigned int wait_ms = dcc->min_discard_issue_time;
1967 int issued;
1968
1969 set_freezable();
1970
1971 do {
1972 wait_event_freezable_timeout(*q,
1973 kthread_should_stop() || dcc->discard_wake,
1974 msecs_to_jiffies(wait_ms));
1975
1976 if (sbi->gc_mode == GC_URGENT_HIGH ||
1977 !f2fs_available_free_memory(sbi, DISCARD_CACHE))
1978 __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE,
1979 MIN_DISCARD_GRANULARITY);
1980 else
1981 __init_discard_policy(sbi, &dpolicy, DPOLICY_BG,
1982 dcc->discard_granularity);
1983
1984 if (dcc->discard_wake)
1985 dcc->discard_wake = false;
1986
1987 /* clean up pending candidates before going to sleep */
1988 if (atomic_read(&dcc->queued_discard))
1989 __wait_all_discard_cmd(sbi, NULL);
1990
1991 if (f2fs_readonly(sbi->sb))
1992 continue;
1993 if (kthread_should_stop())
1994 return 0;
1995 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK) ||
1996 !atomic_read(&dcc->discard_cmd_cnt)) {
1997 wait_ms = dpolicy.max_interval;
1998 continue;
1999 }
2000
2001 sb_start_intwrite(sbi->sb);
2002
2003 issued = __issue_discard_cmd(sbi, &dpolicy);
2004 if (issued > 0) {
2005 __wait_all_discard_cmd(sbi, &dpolicy);
2006 wait_ms = dpolicy.min_interval;
2007 } else if (issued == -1) {
2008 wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME);
2009 if (!wait_ms)
2010 wait_ms = dpolicy.mid_interval;
2011 } else {
2012 wait_ms = dpolicy.max_interval;
2013 }
2014 if (!atomic_read(&dcc->discard_cmd_cnt))
2015 wait_ms = dpolicy.max_interval;
2016
2017 sb_end_intwrite(sbi->sb);
2018
2019 } while (!kthread_should_stop());
2020 return 0;
2021}
2022
2023#ifdef CONFIG_BLK_DEV_ZONED
2024static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,
2025 struct block_device *bdev, block_t blkstart, block_t blklen)
2026{
2027 sector_t sector, nr_sects;
2028 block_t lblkstart = blkstart;
2029 int devi = 0;
2030 u64 remainder = 0;
2031
2032 if (f2fs_is_multi_device(sbi)) {
2033 devi = f2fs_target_device_index(sbi, blkstart);
2034 if (blkstart < FDEV(devi).start_blk ||
2035 blkstart > FDEV(devi).end_blk) {
2036 f2fs_err(sbi, "Invalid block %x", blkstart);
2037 return -EIO;
2038 }
2039 blkstart -= FDEV(devi).start_blk;
2040 }
2041
2042 /* For sequential zones, reset the zone write pointer */
2043 if (f2fs_blkz_is_seq(sbi, devi, blkstart)) {
2044 sector = SECTOR_FROM_BLOCK(blkstart);
2045 nr_sects = SECTOR_FROM_BLOCK(blklen);
2046 div64_u64_rem(sector, bdev_zone_sectors(bdev), &remainder);
2047
2048 if (remainder || nr_sects != bdev_zone_sectors(bdev)) {
2049 f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)",
2050 devi, sbi->s_ndevs ? FDEV(devi).path : "",
2051 blkstart, blklen);
2052 return -EIO;
2053 }
2054
2055 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) {
2056 unsigned int nofs_flags;
2057 int ret;
2058
2059 trace_f2fs_issue_reset_zone(bdev, blkstart);
2060 nofs_flags = memalloc_nofs_save();
2061 ret = blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET,
2062 sector, nr_sects);
2063 memalloc_nofs_restore(nofs_flags);
2064 return ret;
2065 }
2066
2067 __queue_zone_reset_cmd(sbi, bdev, blkstart, lblkstart, blklen);
2068 return 0;
2069 }
2070
2071 /* For conventional zones, use regular discard if supported */
2072 __queue_discard_cmd(sbi, bdev, lblkstart, blklen);
2073 return 0;
2074}
2075#endif
2076
2077static int __issue_discard_async(struct f2fs_sb_info *sbi,
2078 struct block_device *bdev, block_t blkstart, block_t blklen)
2079{
2080#ifdef CONFIG_BLK_DEV_ZONED
2081 if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev))
2082 return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen);
2083#endif
2084 __queue_discard_cmd(sbi, bdev, blkstart, blklen);
2085 return 0;
2086}
2087
2088static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
2089 block_t blkstart, block_t blklen)
2090{
2091 sector_t start = blkstart, len = 0;
2092 struct block_device *bdev;
2093 struct seg_entry *se;
2094 unsigned int offset;
2095 block_t i;
2096 int err = 0;
2097
2098 bdev = f2fs_target_device(sbi, blkstart, NULL);
2099
2100 for (i = blkstart; i < blkstart + blklen; i++, len++) {
2101 if (i != start) {
2102 struct block_device *bdev2 =
2103 f2fs_target_device(sbi, i, NULL);
2104
2105 if (bdev2 != bdev) {
2106 err = __issue_discard_async(sbi, bdev,
2107 start, len);
2108 if (err)
2109 return err;
2110 bdev = bdev2;
2111 start = i;
2112 len = 0;
2113 }
2114 }
2115
2116 se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
2117 offset = GET_BLKOFF_FROM_SEG0(sbi, i);
2118
2119 if (f2fs_block_unit_discard(sbi) &&
2120 !f2fs_test_and_set_bit(offset, se->discard_map))
2121 sbi->discard_blks--;
2122 }
2123
2124 if (len)
2125 err = __issue_discard_async(sbi, bdev, start, len);
2126 return err;
2127}
2128
2129static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc,
2130 bool check_only)
2131{
2132 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
2133 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
2134 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
2135 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
2136 unsigned long *discard_map = (unsigned long *)se->discard_map;
2137 unsigned long *dmap = SIT_I(sbi)->tmp_map;
2138 unsigned int start = 0, end = -1;
2139 bool force = (cpc->reason & CP_DISCARD);
2140 struct discard_entry *de = NULL;
2141 struct list_head *head = &SM_I(sbi)->dcc_info->entry_list;
2142 int i;
2143
2144 if (se->valid_blocks == BLKS_PER_SEG(sbi) ||
2145 !f2fs_hw_support_discard(sbi) ||
2146 !f2fs_block_unit_discard(sbi))
2147 return false;
2148
2149 if (!force) {
2150 if (!f2fs_realtime_discard_enable(sbi) ||
2151 (!se->valid_blocks &&
2152 !is_curseg(sbi, cpc->trim_start)) ||
2153 SM_I(sbi)->dcc_info->nr_discards >=
2154 SM_I(sbi)->dcc_info->max_discards)
2155 return false;
2156 }
2157
2158 /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
2159 for (i = 0; i < entries; i++)
2160 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] :
2161 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
2162
2163 while (force || SM_I(sbi)->dcc_info->nr_discards <=
2164 SM_I(sbi)->dcc_info->max_discards) {
2165 start = __find_rev_next_bit(dmap, BLKS_PER_SEG(sbi), end + 1);
2166 if (start >= BLKS_PER_SEG(sbi))
2167 break;
2168
2169 end = __find_rev_next_zero_bit(dmap,
2170 BLKS_PER_SEG(sbi), start + 1);
2171 if (force && start && end != BLKS_PER_SEG(sbi) &&
2172 (end - start) < cpc->trim_minlen)
2173 continue;
2174
2175 if (check_only)
2176 return true;
2177
2178 if (!de) {
2179 de = f2fs_kmem_cache_alloc(discard_entry_slab,
2180 GFP_F2FS_ZERO, true, NULL);
2181 de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start);
2182 list_add_tail(&de->list, head);
2183 }
2184
2185 for (i = start; i < end; i++)
2186 __set_bit_le(i, (void *)de->discard_map);
2187
2188 SM_I(sbi)->dcc_info->nr_discards += end - start;
2189 }
2190 return false;
2191}
2192
2193static void release_discard_addr(struct discard_entry *entry)
2194{
2195 list_del(&entry->list);
2196 kmem_cache_free(discard_entry_slab, entry);
2197}
2198
2199void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi)
2200{
2201 struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list);
2202 struct discard_entry *entry, *this;
2203
2204 /* drop caches */
2205 list_for_each_entry_safe(entry, this, head, list)
2206 release_discard_addr(entry);
2207}
2208
2209/*
2210 * Should call f2fs_clear_prefree_segments after checkpoint is done.
2211 */
2212static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
2213{
2214 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2215 unsigned int segno;
2216
2217 mutex_lock(&dirty_i->seglist_lock);
2218 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
2219 __set_test_and_free(sbi, segno, false);
2220 mutex_unlock(&dirty_i->seglist_lock);
2221}
2222
2223void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
2224 struct cp_control *cpc)
2225{
2226 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2227 struct list_head *head = &dcc->entry_list;
2228 struct discard_entry *entry, *this;
2229 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2230 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
2231 unsigned int start = 0, end = -1;
2232 unsigned int secno, start_segno;
2233 bool force = (cpc->reason & CP_DISCARD);
2234 bool section_alignment = F2FS_OPTION(sbi).discard_unit ==
2235 DISCARD_UNIT_SECTION;
2236
2237 if (f2fs_lfs_mode(sbi) && __is_large_section(sbi))
2238 section_alignment = true;
2239
2240 mutex_lock(&dirty_i->seglist_lock);
2241
2242 while (1) {
2243 int i;
2244
2245 if (section_alignment && end != -1)
2246 end--;
2247 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
2248 if (start >= MAIN_SEGS(sbi))
2249 break;
2250 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
2251 start + 1);
2252
2253 if (section_alignment) {
2254 start = rounddown(start, SEGS_PER_SEC(sbi));
2255 end = roundup(end, SEGS_PER_SEC(sbi));
2256 }
2257
2258 for (i = start; i < end; i++) {
2259 if (test_and_clear_bit(i, prefree_map))
2260 dirty_i->nr_dirty[PRE]--;
2261 }
2262
2263 if (!f2fs_realtime_discard_enable(sbi))
2264 continue;
2265
2266 if (force && start >= cpc->trim_start &&
2267 (end - 1) <= cpc->trim_end)
2268 continue;
2269
2270 /* Should cover 2MB zoned device for zone-based reset */
2271 if (!f2fs_sb_has_blkzoned(sbi) &&
2272 (!f2fs_lfs_mode(sbi) || !__is_large_section(sbi))) {
2273 f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
2274 SEGS_TO_BLKS(sbi, end - start));
2275 continue;
2276 }
2277next:
2278 secno = GET_SEC_FROM_SEG(sbi, start);
2279 start_segno = GET_SEG_FROM_SEC(sbi, secno);
2280 if (!is_cursec(sbi, secno) &&
2281 !get_valid_blocks(sbi, start, true))
2282 f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno),
2283 BLKS_PER_SEC(sbi));
2284
2285 start = start_segno + SEGS_PER_SEC(sbi);
2286 if (start < end)
2287 goto next;
2288 else
2289 end = start - 1;
2290 }
2291 mutex_unlock(&dirty_i->seglist_lock);
2292
2293 if (!f2fs_block_unit_discard(sbi))
2294 goto wakeup;
2295
2296 /* send small discards */
2297 list_for_each_entry_safe(entry, this, head, list) {
2298 unsigned int cur_pos = 0, next_pos, len, total_len = 0;
2299 bool is_valid = test_bit_le(0, entry->discard_map);
2300
2301find_next:
2302 if (is_valid) {
2303 next_pos = find_next_zero_bit_le(entry->discard_map,
2304 BLKS_PER_SEG(sbi), cur_pos);
2305 len = next_pos - cur_pos;
2306
2307 if (f2fs_sb_has_blkzoned(sbi) ||
2308 (force && len < cpc->trim_minlen))
2309 goto skip;
2310
2311 f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos,
2312 len);
2313 total_len += len;
2314 } else {
2315 next_pos = find_next_bit_le(entry->discard_map,
2316 BLKS_PER_SEG(sbi), cur_pos);
2317 }
2318skip:
2319 cur_pos = next_pos;
2320 is_valid = !is_valid;
2321
2322 if (cur_pos < BLKS_PER_SEG(sbi))
2323 goto find_next;
2324
2325 release_discard_addr(entry);
2326 dcc->nr_discards -= total_len;
2327 }
2328
2329wakeup:
2330 wake_up_discard_thread(sbi, false);
2331}
2332
2333int f2fs_start_discard_thread(struct f2fs_sb_info *sbi)
2334{
2335 dev_t dev = sbi->sb->s_bdev->bd_dev;
2336 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2337 int err = 0;
2338
2339 if (f2fs_sb_has_readonly(sbi)) {
2340 f2fs_info(sbi,
2341 "Skip to start discard thread for readonly image");
2342 return 0;
2343 }
2344
2345 if (!f2fs_realtime_discard_enable(sbi))
2346 return 0;
2347
2348 dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi,
2349 "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev));
2350 if (IS_ERR(dcc->f2fs_issue_discard)) {
2351 err = PTR_ERR(dcc->f2fs_issue_discard);
2352 dcc->f2fs_issue_discard = NULL;
2353 }
2354
2355 return err;
2356}
2357
2358static int create_discard_cmd_control(struct f2fs_sb_info *sbi)
2359{
2360 struct discard_cmd_control *dcc;
2361 int err = 0, i;
2362
2363 if (SM_I(sbi)->dcc_info) {
2364 dcc = SM_I(sbi)->dcc_info;
2365 goto init_thread;
2366 }
2367
2368 dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL);
2369 if (!dcc)
2370 return -ENOMEM;
2371
2372 dcc->discard_io_aware_gran = MAX_PLIST_NUM;
2373 dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY;
2374 dcc->max_ordered_discard = DEFAULT_MAX_ORDERED_DISCARD_GRANULARITY;
2375 dcc->discard_io_aware = DPOLICY_IO_AWARE_ENABLE;
2376 if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT ||
2377 F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SECTION)
2378 dcc->discard_granularity = BLKS_PER_SEG(sbi);
2379
2380 INIT_LIST_HEAD(&dcc->entry_list);
2381 for (i = 0; i < MAX_PLIST_NUM; i++)
2382 INIT_LIST_HEAD(&dcc->pend_list[i]);
2383 INIT_LIST_HEAD(&dcc->wait_list);
2384 INIT_LIST_HEAD(&dcc->fstrim_list);
2385 mutex_init(&dcc->cmd_lock);
2386 atomic_set(&dcc->issued_discard, 0);
2387 atomic_set(&dcc->queued_discard, 0);
2388 atomic_set(&dcc->discard_cmd_cnt, 0);
2389 dcc->nr_discards = 0;
2390 dcc->max_discards = SEGS_TO_BLKS(sbi, MAIN_SEGS(sbi));
2391 dcc->max_discard_request = DEF_MAX_DISCARD_REQUEST;
2392 dcc->min_discard_issue_time = DEF_MIN_DISCARD_ISSUE_TIME;
2393 dcc->mid_discard_issue_time = DEF_MID_DISCARD_ISSUE_TIME;
2394 dcc->max_discard_issue_time = DEF_MAX_DISCARD_ISSUE_TIME;
2395 dcc->discard_urgent_util = DEF_DISCARD_URGENT_UTIL;
2396 dcc->undiscard_blks = 0;
2397 dcc->next_pos = 0;
2398 dcc->root = RB_ROOT_CACHED;
2399 dcc->rbtree_check = false;
2400
2401 init_waitqueue_head(&dcc->discard_wait_queue);
2402 SM_I(sbi)->dcc_info = dcc;
2403init_thread:
2404 err = f2fs_start_discard_thread(sbi);
2405 if (err) {
2406 kfree(dcc);
2407 SM_I(sbi)->dcc_info = NULL;
2408 }
2409
2410 return err;
2411}
2412
2413static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi)
2414{
2415 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2416
2417 if (!dcc)
2418 return;
2419
2420 f2fs_stop_discard_thread(sbi);
2421
2422 /*
2423 * Recovery can cache discard commands, so in error path of
2424 * fill_super(), it needs to give a chance to handle them.
2425 */
2426 f2fs_issue_discard_timeout(sbi, true);
2427
2428 kfree(dcc);
2429 SM_I(sbi)->dcc_info = NULL;
2430}
2431
2432static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
2433{
2434 struct sit_info *sit_i = SIT_I(sbi);
2435
2436 if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) {
2437 sit_i->dirty_sentries++;
2438 return false;
2439 }
2440
2441 return true;
2442}
2443
2444static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
2445 unsigned int segno, int modified)
2446{
2447 struct seg_entry *se = get_seg_entry(sbi, segno);
2448
2449 se->type = type;
2450 if (modified)
2451 __mark_sit_entry_dirty(sbi, segno);
2452}
2453
2454static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi,
2455 block_t blkaddr)
2456{
2457 unsigned int segno = GET_SEGNO(sbi, blkaddr);
2458
2459 if (segno == NULL_SEGNO)
2460 return 0;
2461 return get_seg_entry(sbi, segno)->mtime;
2462}
2463
2464static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr,
2465 unsigned long long old_mtime)
2466{
2467 struct seg_entry *se;
2468 unsigned int segno = GET_SEGNO(sbi, blkaddr);
2469 unsigned long long ctime = get_mtime(sbi, false);
2470 unsigned long long mtime = old_mtime ? old_mtime : ctime;
2471
2472 if (segno == NULL_SEGNO)
2473 return;
2474
2475 se = get_seg_entry(sbi, segno);
2476
2477 if (!se->mtime)
2478 se->mtime = mtime;
2479 else
2480 se->mtime = div_u64(se->mtime * se->valid_blocks + mtime,
2481 se->valid_blocks + 1);
2482
2483 if (ctime > SIT_I(sbi)->max_mtime)
2484 SIT_I(sbi)->max_mtime = ctime;
2485}
2486
2487/*
2488 * NOTE: when updating multiple blocks at the same time, please ensure
2489 * that the consecutive input blocks belong to the same segment.
2490 */
2491static int update_sit_entry_for_release(struct f2fs_sb_info *sbi, struct seg_entry *se,
2492 unsigned int segno, block_t blkaddr, unsigned int offset, int del)
2493{
2494 bool exist;
2495 int i;
2496 int del_count = -del;
2497
2498 f2fs_bug_on(sbi, GET_SEGNO(sbi, blkaddr) != GET_SEGNO(sbi, blkaddr + del_count - 1));
2499
2500 for (i = 0; i < del_count; i++) {
2501 exist = f2fs_test_and_clear_bit(offset + i, se->cur_valid_map);
2502 if (unlikely(!exist)) {
2503 f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u", blkaddr + i);
2504 f2fs_bug_on(sbi, 1);
2505 se->valid_blocks++;
2506 del += 1;
2507 } else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2508 /*
2509 * If checkpoints are off, we must not reuse data that
2510 * was used in the previous checkpoint. If it was used
2511 * before, we must track that to know how much space we
2512 * really have.
2513 */
2514 if (f2fs_test_bit(offset + i, se->ckpt_valid_map)) {
2515 spin_lock(&sbi->stat_lock);
2516 sbi->unusable_block_count++;
2517 spin_unlock(&sbi->stat_lock);
2518 }
2519 }
2520
2521 if (f2fs_block_unit_discard(sbi) &&
2522 f2fs_test_and_clear_bit(offset + i, se->discard_map))
2523 sbi->discard_blks++;
2524
2525 if (!f2fs_test_bit(offset + i, se->ckpt_valid_map)) {
2526 se->ckpt_valid_blocks -= 1;
2527 if (__is_large_section(sbi))
2528 get_sec_entry(sbi, segno)->ckpt_valid_blocks -= 1;
2529 }
2530 }
2531
2532 if (__is_large_section(sbi))
2533 sanity_check_valid_blocks(sbi, segno);
2534
2535 return del;
2536}
2537
2538static int update_sit_entry_for_alloc(struct f2fs_sb_info *sbi, struct seg_entry *se,
2539 unsigned int segno, block_t blkaddr, unsigned int offset, int del)
2540{
2541 bool exist;
2542 int del_count = del;
2543 int i;
2544
2545 f2fs_bug_on(sbi, GET_SEGNO(sbi, blkaddr) != GET_SEGNO(sbi, blkaddr + del_count - 1));
2546
2547 for (i = 0; i < del_count; i++) {
2548 exist = f2fs_test_and_set_bit(offset + i, se->cur_valid_map);
2549 if (unlikely(exist)) {
2550 f2fs_err(sbi, "Bitmap was wrongly set, blk:%u", blkaddr + i);
2551 f2fs_bug_on(sbi, 1);
2552 se->valid_blocks--;
2553 del -= 1;
2554 continue;
2555 }
2556
2557 if (f2fs_block_unit_discard(sbi) &&
2558 !f2fs_test_and_set_bit(offset + i, se->discard_map))
2559 sbi->discard_blks--;
2560
2561 /*
2562 * SSR should never reuse block which is checkpointed
2563 * or newly invalidated.
2564 */
2565 if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {
2566 if (!f2fs_test_and_set_bit(offset + i, se->ckpt_valid_map)) {
2567 se->ckpt_valid_blocks++;
2568 if (__is_large_section(sbi))
2569 get_sec_entry(sbi, segno)->ckpt_valid_blocks++;
2570 }
2571 }
2572
2573 if (!f2fs_test_bit(offset + i, se->ckpt_valid_map)) {
2574 se->ckpt_valid_blocks += 1;
2575 if (__is_large_section(sbi))
2576 get_sec_entry(sbi, segno)->ckpt_valid_blocks += 1;
2577 }
2578 }
2579
2580 if (__is_large_section(sbi))
2581 sanity_check_valid_blocks(sbi, segno);
2582
2583 return del;
2584}
2585
2586/*
2587 * If releasing blocks, this function supports updating multiple consecutive blocks
2588 * at one time, but please note that these consecutive blocks need to belong to the
2589 * same segment.
2590 */
2591static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
2592{
2593 struct seg_entry *se;
2594 unsigned int segno, offset;
2595 long int new_vblocks;
2596
2597 segno = GET_SEGNO(sbi, blkaddr);
2598 if (segno == NULL_SEGNO)
2599 return;
2600
2601 se = get_seg_entry(sbi, segno);
2602 new_vblocks = se->valid_blocks + del;
2603 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2604
2605 f2fs_bug_on(sbi, (new_vblocks < 0 ||
2606 (new_vblocks > f2fs_usable_blks_in_seg(sbi, segno))));
2607
2608 se->valid_blocks = new_vblocks;
2609
2610 /* Update valid block bitmap */
2611 if (del > 0) {
2612 del = update_sit_entry_for_alloc(sbi, se, segno, blkaddr, offset, del);
2613 } else {
2614 del = update_sit_entry_for_release(sbi, se, segno, blkaddr, offset, del);
2615 }
2616
2617 __mark_sit_entry_dirty(sbi, segno);
2618
2619 /* update total number of valid blocks to be written in ckpt area */
2620 SIT_I(sbi)->written_valid_blocks += del;
2621
2622 if (__is_large_section(sbi))
2623 get_sec_entry(sbi, segno)->valid_blocks += del;
2624}
2625
2626void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr,
2627 unsigned int len)
2628{
2629 unsigned int segno = GET_SEGNO(sbi, addr);
2630 struct sit_info *sit_i = SIT_I(sbi);
2631 block_t addr_start = addr, addr_end = addr + len - 1;
2632 unsigned int seg_num;
2633 unsigned int i = 1, max_blocks = sbi->blocks_per_seg, cnt;
2634
2635 if (len == 0)
2636 return;
2637
2638 seg_num = GET_SEGNO(sbi, addr_end) - segno + 1;
2639
2640 f2fs_bug_on(sbi, addr == NULL_ADDR);
2641 if (addr == NEW_ADDR || addr == COMPRESS_ADDR)
2642 return;
2643
2644 f2fs_invalidate_internal_cache(sbi, addr, len);
2645
2646 /* add it into sit main buffer */
2647 down_write(&sit_i->sentry_lock);
2648
2649 if (seg_num == 1)
2650 cnt = len;
2651 else
2652 cnt = max_blocks - GET_BLKOFF_FROM_SEG0(sbi, addr);
2653
2654 do {
2655 update_segment_mtime(sbi, addr_start, 0);
2656 update_sit_entry(sbi, addr_start, -cnt);
2657
2658 /* add it into dirty seglist */
2659 locate_dirty_segment(sbi, segno);
2660
2661 /* update @addr_start and @cnt and @segno */
2662 addr_start = START_BLOCK(sbi, ++segno);
2663 if (++i == seg_num)
2664 cnt = GET_BLKOFF_FROM_SEG0(sbi, addr_end) + 1;
2665 else
2666 cnt = max_blocks;
2667 } while (i <= seg_num);
2668
2669 up_write(&sit_i->sentry_lock);
2670}
2671
2672void f2fs_reserve_device_alias(struct f2fs_sb_info *sbi, block_t addr,
2673 unsigned int len)
2674{
2675 unsigned int segno = GET_SEGNO(sbi, addr);
2676 struct sit_info *sit_i = SIT_I(sbi);
2677 block_t addr_start = addr, addr_end = addr + len - 1;
2678 unsigned int seg_num;
2679 unsigned int i = 1, max_blocks = sbi->blocks_per_seg, cnt;
2680
2681 if (len == 0)
2682 return;
2683
2684 seg_num = GET_SEGNO(sbi, addr_end) - segno + 1;
2685
2686 down_write(&sit_i->sentry_lock);
2687
2688 if (seg_num == 1)
2689 cnt = len;
2690 else
2691 cnt = max_blocks - GET_BLKOFF_FROM_SEG0(sbi, addr);
2692
2693 do {
2694 update_segment_mtime(sbi, addr_start, 0);
2695 update_sit_entry(sbi, addr_start, cnt);
2696 __set_test_and_inuse(sbi, segno);
2697
2698 /* Remove the segment from PRE (prefree) to prevent checkpoint from freeing it! */
2699 mutex_lock(&DIRTY_I(sbi)->seglist_lock);
2700 if (test_and_clear_bit(segno, DIRTY_I(sbi)->dirty_segmap[PRE]))
2701 DIRTY_I(sbi)->nr_dirty[PRE]--;
2702 mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
2703
2704 /* add it into dirty seglist */
2705 locate_dirty_segment(sbi, segno);
2706
2707 /* update @addr_start and @cnt and @segno */
2708 addr_start = START_BLOCK(sbi, ++segno);
2709 if (++i == seg_num)
2710 cnt = GET_BLKOFF_FROM_SEG0(sbi, addr_end) + 1;
2711 else
2712 cnt = max_blocks;
2713 } while (i <= seg_num);
2714
2715 up_write(&sit_i->sentry_lock);
2716}
2717
2718bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
2719{
2720 struct sit_info *sit_i = SIT_I(sbi);
2721 unsigned int segno, offset;
2722 struct seg_entry *se;
2723 bool is_cp = false;
2724
2725 if (!__is_valid_data_blkaddr(blkaddr))
2726 return true;
2727
2728 down_read(&sit_i->sentry_lock);
2729
2730 segno = GET_SEGNO(sbi, blkaddr);
2731 se = get_seg_entry(sbi, segno);
2732 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2733
2734 if (f2fs_test_bit(offset, se->ckpt_valid_map))
2735 is_cp = true;
2736
2737 up_read(&sit_i->sentry_lock);
2738
2739 return is_cp;
2740}
2741
2742static unsigned short f2fs_curseg_valid_blocks(struct f2fs_sb_info *sbi, int type)
2743{
2744 struct curseg_info *curseg = CURSEG_I(sbi, type);
2745
2746 if (sbi->ckpt->alloc_type[type] == SSR)
2747 return BLKS_PER_SEG(sbi);
2748 return curseg->next_blkoff;
2749}
2750
2751/*
2752 * Calculate the number of current summary pages for writing
2753 */
2754int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
2755{
2756 int valid_sum_count = 0;
2757 int i, sum_in_page;
2758
2759 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
2760 if (sbi->ckpt->alloc_type[i] != SSR && for_ra)
2761 valid_sum_count +=
2762 le16_to_cpu(F2FS_CKPT(sbi)->cur_data_blkoff[i]);
2763 else
2764 valid_sum_count += f2fs_curseg_valid_blocks(sbi, i);
2765 }
2766
2767 sum_in_page = (sbi->blocksize - 2 * sbi->sum_journal_size -
2768 SUM_FOOTER_SIZE) / SUMMARY_SIZE;
2769 if (valid_sum_count <= sum_in_page)
2770 return 1;
2771 else if ((valid_sum_count - sum_in_page) <=
2772 (sbi->blocksize - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
2773 return 2;
2774 return 3;
2775}
2776
2777/*
2778 * Caller should put this summary folio
2779 */
2780struct folio *f2fs_get_sum_folio(struct f2fs_sb_info *sbi, unsigned int segno)
2781{
2782 if (unlikely(f2fs_cp_error(sbi)))
2783 return ERR_PTR(-EIO);
2784 return f2fs_get_meta_folio_retry(sbi, GET_SUM_BLOCK(sbi, segno));
2785}
2786
2787void f2fs_update_meta_page(struct f2fs_sb_info *sbi,
2788 void *src, block_t blk_addr)
2789{
2790 struct folio *folio;
2791
2792 if (!f2fs_sb_has_packed_ssa(sbi))
2793 folio = f2fs_grab_meta_folio(sbi, blk_addr);
2794 else
2795 folio = f2fs_get_meta_folio_retry(sbi, blk_addr);
2796
2797 if (IS_ERR(folio))
2798 return;
2799
2800 memcpy(folio_address(folio), src, PAGE_SIZE);
2801 folio_mark_dirty(folio);
2802 f2fs_folio_put(folio, true);
2803}
2804
2805static void write_sum_page(struct f2fs_sb_info *sbi,
2806 struct f2fs_summary_block *sum_blk, unsigned int segno)
2807{
2808 struct folio *folio;
2809
2810 if (!f2fs_sb_has_packed_ssa(sbi))
2811 return f2fs_update_meta_page(sbi, (void *)sum_blk,
2812 GET_SUM_BLOCK(sbi, segno));
2813
2814 folio = f2fs_get_sum_folio(sbi, segno);
2815 if (IS_ERR(folio))
2816 return;
2817
2818 memcpy(SUM_BLK_PAGE_ADDR(sbi, folio, segno), sum_blk,
2819 sbi->sum_blocksize);
2820 folio_mark_dirty(folio);
2821 f2fs_folio_put(folio, true);
2822}
2823
2824static void write_current_sum_page(struct f2fs_sb_info *sbi,
2825 int type, block_t blk_addr)
2826{
2827 struct curseg_info *curseg = CURSEG_I(sbi, type);
2828 struct folio *folio = f2fs_grab_meta_folio(sbi, blk_addr);
2829 struct f2fs_summary_block *src = curseg->sum_blk;
2830 struct f2fs_summary_block *dst;
2831
2832 dst = folio_address(folio);
2833 memset(dst, 0, PAGE_SIZE);
2834
2835 mutex_lock(&curseg->curseg_mutex);
2836
2837 down_read(&curseg->journal_rwsem);
2838 memcpy(sum_journal(sbi, dst), curseg->journal, sbi->sum_journal_size);
2839 up_read(&curseg->journal_rwsem);
2840
2841 memcpy(sum_entries(dst), sum_entries(src), sbi->sum_entry_size);
2842 memcpy(sum_footer(sbi, dst), sum_footer(sbi, src), SUM_FOOTER_SIZE);
2843
2844 mutex_unlock(&curseg->curseg_mutex);
2845
2846 folio_mark_dirty(folio);
2847 f2fs_folio_put(folio, true);
2848}
2849
2850static int is_next_segment_free(struct f2fs_sb_info *sbi,
2851 struct curseg_info *curseg)
2852{
2853 unsigned int segno = curseg->segno + 1;
2854 struct free_segmap_info *free_i = FREE_I(sbi);
2855
2856 if (segno < MAIN_SEGS(sbi) && segno % SEGS_PER_SEC(sbi)) {
2857 int devi = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno));
2858
2859 if (f2fs_dev_is_reserving(sbi, devi))
2860 return 0;
2861 return !test_bit(segno, free_i->free_segmap);
2862 }
2863 return 0;
2864}
2865
2866/*
2867 * Find a new segment from the free segments bitmap to right order
2868 * This function should be returned with success, otherwise BUG
2869 */
2870static int get_new_segment(struct f2fs_sb_info *sbi,
2871 unsigned int *newseg, bool new_sec, bool pinning)
2872{
2873 struct free_segmap_info *free_i = FREE_I(sbi);
2874 unsigned int segno, secno, zoneno;
2875 unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;
2876 unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg);
2877 unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg);
2878 unsigned int alloc_policy = sbi->allocate_section_policy;
2879 unsigned int alloc_hint = sbi->allocate_section_hint;
2880 unsigned int max_secno = MAIN_SECS(sbi);
2881 bool init = true;
2882 bool looped = false;
2883 int i, devi;
2884 int ret = 0;
2885
2886 spin_lock(&free_i->segmap_lock);
2887
2888 if (time_to_inject(sbi, FAULT_NO_SEGMENT)) {
2889 ret = -ENOSPC;
2890 goto out_unlock;
2891 }
2892
2893 if (!new_sec && ((*newseg + 1) % SEGS_PER_SEC(sbi))) {
2894 segno = find_next_zero_bit(free_i->free_segmap,
2895 GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1);
2896 if (segno < GET_SEG_FROM_SEC(sbi, hint + 1)) {
2897 devi = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno));
2898
2899 if (f2fs_dev_is_alloc_blocked(sbi, devi, pinning))
2900 goto find_other_zone;
2901 goto got_it;
2902 }
2903 }
2904
2905#ifdef CONFIG_BLK_DEV_ZONED
2906 /*
2907 * If we format f2fs on zoned storage, let's try to get pinned sections
2908 * from beginning of the storage, which should be a conventional one.
2909 */
2910 if (f2fs_sb_has_blkzoned(sbi)) {
2911 /* Prioritize writing to conventional zones */
2912 if (sbi->blkzone_alloc_policy == BLKZONE_ALLOC_PRIOR_CONV)
2913 segno = 0;
2914 else
2915 segno = max(sbi->first_seq_zone_segno, *newseg);
2916 hint = GET_SEC_FROM_SEG(sbi, segno);
2917 }
2918#endif
2919
2920 /*
2921 * Prevent allocate_section_hint from exceeding MAIN_SECS()
2922 * due to desynchronization.
2923 */
2924 if (alloc_policy != ALLOCATE_FORWARD_NOHINT &&
2925 alloc_hint > MAIN_SECS(sbi))
2926 alloc_hint = MAIN_SECS(sbi);
2927
2928 if (pinning) {
2929 max_secno = sbi->pinned_area_max_secno;
2930 hint = 0;
2931 } else if (alloc_policy == ALLOCATE_FORWARD_FROM_HINT &&
2932 hint < alloc_hint) {
2933 hint = alloc_hint;
2934 } else if (alloc_policy == ALLOCATE_FORWARD_WITHIN_HINT &&
2935 hint >= alloc_hint) {
2936 hint = 0;
2937 }
2938
2939find_other_zone:
2940 secno = find_next_zero_bit(free_i->free_secmap, max_secno, hint);
2941
2942 if (secno >= max_secno) {
2943 if (looped) {
2944 ret = (pinning && has_unpinned_area(sbi)) ?
2945 -EAGAIN : -ENOSPC;
2946 f2fs_bug_on(sbi, !pinning);
2947 goto out_unlock;
2948 }
2949 hint = 0;
2950#ifdef CONFIG_BLK_DEV_ZONED
2951 /* Write only to sequential zones */
2952 if (f2fs_sb_has_blkzoned(sbi) &&
2953 sbi->blkzone_alloc_policy == BLKZONE_ALLOC_ONLY_SEQ)
2954 hint = GET_SEC_FROM_SEG(sbi, sbi->first_seq_zone_segno);
2955#endif
2956 looped = true;
2957 goto find_other_zone;
2958 }
2959
2960 segno = GET_SEG_FROM_SEC(sbi, secno);
2961
2962 devi = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno));
2963
2964 if (f2fs_dev_is_alloc_blocked(sbi, devi, pinning)) {
2965 while (devi < sbi->s_ndevs &&
2966 f2fs_dev_is_alloc_blocked(sbi, devi, pinning)) {
2967 unsigned int end_segno = GET_SEGNO(sbi, FDEV(devi).end_blk);
2968
2969 hint = GET_SEC_FROM_SEG(sbi, end_segno) + 1;
2970 devi++;
2971 }
2972 goto find_other_zone;
2973 }
2974
2975 if (sec_usage_check(sbi, secno)) {
2976 hint = secno + 1;
2977 goto find_other_zone;
2978 }
2979 zoneno = GET_ZONE_FROM_SEC(sbi, secno);
2980
2981 /* give up on finding another zone */
2982 if (!init)
2983 goto got_it;
2984 if (sbi->secs_per_zone == 1)
2985 goto got_it;
2986 if (zoneno == old_zoneno)
2987 goto got_it;
2988 for (i = 0; i < NR_CURSEG_TYPE; i++)
2989 if (CURSEG_I(sbi, i)->zone == zoneno)
2990 break;
2991
2992 if (i < NR_CURSEG_TYPE) {
2993 /* zone is in user, try another */
2994 if (zoneno + 1 >= total_zones)
2995 hint = 0;
2996 else
2997 hint = (zoneno + 1) * sbi->secs_per_zone;
2998 init = false;
2999 goto find_other_zone;
3000 }
3001got_it:
3002 /* set it as dirty segment in free segmap */
3003 if (test_bit(segno, free_i->free_segmap)) {
3004 ret = -EFSCORRUPTED;
3005 fserror_report_metadata(sbi->sb, -EFSCORRUPTED, GFP_NOFS);
3006 f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_CORRUPTED_FREE_BITMAP);
3007 goto out_unlock;
3008 }
3009
3010 __set_inuse(sbi, segno);
3011 *newseg = segno;
3012out_unlock:
3013 spin_unlock(&free_i->segmap_lock);
3014
3015 if (ret == -ENOSPC && !pinning)
3016 f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_NO_SEGMENT);
3017 return ret;
3018}
3019
3020static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
3021{
3022 struct curseg_info *curseg = CURSEG_I(sbi, type);
3023 struct summary_footer *sum_footer;
3024 unsigned short seg_type = curseg->seg_type;
3025
3026 /* only happen when get_new_segment() fails */
3027 if (curseg->next_segno == NULL_SEGNO)
3028 return;
3029
3030 curseg->inited = true;
3031 curseg->segno = curseg->next_segno;
3032 curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno);
3033 curseg->next_blkoff = 0;
3034 curseg->next_segno = NULL_SEGNO;
3035
3036 sum_footer = sum_footer(sbi, curseg->sum_blk);
3037 memset(sum_footer, 0, sizeof(struct summary_footer));
3038
3039 sanity_check_seg_type(sbi, seg_type);
3040
3041 if (IS_DATASEG(seg_type))
3042 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
3043 if (IS_NODESEG(seg_type))
3044 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
3045 __set_sit_entry_type(sbi, seg_type, curseg->segno, modified);
3046}
3047
3048static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type)
3049{
3050 struct curseg_info *curseg = CURSEG_I(sbi, type);
3051 unsigned short seg_type = curseg->seg_type;
3052
3053 sanity_check_seg_type(sbi, seg_type);
3054 if (__is_large_section(sbi)) {
3055 if (f2fs_need_rand_seg_blk(sbi, type)) {
3056 unsigned int hint = GET_SEC_FROM_SEG(sbi, curseg->segno);
3057
3058 if (GET_SEC_FROM_SEG(sbi, curseg->segno + 1) != hint)
3059 return curseg->segno;
3060 return get_random_u32_inclusive(curseg->segno + 1,
3061 GET_SEG_FROM_SEC(sbi, hint + 1) - 1);
3062 }
3063 return curseg->segno;
3064 } else if (f2fs_need_rand_seg_blk(sbi, type)) {
3065 return get_random_u32_below(MAIN_SECS(sbi) * SEGS_PER_SEC(sbi));
3066 }
3067
3068 /* inmem log may not locate on any segment after mount */
3069 if (!curseg->inited)
3070 return 0;
3071
3072 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
3073 return 0;
3074
3075 if (seg_type == CURSEG_HOT_DATA || IS_NODESEG(seg_type))
3076 return 0;
3077
3078 if (SIT_I(sbi)->last_victim[ALLOC_NEXT])
3079 return SIT_I(sbi)->last_victim[ALLOC_NEXT];
3080
3081 /* find segments from 0 to reuse freed segments */
3082 if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE)
3083 return 0;
3084
3085 return curseg->segno;
3086}
3087
3088static void reset_curseg_fields(struct curseg_info *curseg)
3089{
3090 curseg->inited = false;
3091 curseg->segno = NULL_SEGNO;
3092 curseg->next_segno = 0;
3093}
3094
3095/*
3096 * Allocate a current working segment.
3097 * This function always allocates a free segment in LFS manner.
3098 */
3099static int new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
3100{
3101 struct curseg_info *curseg = CURSEG_I(sbi, type);
3102 unsigned int segno = curseg->segno;
3103 bool pinning = type == CURSEG_COLD_DATA_PINNED;
3104 int ret;
3105
3106 if (curseg->inited)
3107 write_sum_page(sbi, curseg->sum_blk, segno);
3108
3109 segno = __get_next_segno(sbi, type);
3110 ret = get_new_segment(sbi, &segno, new_sec, pinning);
3111 if (ret) {
3112 if (ret == -ENOSPC)
3113 reset_curseg_fields(curseg);
3114 return ret;
3115 }
3116
3117 curseg->next_segno = segno;
3118 reset_curseg(sbi, type, 1);
3119 curseg->alloc_type = LFS;
3120 if (f2fs_need_rand_blk(sbi, type))
3121 curseg->fragment_remained_chunk =
3122 get_random_u32_inclusive(1, sbi->max_fragment_chunk);
3123 return 0;
3124}
3125
3126static int __next_free_blkoff(struct f2fs_sb_info *sbi,
3127 int segno, block_t start)
3128{
3129 struct seg_entry *se = get_seg_entry(sbi, segno);
3130 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
3131 unsigned long *target_map = SIT_I(sbi)->tmp_map;
3132 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
3133 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
3134 int i;
3135
3136 for (i = 0; i < entries; i++)
3137 target_map[i] = ckpt_map[i] | cur_map[i];
3138
3139 return __find_rev_next_zero_bit(target_map, BLKS_PER_SEG(sbi), start);
3140}
3141
3142static int f2fs_find_next_ssr_block(struct f2fs_sb_info *sbi,
3143 struct curseg_info *seg)
3144{
3145 return __next_free_blkoff(sbi, seg->segno, seg->next_blkoff + 1);
3146}
3147
3148bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno)
3149{
3150 return __next_free_blkoff(sbi, segno, 0) < BLKS_PER_SEG(sbi);
3151}
3152
3153/*
3154 * This function always allocates a used segment(from dirty seglist) by SSR
3155 * manner, so it should recover the existing segment information of valid blocks
3156 */
3157static int change_curseg(struct f2fs_sb_info *sbi, int type)
3158{
3159 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
3160 struct curseg_info *curseg = CURSEG_I(sbi, type);
3161 unsigned int new_segno = curseg->next_segno;
3162 struct f2fs_summary_block *sum_node;
3163 struct folio *sum_folio;
3164
3165 if (curseg->inited)
3166 write_sum_page(sbi, curseg->sum_blk, curseg->segno);
3167
3168 __set_test_and_inuse(sbi, new_segno);
3169
3170 mutex_lock(&dirty_i->seglist_lock);
3171 __remove_dirty_segment(sbi, new_segno, PRE);
3172 __remove_dirty_segment(sbi, new_segno, DIRTY);
3173 mutex_unlock(&dirty_i->seglist_lock);
3174
3175 reset_curseg(sbi, type, 1);
3176 curseg->alloc_type = SSR;
3177 curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0);
3178
3179 sum_folio = f2fs_get_sum_folio(sbi, new_segno);
3180 if (IS_ERR(sum_folio)) {
3181 /* GC won't be able to use stale summary pages by cp_error */
3182 memset(curseg->sum_blk, 0, sbi->sum_entry_size);
3183 return PTR_ERR(sum_folio);
3184 }
3185 sum_node = SUM_BLK_PAGE_ADDR(sbi, sum_folio, new_segno);
3186 memcpy(curseg->sum_blk, sum_node, sbi->sum_entry_size);
3187 f2fs_folio_put(sum_folio, true);
3188 return 0;
3189}
3190
3191static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
3192 int alloc_mode, unsigned long long age);
3193
3194static int get_atssr_segment(struct f2fs_sb_info *sbi, int type,
3195 int target_type, int alloc_mode,
3196 unsigned long long age)
3197{
3198 struct curseg_info *curseg = CURSEG_I(sbi, type);
3199 int ret = 0;
3200
3201 curseg->seg_type = target_type;
3202
3203 if (get_ssr_segment(sbi, type, alloc_mode, age)) {
3204 struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno);
3205
3206 curseg->seg_type = se->type;
3207 ret = change_curseg(sbi, type);
3208 } else {
3209 /* allocate cold segment by default */
3210 curseg->seg_type = CURSEG_COLD_DATA;
3211 ret = new_curseg(sbi, type, true);
3212 }
3213 stat_inc_seg_type(sbi, curseg);
3214 return ret;
3215}
3216
3217static int __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi, bool force)
3218{
3219 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC);
3220 int ret = 0;
3221
3222 if (!sbi->am.atgc_enabled && !force)
3223 return 0;
3224
3225 f2fs_down_read(&SM_I(sbi)->curseg_lock);
3226
3227 mutex_lock(&curseg->curseg_mutex);
3228 down_write(&SIT_I(sbi)->sentry_lock);
3229
3230 ret = get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC,
3231 CURSEG_COLD_DATA, SSR, 0);
3232
3233 up_write(&SIT_I(sbi)->sentry_lock);
3234 mutex_unlock(&curseg->curseg_mutex);
3235
3236 f2fs_up_read(&SM_I(sbi)->curseg_lock);
3237 return ret;
3238}
3239
3240int f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi)
3241{
3242 return __f2fs_init_atgc_curseg(sbi, false);
3243}
3244
3245int f2fs_reinit_atgc_curseg(struct f2fs_sb_info *sbi)
3246{
3247 int ret;
3248
3249 if (!test_opt(sbi, ATGC))
3250 return 0;
3251 if (sbi->am.atgc_enabled)
3252 return 0;
3253 if (le64_to_cpu(F2FS_CKPT(sbi)->elapsed_time) <
3254 sbi->am.age_threshold)
3255 return 0;
3256
3257 ret = __f2fs_init_atgc_curseg(sbi, true);
3258 if (!ret) {
3259 sbi->am.atgc_enabled = true;
3260 f2fs_info(sbi, "reenabled age threshold GC");
3261 }
3262 return ret;
3263}
3264
3265static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type)
3266{
3267 struct curseg_info *curseg = CURSEG_I(sbi, type);
3268
3269 mutex_lock(&curseg->curseg_mutex);
3270 if (!curseg->inited)
3271 goto out;
3272
3273 if (get_valid_blocks(sbi, curseg->segno, false)) {
3274 write_sum_page(sbi, curseg->sum_blk, curseg->segno);
3275 } else {
3276 mutex_lock(&DIRTY_I(sbi)->seglist_lock);
3277 __set_test_and_free(sbi, curseg->segno, true);
3278 mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
3279 }
3280out:
3281 mutex_unlock(&curseg->curseg_mutex);
3282}
3283
3284void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi)
3285{
3286 __f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
3287
3288 if (sbi->am.atgc_enabled)
3289 __f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
3290}
3291
3292static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type)
3293{
3294 struct curseg_info *curseg = CURSEG_I(sbi, type);
3295
3296 mutex_lock(&curseg->curseg_mutex);
3297 if (!curseg->inited)
3298 goto out;
3299 if (get_valid_blocks(sbi, curseg->segno, false))
3300 goto out;
3301
3302 mutex_lock(&DIRTY_I(sbi)->seglist_lock);
3303 __set_test_and_inuse(sbi, curseg->segno);
3304 mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
3305out:
3306 mutex_unlock(&curseg->curseg_mutex);
3307}
3308
3309void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi)
3310{
3311 __f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
3312
3313 if (sbi->am.atgc_enabled)
3314 __f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
3315}
3316
3317static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
3318 int alloc_mode, unsigned long long age)
3319{
3320 struct curseg_info *curseg = CURSEG_I(sbi, type);
3321 unsigned segno = NULL_SEGNO;
3322 unsigned short seg_type = curseg->seg_type;
3323 int i, cnt;
3324 bool reversed = false;
3325
3326 sanity_check_seg_type(sbi, seg_type);
3327
3328 /* f2fs_need_SSR() already forces to do this */
3329 if (!f2fs_get_victim(sbi, &segno, BG_GC, seg_type,
3330 alloc_mode, age, false)) {
3331 curseg->next_segno = segno;
3332 return 1;
3333 }
3334
3335 /* For node segments, let's do SSR more intensively */
3336 if (IS_NODESEG(seg_type)) {
3337 if (seg_type >= CURSEG_WARM_NODE) {
3338 reversed = true;
3339 i = CURSEG_COLD_NODE;
3340 } else {
3341 i = CURSEG_HOT_NODE;
3342 }
3343 cnt = NR_CURSEG_NODE_TYPE;
3344 } else {
3345 if (seg_type >= CURSEG_WARM_DATA) {
3346 reversed = true;
3347 i = CURSEG_COLD_DATA;
3348 } else {
3349 i = CURSEG_HOT_DATA;
3350 }
3351 cnt = NR_CURSEG_DATA_TYPE;
3352 }
3353
3354 for (; cnt-- > 0; reversed ? i-- : i++) {
3355 if (i == seg_type)
3356 continue;
3357 if (!f2fs_get_victim(sbi, &segno, BG_GC, i,
3358 alloc_mode, age, false)) {
3359 curseg->next_segno = segno;
3360 return 1;
3361 }
3362 }
3363
3364 /* find valid_blocks=0 in dirty list */
3365 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
3366 segno = get_free_segment(sbi);
3367 if (segno != NULL_SEGNO) {
3368 curseg->next_segno = segno;
3369 return 1;
3370 }
3371 }
3372 return 0;
3373}
3374
3375static bool need_new_seg(struct f2fs_sb_info *sbi, int type)
3376{
3377 struct curseg_info *curseg = CURSEG_I(sbi, type);
3378
3379 if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) &&
3380 curseg->seg_type == CURSEG_WARM_NODE)
3381 return true;
3382 if (curseg->alloc_type == LFS && is_next_segment_free(sbi, curseg) &&
3383 likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
3384 return true;
3385 if (!f2fs_need_SSR(sbi) || !get_ssr_segment(sbi, type, SSR, 0))
3386 return true;
3387 return false;
3388}
3389
3390int f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
3391 unsigned int start, unsigned int end)
3392{
3393 struct curseg_info *curseg = CURSEG_I(sbi, type);
3394 unsigned int segno;
3395 int ret = 0;
3396
3397 f2fs_down_read(&SM_I(sbi)->curseg_lock);
3398 mutex_lock(&curseg->curseg_mutex);
3399 down_write(&SIT_I(sbi)->sentry_lock);
3400
3401 segno = CURSEG_I(sbi, type)->segno;
3402 if (segno < start || segno > end)
3403 goto unlock;
3404
3405 if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0))
3406 ret = change_curseg(sbi, type);
3407 else
3408 ret = new_curseg(sbi, type, true);
3409
3410 stat_inc_seg_type(sbi, curseg);
3411
3412 locate_dirty_segment(sbi, segno);
3413unlock:
3414 up_write(&SIT_I(sbi)->sentry_lock);
3415
3416 if (segno != curseg->segno)
3417 f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u",
3418 type, segno, curseg->segno);
3419
3420 mutex_unlock(&curseg->curseg_mutex);
3421 f2fs_up_read(&SM_I(sbi)->curseg_lock);
3422 return ret;
3423}
3424
3425static int __allocate_new_segment(struct f2fs_sb_info *sbi, int type,
3426 bool new_sec, bool force)
3427{
3428 struct curseg_info *curseg = CURSEG_I(sbi, type);
3429 unsigned int old_segno;
3430 int err = 0;
3431
3432 if (type == CURSEG_COLD_DATA_PINNED && !curseg->inited)
3433 goto allocate;
3434
3435 if (!force && curseg->inited &&
3436 !curseg->next_blkoff &&
3437 !get_valid_blocks(sbi, curseg->segno, new_sec) &&
3438 !get_ckpt_valid_blocks(sbi, curseg->segno, new_sec))
3439 return 0;
3440
3441allocate:
3442 old_segno = curseg->segno;
3443 err = new_curseg(sbi, type, true);
3444 if (err)
3445 return err;
3446 stat_inc_seg_type(sbi, curseg);
3447 locate_dirty_segment(sbi, old_segno);
3448 return 0;
3449}
3450
3451int f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force)
3452{
3453 int ret;
3454
3455 f2fs_down_read(&SM_I(sbi)->curseg_lock);
3456 down_write(&SIT_I(sbi)->sentry_lock);
3457 ret = __allocate_new_segment(sbi, type, true, force);
3458 up_write(&SIT_I(sbi)->sentry_lock);
3459 f2fs_up_read(&SM_I(sbi)->curseg_lock);
3460
3461 return ret;
3462}
3463
3464int f2fs_allocate_pinning_section(struct f2fs_sb_info *sbi)
3465{
3466 struct f2fs_lock_context lc;
3467 int err;
3468 bool gc_required = true;
3469
3470retry:
3471 f2fs_lock_op(sbi, &lc);
3472 err = f2fs_allocate_new_section(sbi, CURSEG_COLD_DATA_PINNED, false);
3473 f2fs_unlock_op(sbi, &lc);
3474
3475 if (has_unpinned_area(sbi) && err == -EAGAIN && gc_required) {
3476 err = f2fs_gc_range(sbi, 0,
3477 sbi->pinned_area_max_secno * SEGS_PER_SEC(sbi) - 1,
3478 true, ZONED_PIN_SEC_REQUIRED_COUNT, true);
3479 if (err)
3480 return err;
3481 err = f2fs_sync_fs(sbi->sb, 1);
3482 if (!err) {
3483 gc_required = false;
3484 goto retry;
3485 }
3486 }
3487
3488 return err;
3489}
3490
3491int f2fs_allocate_new_segments(struct f2fs_sb_info *sbi)
3492{
3493 int i;
3494 int err = 0;
3495
3496 f2fs_down_read(&SM_I(sbi)->curseg_lock);
3497 down_write(&SIT_I(sbi)->sentry_lock);
3498 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++)
3499 err += __allocate_new_segment(sbi, i, false, false);
3500 up_write(&SIT_I(sbi)->sentry_lock);
3501 f2fs_up_read(&SM_I(sbi)->curseg_lock);
3502
3503 return err;
3504}
3505
3506bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
3507 struct cp_control *cpc)
3508{
3509 __u64 trim_start = cpc->trim_start;
3510 bool has_candidate = false;
3511
3512 down_write(&SIT_I(sbi)->sentry_lock);
3513 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) {
3514 if (add_discard_addrs(sbi, cpc, true)) {
3515 has_candidate = true;
3516 break;
3517 }
3518 }
3519 up_write(&SIT_I(sbi)->sentry_lock);
3520
3521 cpc->trim_start = trim_start;
3522 return has_candidate;
3523}
3524
3525static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi,
3526 struct discard_policy *dpolicy,
3527 unsigned int start, unsigned int end)
3528{
3529 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
3530 struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
3531 struct rb_node **insert_p = NULL, *insert_parent = NULL;
3532 struct discard_cmd *dc;
3533 struct blk_plug plug;
3534 int issued;
3535 unsigned int trimmed = 0;
3536
3537next:
3538 issued = 0;
3539
3540 mutex_lock(&dcc->cmd_lock);
3541 if (unlikely(dcc->rbtree_check))
3542 f2fs_bug_on(sbi, !f2fs_check_discard_tree(sbi));
3543
3544 dc = __lookup_discard_cmd_ret(&dcc->root, start,
3545 &prev_dc, &next_dc, &insert_p, &insert_parent);
3546 if (!dc)
3547 dc = next_dc;
3548
3549 blk_start_plug(&plug);
3550
3551 while (dc && dc->di.lstart <= end) {
3552 struct rb_node *node;
3553 int err = 0;
3554
3555 if (dc->di.len < dpolicy->granularity)
3556 goto skip;
3557
3558 if (dc->state != D_PREP) {
3559 list_move_tail(&dc->list, &dcc->fstrim_list);
3560 goto skip;
3561 }
3562
3563 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
3564
3565 if (issued >= dpolicy->max_requests) {
3566 start = dc->di.lstart + dc->di.len;
3567
3568 if (err)
3569 __remove_discard_cmd(sbi, dc);
3570
3571 blk_finish_plug(&plug);
3572 mutex_unlock(&dcc->cmd_lock);
3573 trimmed += __wait_all_discard_cmd(sbi, NULL);
3574 f2fs_schedule_timeout(DEFAULT_DISCARD_INTERVAL);
3575 goto next;
3576 }
3577skip:
3578 node = rb_next(&dc->rb_node);
3579 if (err)
3580 __remove_discard_cmd(sbi, dc);
3581 dc = rb_entry_safe(node, struct discard_cmd, rb_node);
3582
3583 if (fatal_signal_pending(current))
3584 break;
3585 }
3586
3587 blk_finish_plug(&plug);
3588 mutex_unlock(&dcc->cmd_lock);
3589
3590 return trimmed;
3591}
3592
3593int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
3594{
3595 __u64 start = F2FS_BYTES_TO_BLK(range->start);
3596 __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
3597 unsigned int start_segno, end_segno;
3598 block_t start_block, end_block;
3599 struct cp_control cpc;
3600 struct discard_policy dpolicy;
3601 struct f2fs_lock_context lc;
3602 unsigned long long trimmed = 0;
3603 int err = 0;
3604 bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi);
3605
3606 if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)
3607 return -EINVAL;
3608
3609 if (end < MAIN_BLKADDR(sbi))
3610 goto out;
3611
3612 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
3613 f2fs_warn(sbi, "Found FS corruption, run fsck to fix.");
3614 return -EFSCORRUPTED;
3615 }
3616
3617 /* start/end segment number in main_area */
3618 start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);
3619 end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :
3620 GET_SEGNO(sbi, end);
3621 if (need_align) {
3622 start_segno = rounddown(start_segno, SEGS_PER_SEC(sbi));
3623 end_segno = roundup(end_segno + 1, SEGS_PER_SEC(sbi)) - 1;
3624 }
3625
3626 cpc.reason = CP_DISCARD;
3627 cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
3628 cpc.trim_start = start_segno;
3629 cpc.trim_end = end_segno;
3630
3631 if (sbi->discard_blks == 0)
3632 goto out;
3633
3634 f2fs_down_write_trace(&sbi->gc_lock, &lc);
3635 stat_inc_cp_call_count(sbi, TOTAL_CALL);
3636 err = f2fs_write_checkpoint(sbi, &cpc);
3637 f2fs_up_write_trace(&sbi->gc_lock, &lc);
3638 if (err)
3639 goto out;
3640
3641 /*
3642 * We filed discard candidates, but actually we don't need to wait for
3643 * all of them, since they'll be issued in idle time along with runtime
3644 * discard option. User configuration looks like using runtime discard
3645 * or periodic fstrim instead of it.
3646 */
3647 if (f2fs_realtime_discard_enable(sbi))
3648 goto out;
3649
3650 start_block = START_BLOCK(sbi, start_segno);
3651 end_block = START_BLOCK(sbi, end_segno + 1);
3652
3653 __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen);
3654 trimmed = __issue_discard_cmd_range(sbi, &dpolicy,
3655 start_block, end_block);
3656
3657 trimmed += __wait_discard_cmd_range(sbi, &dpolicy,
3658 start_block, end_block);
3659out:
3660 if (!err)
3661 range->len = F2FS_BLK_TO_BYTES(trimmed);
3662 return err;
3663}
3664
3665int f2fs_rw_hint_to_seg_type(struct f2fs_sb_info *sbi, enum rw_hint hint)
3666{
3667 if (F2FS_OPTION(sbi).active_logs == 2)
3668 return CURSEG_HOT_DATA;
3669 else if (F2FS_OPTION(sbi).active_logs == 4)
3670 return CURSEG_COLD_DATA;
3671
3672 /* active_log == 6 */
3673 switch (hint) {
3674 case WRITE_LIFE_SHORT:
3675 return CURSEG_HOT_DATA;
3676 case WRITE_LIFE_EXTREME:
3677 return CURSEG_COLD_DATA;
3678 default:
3679 return CURSEG_WARM_DATA;
3680 }
3681}
3682
3683/*
3684 * This returns write hints for each segment type. This hints will be
3685 * passed down to block layer as below by default.
3686 *
3687 * User F2FS Block
3688 * ---- ---- -----
3689 * META WRITE_LIFE_NONE|REQ_META
3690 * HOT_NODE WRITE_LIFE_NONE
3691 * WARM_NODE WRITE_LIFE_MEDIUM
3692 * COLD_NODE WRITE_LIFE_LONG
3693 * ioctl(COLD) COLD_DATA WRITE_LIFE_EXTREME
3694 * extension list " "
3695 *
3696 * -- buffered io
3697 * COLD_DATA WRITE_LIFE_EXTREME
3698 * HOT_DATA WRITE_LIFE_SHORT
3699 * WARM_DATA WRITE_LIFE_NOT_SET
3700 *
3701 * -- direct io
3702 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME
3703 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT
3704 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET
3705 * WRITE_LIFE_NONE " WRITE_LIFE_NONE
3706 * WRITE_LIFE_MEDIUM " WRITE_LIFE_MEDIUM
3707 * WRITE_LIFE_LONG " WRITE_LIFE_LONG
3708 */
3709enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi,
3710 enum page_type type, enum temp_type temp)
3711{
3712 switch (type) {
3713 case DATA:
3714 switch (temp) {
3715 case WARM:
3716 return WRITE_LIFE_NOT_SET;
3717 case HOT:
3718 return WRITE_LIFE_SHORT;
3719 case COLD:
3720 return WRITE_LIFE_EXTREME;
3721 default:
3722 return WRITE_LIFE_NONE;
3723 }
3724 case NODE:
3725 switch (temp) {
3726 case WARM:
3727 return WRITE_LIFE_MEDIUM;
3728 case HOT:
3729 return WRITE_LIFE_NONE;
3730 case COLD:
3731 return WRITE_LIFE_LONG;
3732 default:
3733 return WRITE_LIFE_NONE;
3734 }
3735 case META:
3736 return WRITE_LIFE_NONE;
3737 default:
3738 return WRITE_LIFE_NONE;
3739 }
3740}
3741
3742u8 f2fs_io_type_to_write_stream(struct block_device *bdev,
3743 enum page_type type, enum temp_type temp)
3744{
3745 unsigned short nr = bdev_max_write_streams(bdev);
3746
3747 if (type != DATA || !nr)
3748 return 0;
3749 if (nr < NR_TEMP_TYPE)
3750 return temp == COLD ? nr : HOT + 1;
3751
3752 return temp + 1;
3753}
3754
3755static int __get_segment_type_2(struct f2fs_io_info *fio)
3756{
3757 if (fio->type == DATA)
3758 return CURSEG_HOT_DATA;
3759 else
3760 return CURSEG_HOT_NODE;
3761}
3762
3763static int __get_segment_type_4(struct f2fs_io_info *fio)
3764{
3765 if (fio->type == DATA) {
3766 struct inode *inode = fio_inode(fio);
3767
3768 if (S_ISDIR(inode->i_mode))
3769 return CURSEG_HOT_DATA;
3770 else
3771 return CURSEG_COLD_DATA;
3772 } else {
3773 if (IS_DNODE(fio->folio) && is_cold_node(fio->folio))
3774 return CURSEG_WARM_NODE;
3775 else
3776 return CURSEG_COLD_NODE;
3777 }
3778}
3779
3780static int __get_age_segment_type(struct inode *inode, pgoff_t pgofs)
3781{
3782 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3783 struct extent_info ei = {};
3784
3785 if (f2fs_lookup_age_extent_cache(inode, pgofs, &ei)) {
3786 if (!ei.age)
3787 return NO_CHECK_TYPE;
3788 if (ei.age <= sbi->hot_data_age_threshold)
3789 return CURSEG_HOT_DATA;
3790 if (ei.age <= sbi->warm_data_age_threshold)
3791 return CURSEG_WARM_DATA;
3792 return CURSEG_COLD_DATA;
3793 }
3794 return NO_CHECK_TYPE;
3795}
3796
3797static int __get_segment_type_6(struct f2fs_io_info *fio)
3798{
3799 if (fio->type == DATA) {
3800 struct inode *inode = fio_inode(fio);
3801 int type;
3802
3803 if (is_inode_flag_set(inode, FI_ALIGNED_WRITE))
3804 return CURSEG_COLD_DATA_PINNED;
3805
3806 if (page_private_gcing(fio->page)) {
3807 if (fio->sbi->am.atgc_enabled &&
3808 (fio->io_type == FS_DATA_IO) &&
3809 (fio->sbi->gc_mode != GC_URGENT_HIGH) &&
3810 __is_valid_data_blkaddr(fio->old_blkaddr) &&
3811 !is_inode_flag_set(inode, FI_OPU_WRITE))
3812 return CURSEG_ALL_DATA_ATGC;
3813 else
3814 return CURSEG_COLD_DATA;
3815 }
3816 if (file_is_cold(inode) || f2fs_need_compress_data(inode))
3817 return CURSEG_COLD_DATA;
3818
3819 type = __get_age_segment_type(inode, fio->folio->index);
3820 if (type != NO_CHECK_TYPE)
3821 return type;
3822
3823 if (file_is_hot(inode) ||
3824 is_inode_flag_set(inode, FI_HOT_DATA) ||
3825 f2fs_is_cow_file(inode) ||
3826 is_inode_flag_set(inode, FI_NEED_IPU))
3827 return CURSEG_HOT_DATA;
3828 return f2fs_rw_hint_to_seg_type(F2FS_I_SB(inode),
3829 inode->i_write_hint);
3830 } else {
3831 if (IS_DNODE(fio->folio))
3832 return is_cold_node(fio->folio) ? CURSEG_WARM_NODE :
3833 CURSEG_HOT_NODE;
3834 return CURSEG_COLD_NODE;
3835 }
3836}
3837
3838enum temp_type f2fs_get_segment_temp(struct f2fs_sb_info *sbi,
3839 enum log_type type)
3840{
3841 struct curseg_info *curseg = CURSEG_I(sbi, type);
3842 enum temp_type temp = COLD;
3843
3844 switch (curseg->seg_type) {
3845 case CURSEG_HOT_NODE:
3846 case CURSEG_HOT_DATA:
3847 temp = HOT;
3848 break;
3849 case CURSEG_WARM_NODE:
3850 case CURSEG_WARM_DATA:
3851 temp = WARM;
3852 break;
3853 case CURSEG_COLD_NODE:
3854 case CURSEG_COLD_DATA:
3855 temp = COLD;
3856 break;
3857 default:
3858 f2fs_bug_on(sbi, 1);
3859 }
3860
3861 return temp;
3862}
3863
3864static int __get_segment_type(struct f2fs_io_info *fio)
3865{
3866 enum log_type type = CURSEG_HOT_DATA;
3867
3868 switch (F2FS_OPTION(fio->sbi).active_logs) {
3869 case 2:
3870 type = __get_segment_type_2(fio);
3871 break;
3872 case 4:
3873 type = __get_segment_type_4(fio);
3874 break;
3875 case 6:
3876 type = __get_segment_type_6(fio);
3877 break;
3878 default:
3879 f2fs_bug_on(fio->sbi, true);
3880 }
3881
3882 fio->temp = f2fs_get_segment_temp(fio->sbi, type);
3883
3884 return type;
3885}
3886
3887static void f2fs_randomize_chunk(struct f2fs_sb_info *sbi,
3888 struct curseg_info *seg)
3889{
3890 /* To allocate block chunks in different sizes, use random number */
3891 if (--seg->fragment_remained_chunk > 0)
3892 return;
3893
3894 seg->fragment_remained_chunk =
3895 get_random_u32_inclusive(1, sbi->max_fragment_chunk);
3896 seg->next_blkoff +=
3897 get_random_u32_inclusive(1, sbi->max_fragment_hole);
3898}
3899
3900int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio,
3901 block_t old_blkaddr, block_t *new_blkaddr,
3902 struct f2fs_summary *sum, int type,
3903 struct f2fs_io_info *fio)
3904{
3905 struct sit_info *sit_i = SIT_I(sbi);
3906 struct curseg_info *curseg = CURSEG_I(sbi, type);
3907 unsigned long long old_mtime;
3908 bool from_gc = (type == CURSEG_ALL_DATA_ATGC);
3909 struct seg_entry *se = NULL;
3910 bool segment_full = false;
3911 int ret = 0;
3912
3913 f2fs_down_read(&SM_I(sbi)->curseg_lock);
3914
3915 mutex_lock(&curseg->curseg_mutex);
3916 down_write(&sit_i->sentry_lock);
3917
3918 if (curseg->segno == NULL_SEGNO) {
3919 ret = -ENOSPC;
3920 goto out_err;
3921 }
3922
3923 if (from_gc) {
3924 f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO);
3925 se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr));
3926 sanity_check_seg_type(sbi, se->type);
3927 f2fs_bug_on(sbi, IS_NODESEG(se->type));
3928 }
3929 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
3930
3931 f2fs_bug_on(sbi, curseg->next_blkoff >= BLKS_PER_SEG(sbi));
3932
3933 f2fs_wait_discard_bio(sbi, *new_blkaddr);
3934
3935 sum_entries(curseg->sum_blk)[curseg->next_blkoff] = *sum;
3936 if (curseg->alloc_type == SSR) {
3937 curseg->next_blkoff = f2fs_find_next_ssr_block(sbi, curseg);
3938 } else {
3939 curseg->next_blkoff++;
3940 if (f2fs_need_rand_blk(sbi, type))
3941 f2fs_randomize_chunk(sbi, curseg);
3942 }
3943 if (curseg->next_blkoff >= f2fs_usable_blks_in_seg(sbi, curseg->segno))
3944 segment_full = true;
3945 stat_inc_block_count(sbi, curseg);
3946
3947 if (from_gc) {
3948 old_mtime = get_segment_mtime(sbi, old_blkaddr);
3949 } else {
3950 update_segment_mtime(sbi, old_blkaddr, 0);
3951 old_mtime = 0;
3952 }
3953 update_segment_mtime(sbi, *new_blkaddr, old_mtime);
3954
3955 /*
3956 * SIT information should be updated before segment allocation,
3957 * since SSR needs latest valid block information.
3958 */
3959 update_sit_entry(sbi, *new_blkaddr, 1);
3960 update_sit_entry(sbi, old_blkaddr, -1);
3961
3962 /*
3963 * If the current segment is full, flush it out and replace it with a
3964 * new segment.
3965 */
3966 if (segment_full) {
3967 if (type == CURSEG_COLD_DATA_PINNED &&
3968 !((curseg->segno + 1) % sbi->segs_per_sec)) {
3969 write_sum_page(sbi, curseg->sum_blk, curseg->segno);
3970 reset_curseg_fields(curseg);
3971 goto skip_new_segment;
3972 }
3973
3974 if (from_gc) {
3975 ret = get_atssr_segment(sbi, type, se->type,
3976 AT_SSR, se->mtime);
3977 } else {
3978 if (need_new_seg(sbi, type))
3979 ret = new_curseg(sbi, type, false);
3980 else
3981 ret = change_curseg(sbi, type);
3982 stat_inc_seg_type(sbi, curseg);
3983 }
3984
3985 if (ret)
3986 goto out_err;
3987 }
3988
3989skip_new_segment:
3990 /*
3991 * segment dirty status should be updated after segment allocation,
3992 * so we just need to update status only one time after previous
3993 * segment being closed.
3994 */
3995 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3996 locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr));
3997
3998 if (IS_DATASEG(curseg->seg_type)) {
3999 unsigned long long new_val;
4000
4001 new_val = atomic64_inc_return(&sbi->allocated_data_blocks);
4002 if (unlikely(new_val == ULLONG_MAX))
4003 atomic64_set(&sbi->allocated_data_blocks, 0);
4004 }
4005
4006 up_write(&sit_i->sentry_lock);
4007
4008 if (folio && IS_NODESEG(curseg->seg_type)) {
4009 fill_node_footer_blkaddr(folio, NEXT_FREE_BLKADDR(sbi, curseg));
4010
4011 f2fs_inode_chksum_set(sbi, folio);
4012 }
4013
4014 if (fio) {
4015 struct f2fs_bio_info *io;
4016
4017 INIT_LIST_HEAD(&fio->list);
4018 fio->in_list = 1;
4019 io = sbi->write_io[fio->type] + fio->temp;
4020 spin_lock(&io->io_lock);
4021 list_add_tail(&fio->list, &io->io_list);
4022 spin_unlock(&io->io_lock);
4023 }
4024
4025 mutex_unlock(&curseg->curseg_mutex);
4026 f2fs_up_read(&SM_I(sbi)->curseg_lock);
4027 return 0;
4028
4029out_err:
4030 *new_blkaddr = NULL_ADDR;
4031 up_write(&sit_i->sentry_lock);
4032 mutex_unlock(&curseg->curseg_mutex);
4033 f2fs_up_read(&SM_I(sbi)->curseg_lock);
4034 return ret;
4035}
4036
4037void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino,
4038 block_t blkaddr, unsigned int blkcnt)
4039{
4040 if (!f2fs_is_multi_device(sbi))
4041 return;
4042
4043 while (1) {
4044 unsigned int devidx = f2fs_target_device_index(sbi, blkaddr);
4045 unsigned int blks = FDEV(devidx).end_blk - blkaddr + 1;
4046
4047 /* update device state for fsync */
4048 f2fs_set_dirty_device(sbi, ino, devidx, FLUSH_INO);
4049
4050 /* update device state for checkpoint */
4051 if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) {
4052 spin_lock(&sbi->dev_lock);
4053 f2fs_set_bit(devidx, (char *)&sbi->dirty_device);
4054 spin_unlock(&sbi->dev_lock);
4055 }
4056
4057 if (blkcnt <= blks)
4058 break;
4059 blkcnt -= blks;
4060 blkaddr += blks;
4061 }
4062}
4063
4064static int log_type_to_seg_type(enum log_type type)
4065{
4066 int seg_type = CURSEG_COLD_DATA;
4067
4068 switch (type) {
4069 case CURSEG_HOT_DATA:
4070 case CURSEG_WARM_DATA:
4071 case CURSEG_COLD_DATA:
4072 case CURSEG_HOT_NODE:
4073 case CURSEG_WARM_NODE:
4074 case CURSEG_COLD_NODE:
4075 seg_type = (int)type;
4076 break;
4077 case CURSEG_COLD_DATA_PINNED:
4078 case CURSEG_ALL_DATA_ATGC:
4079 seg_type = CURSEG_COLD_DATA;
4080 break;
4081 default:
4082 break;
4083 }
4084 return seg_type;
4085}
4086
4087static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
4088{
4089 struct folio *folio = fio->folio;
4090 enum log_type type = __get_segment_type(fio);
4091 int seg_type = log_type_to_seg_type(type);
4092 bool keep_order = (f2fs_lfs_mode(fio->sbi) &&
4093 seg_type == CURSEG_COLD_DATA);
4094 int err;
4095
4096 if (keep_order)
4097 f2fs_down_read(&fio->sbi->io_order_lock);
4098
4099 err = f2fs_allocate_data_block(fio->sbi, folio, fio->old_blkaddr,
4100 &fio->new_blkaddr, sum, type, fio);
4101 if (unlikely(err)) {
4102 f2fs_err_ratelimited(fio->sbi,
4103 "%s Failed to allocate data block, ino:%u, index:%lu, type:%d, old_blkaddr:0x%x, new_blkaddr:0x%x, err:%d",
4104 __func__, fio->ino, folio->index, type,
4105 fio->old_blkaddr, fio->new_blkaddr, err);
4106 folio_end_writeback(folio);
4107 if (f2fs_in_warm_node_list(folio))
4108 f2fs_del_fsync_node_entry(fio->sbi, folio);
4109 f2fs_bug_on(fio->sbi, !is_set_ckpt_flags(fio->sbi,
4110 CP_ERROR_FLAG));
4111 goto out;
4112 }
4113
4114 f2fs_bug_on(fio->sbi, !f2fs_is_valid_blkaddr_raw(fio->sbi,
4115 fio->new_blkaddr, DATA_GENERIC_ENHANCE));
4116
4117 if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO)
4118 f2fs_invalidate_internal_cache(fio->sbi, fio->old_blkaddr, 1);
4119
4120 /* writeout dirty page into bdev */
4121 f2fs_submit_page_write(fio);
4122
4123 f2fs_update_device_state(fio->sbi, fio->ino, fio->new_blkaddr, 1);
4124out:
4125 if (keep_order)
4126 f2fs_up_read(&fio->sbi->io_order_lock);
4127}
4128
4129void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct folio *folio,
4130 enum iostat_type io_type)
4131{
4132 struct f2fs_io_info fio = {
4133 .sbi = sbi,
4134 .type = META,
4135 .temp = HOT,
4136 .op = REQ_OP_WRITE,
4137 .op_flags = REQ_SYNC | REQ_META | REQ_PRIO,
4138 .old_blkaddr = folio->index,
4139 .new_blkaddr = folio->index,
4140 .folio = folio,
4141 .encrypted_page = NULL,
4142 .in_list = 0,
4143 };
4144
4145 if (unlikely(folio->index >= MAIN_BLKADDR(sbi)))
4146 fio.op_flags &= ~REQ_META;
4147
4148 folio_start_writeback(folio);
4149 f2fs_submit_page_write(&fio);
4150
4151 stat_inc_meta_count(sbi, folio->index);
4152 f2fs_update_iostat(sbi, NULL, io_type, F2FS_BLKSIZE);
4153}
4154
4155void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio)
4156{
4157 struct f2fs_summary sum;
4158
4159 set_summary(&sum, nid, 0, 0);
4160 do_write_page(&sum, fio);
4161
4162 f2fs_update_iostat(fio->sbi, NULL, fio->io_type, F2FS_BLKSIZE);
4163}
4164
4165void f2fs_outplace_write_data(struct dnode_of_data *dn,
4166 struct f2fs_io_info *fio)
4167{
4168 struct f2fs_sb_info *sbi = fio->sbi;
4169 struct f2fs_summary sum;
4170
4171 f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
4172 if (fio->io_type == FS_DATA_IO || fio->io_type == FS_CP_DATA_IO)
4173 f2fs_update_age_extent_cache(dn);
4174 set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version);
4175 do_write_page(&sum, fio);
4176 f2fs_update_data_blkaddr(dn, fio->new_blkaddr);
4177
4178 f2fs_update_iostat(sbi, dn->inode, fio->io_type, F2FS_BLKSIZE);
4179}
4180
4181int f2fs_inplace_write_data(struct f2fs_io_info *fio)
4182{
4183 int err;
4184 struct f2fs_sb_info *sbi = fio->sbi;
4185 unsigned int segno;
4186
4187 fio->new_blkaddr = fio->old_blkaddr;
4188 /* i/o temperature is needed for passing down write hints */
4189 __get_segment_type(fio);
4190
4191 segno = GET_SEGNO(sbi, fio->new_blkaddr);
4192
4193 if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) {
4194 set_sbi_flag(sbi, SBI_NEED_FSCK);
4195 f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.",
4196 __func__, segno);
4197 err = -EFSCORRUPTED;
4198 f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE);
4199 goto drop_bio;
4200 }
4201
4202 if (f2fs_cp_error(sbi)) {
4203 err = -EIO;
4204 goto drop_bio;
4205 }
4206
4207 if (fio->meta_gc)
4208 f2fs_truncate_meta_inode_pages(sbi, fio->new_blkaddr, 1);
4209
4210 stat_inc_inplace_blocks(fio->sbi);
4211
4212 if (fio->bio && !IS_F2FS_IPU_NOCACHE(sbi))
4213 err = f2fs_merge_page_bio(fio);
4214 else
4215 err = f2fs_submit_page_bio(fio);
4216 if (!err) {
4217 f2fs_update_device_state(fio->sbi, fio->ino,
4218 fio->new_blkaddr, 1);
4219 f2fs_update_iostat(fio->sbi, fio_inode(fio),
4220 fio->io_type, F2FS_BLKSIZE);
4221 }
4222
4223 return err;
4224drop_bio:
4225 if (fio->bio && *(fio->bio)) {
4226 struct bio *bio = *(fio->bio);
4227
4228 bio->bi_status = BLK_STS_IOERR;
4229 bio_endio(bio);
4230 *(fio->bio) = NULL;
4231 }
4232 return err;
4233}
4234
4235static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi,
4236 unsigned int segno)
4237{
4238 int i;
4239
4240 for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) {
4241 if (CURSEG_I(sbi, i)->segno == segno)
4242 break;
4243 }
4244 return i;
4245}
4246
4247void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
4248 block_t old_blkaddr, block_t new_blkaddr,
4249 bool recover_curseg, bool recover_newaddr,
4250 bool from_gc)
4251{
4252 struct sit_info *sit_i = SIT_I(sbi);
4253 struct curseg_info *curseg;
4254 unsigned int segno, old_cursegno;
4255 struct seg_entry *se;
4256 int type;
4257 unsigned short old_blkoff;
4258 unsigned char old_alloc_type;
4259
4260 segno = GET_SEGNO(sbi, new_blkaddr);
4261 se = get_seg_entry(sbi, segno);
4262 type = se->type;
4263
4264 f2fs_down_write(&SM_I(sbi)->curseg_lock);
4265
4266 if (!recover_curseg) {
4267 /* for recovery flow */
4268 if (se->valid_blocks == 0 && !is_curseg(sbi, segno)) {
4269 if (old_blkaddr == NULL_ADDR)
4270 type = CURSEG_COLD_DATA;
4271 else
4272 type = CURSEG_WARM_DATA;
4273 }
4274 } else {
4275 if (is_curseg(sbi, segno)) {
4276 /* se->type is volatile as SSR allocation */
4277 type = __f2fs_get_curseg(sbi, segno);
4278 f2fs_bug_on(sbi, type == NO_CHECK_TYPE);
4279 } else {
4280 type = CURSEG_WARM_DATA;
4281 }
4282 }
4283
4284 curseg = CURSEG_I(sbi, type);
4285 f2fs_bug_on(sbi, !IS_DATASEG(curseg->seg_type));
4286
4287 mutex_lock(&curseg->curseg_mutex);
4288 down_write(&sit_i->sentry_lock);
4289
4290 old_cursegno = curseg->segno;
4291 old_blkoff = curseg->next_blkoff;
4292 old_alloc_type = curseg->alloc_type;
4293
4294 /* change the current segment */
4295 if (segno != curseg->segno) {
4296 curseg->next_segno = segno;
4297 if (change_curseg(sbi, type))
4298 goto out_unlock;
4299 }
4300
4301 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
4302 sum_entries(curseg->sum_blk)[curseg->next_blkoff] = *sum;
4303
4304 if (!recover_curseg || recover_newaddr) {
4305 if (!from_gc)
4306 update_segment_mtime(sbi, new_blkaddr, 0);
4307 update_sit_entry(sbi, new_blkaddr, 1);
4308 }
4309 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) {
4310 f2fs_invalidate_internal_cache(sbi, old_blkaddr, 1);
4311 if (!from_gc)
4312 update_segment_mtime(sbi, old_blkaddr, 0);
4313 update_sit_entry(sbi, old_blkaddr, -1);
4314 }
4315
4316 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
4317 locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));
4318
4319 locate_dirty_segment(sbi, old_cursegno);
4320
4321 if (recover_curseg) {
4322 if (old_cursegno != curseg->segno) {
4323 curseg->next_segno = old_cursegno;
4324 if (change_curseg(sbi, type))
4325 goto out_unlock;
4326 }
4327 curseg->next_blkoff = old_blkoff;
4328 curseg->alloc_type = old_alloc_type;
4329 }
4330
4331out_unlock:
4332 up_write(&sit_i->sentry_lock);
4333 mutex_unlock(&curseg->curseg_mutex);
4334 f2fs_up_write(&SM_I(sbi)->curseg_lock);
4335}
4336
4337void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
4338 block_t old_addr, block_t new_addr,
4339 unsigned char version, bool recover_curseg,
4340 bool recover_newaddr)
4341{
4342 struct f2fs_summary sum;
4343
4344 set_summary(&sum, dn->nid, dn->ofs_in_node, version);
4345
4346 f2fs_do_replace_block(sbi, &sum, old_addr, new_addr,
4347 recover_curseg, recover_newaddr, false);
4348
4349 f2fs_update_data_blkaddr(dn, new_addr);
4350}
4351
4352void f2fs_folio_wait_writeback(struct folio *folio, enum page_type type,
4353 bool ordered, bool locked)
4354{
4355 if (folio_test_writeback(folio)) {
4356 struct f2fs_sb_info *sbi = F2FS_F_SB(folio);
4357
4358 /* submit cached LFS IO */
4359 f2fs_submit_merged_write_folio(sbi, folio, type);
4360 /* submit cached IPU IO */
4361 f2fs_submit_merged_ipu_write(sbi, NULL, folio);
4362 if (ordered) {
4363 folio_wait_writeback(folio);
4364 f2fs_bug_on(sbi, locked && folio_test_writeback(folio));
4365 } else {
4366 folio_wait_stable(folio);
4367 }
4368 }
4369}
4370
4371void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr)
4372{
4373 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
4374 struct folio *cfolio;
4375
4376 if (!f2fs_meta_inode_gc_required(inode))
4377 return;
4378
4379 if (!__is_valid_data_blkaddr(blkaddr))
4380 return;
4381
4382 cfolio = filemap_lock_folio(META_MAPPING(sbi), blkaddr);
4383 if (!IS_ERR(cfolio)) {
4384 f2fs_folio_wait_writeback(cfolio, DATA, true, true);
4385 f2fs_folio_put(cfolio, true);
4386 }
4387}
4388
4389void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
4390 block_t len)
4391{
4392 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
4393 block_t i;
4394
4395 if (!f2fs_meta_inode_gc_required(inode))
4396 return;
4397
4398 for (i = 0; i < len; i++)
4399 f2fs_wait_on_block_writeback(inode, blkaddr + i);
4400
4401 f2fs_truncate_meta_inode_pages(sbi, blkaddr, len);
4402}
4403
4404static int read_compacted_summaries(struct f2fs_sb_info *sbi)
4405{
4406 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
4407 struct curseg_info *seg_i;
4408 unsigned char *kaddr;
4409 struct folio *folio;
4410 block_t start;
4411 int i, j, offset;
4412
4413 start = start_sum_block(sbi);
4414
4415 folio = f2fs_get_meta_folio(sbi, start++);
4416 if (IS_ERR(folio))
4417 return PTR_ERR(folio);
4418 kaddr = folio_address(folio);
4419
4420 /* Step 1: restore nat cache */
4421 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
4422 memcpy(seg_i->journal, kaddr, sbi->sum_journal_size);
4423
4424 /* Step 2: restore sit cache */
4425 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
4426 memcpy(seg_i->journal, kaddr + sbi->sum_journal_size, sbi->sum_journal_size);
4427 offset = 2 * sbi->sum_journal_size;
4428
4429 /* Step 3: restore summary entries */
4430 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
4431 unsigned short blk_off;
4432 unsigned int segno;
4433
4434 seg_i = CURSEG_I(sbi, i);
4435 segno = le32_to_cpu(ckpt->cur_data_segno[i]);
4436 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
4437 seg_i->next_segno = segno;
4438 reset_curseg(sbi, i, 0);
4439 seg_i->alloc_type = ckpt->alloc_type[i];
4440 seg_i->next_blkoff = blk_off;
4441
4442 if (seg_i->alloc_type == SSR)
4443 blk_off = BLKS_PER_SEG(sbi);
4444
4445 for (j = 0; j < blk_off; j++) {
4446 struct f2fs_summary *s;
4447
4448 s = (struct f2fs_summary *)(kaddr + offset);
4449 sum_entries(seg_i->sum_blk)[j] = *s;
4450 offset += SUMMARY_SIZE;
4451 if (offset + SUMMARY_SIZE <= sbi->blocksize -
4452 SUM_FOOTER_SIZE)
4453 continue;
4454
4455 f2fs_folio_put(folio, true);
4456
4457 folio = f2fs_get_meta_folio(sbi, start++);
4458 if (IS_ERR(folio))
4459 return PTR_ERR(folio);
4460 kaddr = folio_address(folio);
4461 offset = 0;
4462 }
4463 }
4464 f2fs_folio_put(folio, true);
4465 return 0;
4466}
4467
4468static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
4469{
4470 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
4471 struct f2fs_summary_block *sum;
4472 struct curseg_info *curseg;
4473 struct folio *new;
4474 unsigned short blk_off;
4475 unsigned int segno = 0;
4476 block_t blk_addr = 0;
4477 int err = 0;
4478
4479 /* get segment number and block addr */
4480 if (IS_DATASEG(type)) {
4481 segno = le32_to_cpu(ckpt->cur_data_segno[type]);
4482 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
4483 CURSEG_HOT_DATA]);
4484 if (__exist_node_summaries(sbi))
4485 blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type);
4486 else
4487 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
4488 } else {
4489 segno = le32_to_cpu(ckpt->cur_node_segno[type -
4490 CURSEG_HOT_NODE]);
4491 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
4492 CURSEG_HOT_NODE]);
4493 if (__exist_node_summaries(sbi))
4494 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
4495 type - CURSEG_HOT_NODE);
4496 else
4497 blk_addr = GET_SUM_BLOCK(sbi, segno);
4498 }
4499
4500 new = f2fs_get_meta_folio(sbi, blk_addr);
4501 if (IS_ERR(new))
4502 return PTR_ERR(new);
4503 sum = folio_address(new);
4504
4505 if (IS_NODESEG(type)) {
4506 if (__exist_node_summaries(sbi)) {
4507 struct f2fs_summary *ns = sum_entries(sum);
4508 int i;
4509
4510 for (i = 0; i < BLKS_PER_SEG(sbi); i++, ns++) {
4511 ns->version = 0;
4512 ns->ofs_in_node = 0;
4513 }
4514 } else {
4515 err = f2fs_restore_node_summary(sbi, segno, sum);
4516 if (err)
4517 goto out;
4518 }
4519 }
4520
4521 /* set uncompleted segment to curseg */
4522 curseg = CURSEG_I(sbi, type);
4523 mutex_lock(&curseg->curseg_mutex);
4524
4525 /* update journal info */
4526 down_write(&curseg->journal_rwsem);
4527 memcpy(curseg->journal, sum_journal(sbi, sum), sbi->sum_journal_size);
4528 up_write(&curseg->journal_rwsem);
4529
4530 memcpy(sum_entries(curseg->sum_blk), sum_entries(sum),
4531 sbi->sum_entry_size);
4532 memcpy(sum_footer(sbi, curseg->sum_blk), sum_footer(sbi, sum),
4533 SUM_FOOTER_SIZE);
4534 curseg->next_segno = segno;
4535 reset_curseg(sbi, type, 0);
4536 curseg->alloc_type = ckpt->alloc_type[type];
4537 curseg->next_blkoff = blk_off;
4538 mutex_unlock(&curseg->curseg_mutex);
4539out:
4540 f2fs_folio_put(new, true);
4541 return err;
4542}
4543
4544static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
4545{
4546 struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal;
4547 struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal;
4548 int type = CURSEG_HOT_DATA;
4549 int err;
4550
4551 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) {
4552 int npages = f2fs_npages_for_summary_flush(sbi, true);
4553
4554 if (npages >= 2)
4555 f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages,
4556 META_CP, true);
4557
4558 /* restore for compacted data summary */
4559 err = read_compacted_summaries(sbi);
4560 if (err)
4561 return err;
4562 type = CURSEG_HOT_NODE;
4563 }
4564
4565 if (__exist_node_summaries(sbi))
4566 f2fs_ra_meta_pages(sbi,
4567 sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type),
4568 NR_CURSEG_PERSIST_TYPE - type, META_CP, true);
4569
4570 for (; type <= CURSEG_COLD_NODE; type++) {
4571 err = read_normal_summaries(sbi, type);
4572 if (err)
4573 return err;
4574 }
4575
4576 /* sanity check for summary blocks */
4577 if (nats_in_cursum(nat_j) > sbi->nat_journal_entries ||
4578 sits_in_cursum(sit_j) > sbi->sit_journal_entries) {
4579 f2fs_err(sbi, "invalid journal entries nats %u sits %u",
4580 nats_in_cursum(nat_j), sits_in_cursum(sit_j));
4581 return -EINVAL;
4582 }
4583
4584 return 0;
4585}
4586
4587static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
4588{
4589 struct folio *folio;
4590 unsigned char *kaddr;
4591 struct f2fs_summary *summary;
4592 struct curseg_info *seg_i;
4593 int written_size = 0;
4594 int i, j;
4595
4596 folio = f2fs_grab_meta_folio(sbi, blkaddr++);
4597 kaddr = folio_address(folio);
4598 memset(kaddr, 0, PAGE_SIZE);
4599
4600 /* Step 1: write nat cache */
4601 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
4602 memcpy(kaddr, seg_i->journal, sbi->sum_journal_size);
4603 written_size += sbi->sum_journal_size;
4604
4605 /* Step 2: write sit cache */
4606 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
4607 memcpy(kaddr + written_size, seg_i->journal, sbi->sum_journal_size);
4608 written_size += sbi->sum_journal_size;
4609
4610 /* Step 3: write summary entries */
4611 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
4612 seg_i = CURSEG_I(sbi, i);
4613 for (j = 0; j < f2fs_curseg_valid_blocks(sbi, i); j++) {
4614 if (!folio) {
4615 folio = f2fs_grab_meta_folio(sbi, blkaddr++);
4616 kaddr = folio_address(folio);
4617 memset(kaddr, 0, PAGE_SIZE);
4618 written_size = 0;
4619 }
4620 summary = (struct f2fs_summary *)(kaddr + written_size);
4621 *summary = sum_entries(seg_i->sum_blk)[j];
4622 written_size += SUMMARY_SIZE;
4623
4624 if (written_size + SUMMARY_SIZE <= sbi->blocksize -
4625 SUM_FOOTER_SIZE)
4626 continue;
4627
4628 folio_mark_dirty(folio);
4629 f2fs_folio_put(folio, true);
4630 folio = NULL;
4631 }
4632 }
4633 if (folio) {
4634 folio_mark_dirty(folio);
4635 f2fs_folio_put(folio, true);
4636 }
4637}
4638
4639static void write_normal_summaries(struct f2fs_sb_info *sbi,
4640 block_t blkaddr, int type)
4641{
4642 int i, end;
4643
4644 if (IS_DATASEG(type))
4645 end = type + NR_CURSEG_DATA_TYPE;
4646 else
4647 end = type + NR_CURSEG_NODE_TYPE;
4648
4649 for (i = type; i < end; i++)
4650 write_current_sum_page(sbi, i, blkaddr + (i - type));
4651}
4652
4653void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
4654{
4655 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG))
4656 write_compacted_summaries(sbi, start_blk);
4657 else
4658 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
4659}
4660
4661void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
4662{
4663 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
4664}
4665
4666int f2fs_lookup_journal_in_cursum(struct f2fs_sb_info *sbi,
4667 struct f2fs_journal *journal, int type,
4668 unsigned int val, int alloc)
4669{
4670 int i;
4671
4672 if (type == NAT_JOURNAL) {
4673 for (i = 0; i < nats_in_cursum(journal); i++) {
4674 if (le32_to_cpu(nid_in_journal(journal, i)) == val)
4675 return i;
4676 }
4677 if (alloc && __has_cursum_space(sbi, journal, 1, NAT_JOURNAL))
4678 return update_nats_in_cursum(journal, 1);
4679 } else if (type == SIT_JOURNAL) {
4680 for (i = 0; i < sits_in_cursum(journal); i++)
4681 if (le32_to_cpu(segno_in_journal(journal, i)) == val)
4682 return i;
4683 if (alloc && __has_cursum_space(sbi, journal, 1, SIT_JOURNAL))
4684 return update_sits_in_cursum(journal, 1);
4685 }
4686 return -1;
4687}
4688
4689static struct folio *get_current_sit_folio(struct f2fs_sb_info *sbi,
4690 unsigned int segno)
4691{
4692 return f2fs_get_meta_folio(sbi, current_sit_addr(sbi, segno));
4693}
4694
4695static struct folio *get_next_sit_folio(struct f2fs_sb_info *sbi,
4696 unsigned int start)
4697{
4698 struct sit_info *sit_i = SIT_I(sbi);
4699 struct folio *folio;
4700 pgoff_t src_off, dst_off;
4701
4702 src_off = current_sit_addr(sbi, start);
4703 dst_off = next_sit_addr(sbi, src_off);
4704
4705 folio = f2fs_grab_meta_folio(sbi, dst_off);
4706 seg_info_to_sit_folio(sbi, folio, start);
4707
4708 folio_mark_dirty(folio);
4709 set_to_next_sit(sit_i, start);
4710
4711 return folio;
4712}
4713
4714static struct sit_entry_set *grab_sit_entry_set(void)
4715{
4716 struct sit_entry_set *ses =
4717 f2fs_kmem_cache_alloc(sit_entry_set_slab,
4718 GFP_NOFS, true, NULL);
4719
4720 ses->entry_cnt = 0;
4721 INIT_LIST_HEAD(&ses->set_list);
4722 return ses;
4723}
4724
4725static void release_sit_entry_set(struct sit_entry_set *ses)
4726{
4727 list_del(&ses->set_list);
4728 kmem_cache_free(sit_entry_set_slab, ses);
4729}
4730
4731static void adjust_sit_entry_set(struct sit_entry_set *ses,
4732 struct list_head *head)
4733{
4734 struct sit_entry_set *next = ses;
4735
4736 if (list_is_last(&ses->set_list, head))
4737 return;
4738
4739 list_for_each_entry_continue(next, head, set_list)
4740 if (ses->entry_cnt <= next->entry_cnt) {
4741 list_move_tail(&ses->set_list, &next->set_list);
4742 return;
4743 }
4744
4745 list_move_tail(&ses->set_list, head);
4746}
4747
4748static void add_sit_entry(unsigned int segno, struct list_head *head)
4749{
4750 struct sit_entry_set *ses;
4751 unsigned int start_segno = START_SEGNO(segno);
4752
4753 list_for_each_entry(ses, head, set_list) {
4754 if (ses->start_segno == start_segno) {
4755 ses->entry_cnt++;
4756 adjust_sit_entry_set(ses, head);
4757 return;
4758 }
4759 }
4760
4761 ses = grab_sit_entry_set();
4762
4763 ses->start_segno = start_segno;
4764 ses->entry_cnt++;
4765 list_add(&ses->set_list, head);
4766}
4767
4768static void add_sits_in_set(struct f2fs_sb_info *sbi)
4769{
4770 struct f2fs_sm_info *sm_info = SM_I(sbi);
4771 struct list_head *set_list = &sm_info->sit_entry_set;
4772 unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;
4773 unsigned int segno;
4774
4775 for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
4776 add_sit_entry(segno, set_list);
4777}
4778
4779static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
4780{
4781 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4782 struct f2fs_journal *journal = curseg->journal;
4783 int i;
4784
4785 down_write(&curseg->journal_rwsem);
4786 for (i = 0; i < sits_in_cursum(journal); i++) {
4787 unsigned int segno;
4788 bool dirtied;
4789
4790 segno = le32_to_cpu(segno_in_journal(journal, i));
4791 dirtied = __mark_sit_entry_dirty(sbi, segno);
4792
4793 if (!dirtied)
4794 add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
4795 }
4796 update_sits_in_cursum(journal, -i);
4797 up_write(&curseg->journal_rwsem);
4798}
4799
4800/*
4801 * CP calls this function, which flushes SIT entries including sit_journal,
4802 * and moves prefree segs to free segs.
4803 */
4804void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
4805{
4806 struct sit_info *sit_i = SIT_I(sbi);
4807 unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
4808 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4809 struct f2fs_journal *journal = curseg->journal;
4810 struct sit_entry_set *ses, *tmp;
4811 struct list_head *head = &SM_I(sbi)->sit_entry_set;
4812 bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS);
4813 struct seg_entry *se;
4814
4815 down_write(&sit_i->sentry_lock);
4816
4817 if (!sit_i->dirty_sentries)
4818 goto out;
4819
4820 /*
4821 * add and account sit entries of dirty bitmap in sit entry
4822 * set temporarily
4823 */
4824 add_sits_in_set(sbi);
4825
4826 /*
4827 * if there are no enough space in journal to store dirty sit
4828 * entries, remove all entries from journal and add and account
4829 * them in sit entry set.
4830 */
4831 if (!__has_cursum_space(sbi, journal,
4832 sit_i->dirty_sentries, SIT_JOURNAL) || !to_journal)
4833 remove_sits_in_journal(sbi);
4834
4835 /*
4836 * there are two steps to flush sit entries:
4837 * #1, flush sit entries to journal in current cold data summary block.
4838 * #2, flush sit entries to sit page.
4839 */
4840 list_for_each_entry_safe(ses, tmp, head, set_list) {
4841 struct folio *folio = NULL;
4842 struct f2fs_sit_block *raw_sit = NULL;
4843 unsigned int start_segno = ses->start_segno;
4844 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
4845 (unsigned long)MAIN_SEGS(sbi));
4846 unsigned int segno = start_segno;
4847
4848 if (to_journal &&
4849 !__has_cursum_space(sbi, journal, ses->entry_cnt,
4850 SIT_JOURNAL))
4851 to_journal = false;
4852
4853 if (to_journal) {
4854 down_write(&curseg->journal_rwsem);
4855 } else {
4856 folio = get_next_sit_folio(sbi, start_segno);
4857 raw_sit = folio_address(folio);
4858 }
4859
4860 /* flush dirty sit entries in region of current sit set */
4861 for_each_set_bit_from(segno, bitmap, end) {
4862 int offset, sit_offset;
4863
4864 se = get_seg_entry(sbi, segno);
4865
4866 /* add discard candidates */
4867 if (!(cpc->reason & CP_DISCARD)) {
4868 cpc->trim_start = segno;
4869 add_discard_addrs(sbi, cpc, false);
4870 }
4871
4872 if (to_journal) {
4873 offset = f2fs_lookup_journal_in_cursum(sbi, journal,
4874 SIT_JOURNAL, segno, 1);
4875 f2fs_bug_on(sbi, offset < 0);
4876 segno_in_journal(journal, offset) =
4877 cpu_to_le32(segno);
4878 seg_info_to_raw_sit(se,
4879 &sit_in_journal(journal, offset));
4880 check_block_count(sbi, segno,
4881 &sit_in_journal(journal, offset));
4882 } else {
4883 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
4884 seg_info_to_raw_sit(se,
4885 &raw_sit->entries[sit_offset]);
4886 check_block_count(sbi, segno,
4887 &raw_sit->entries[sit_offset]);
4888 }
4889
4890 /* update ckpt_valid_block */
4891 if (__is_large_section(sbi))
4892 set_ckpt_valid_blocks(sbi, segno);
4893
4894 __clear_bit(segno, bitmap);
4895 sit_i->dirty_sentries--;
4896 ses->entry_cnt--;
4897 }
4898
4899 if (to_journal)
4900 up_write(&curseg->journal_rwsem);
4901 else
4902 f2fs_folio_put(folio, true);
4903
4904 f2fs_bug_on(sbi, ses->entry_cnt);
4905 release_sit_entry_set(ses);
4906 }
4907
4908 f2fs_bug_on(sbi, !list_empty(head));
4909 f2fs_bug_on(sbi, sit_i->dirty_sentries);
4910out:
4911 if (cpc->reason & CP_DISCARD) {
4912 __u64 trim_start = cpc->trim_start;
4913
4914 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++)
4915 add_discard_addrs(sbi, cpc, false);
4916
4917 cpc->trim_start = trim_start;
4918 }
4919 up_write(&sit_i->sentry_lock);
4920
4921 set_prefree_as_free_segments(sbi);
4922}
4923
4924static int build_sit_info(struct f2fs_sb_info *sbi)
4925{
4926 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
4927 struct sit_info *sit_i;
4928 unsigned int sit_segs, start;
4929 char *src_bitmap, *bitmap;
4930 unsigned int bitmap_size, main_bitmap_size, sit_bitmap_size;
4931 unsigned int discard_map = f2fs_block_unit_discard(sbi) ? 1 : 0;
4932
4933 /* allocate memory for SIT information */
4934 sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL);
4935 if (!sit_i)
4936 return -ENOMEM;
4937
4938 SM_I(sbi)->sit_info = sit_i;
4939
4940 sit_i->sentries =
4941 f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry),
4942 MAIN_SEGS(sbi)),
4943 GFP_KERNEL);
4944 if (!sit_i->sentries)
4945 return -ENOMEM;
4946
4947 main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4948 sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size,
4949 GFP_KERNEL);
4950 if (!sit_i->dirty_sentries_bitmap)
4951 return -ENOMEM;
4952
4953 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (2 + discard_map);
4954 sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4955 if (!sit_i->bitmap)
4956 return -ENOMEM;
4957
4958 bitmap = sit_i->bitmap;
4959
4960 for (start = 0; start < MAIN_SEGS(sbi); start++) {
4961 sit_i->sentries[start].cur_valid_map = bitmap;
4962 bitmap += SIT_VBLOCK_MAP_SIZE;
4963
4964 sit_i->sentries[start].ckpt_valid_map = bitmap;
4965 bitmap += SIT_VBLOCK_MAP_SIZE;
4966
4967 if (discard_map) {
4968 sit_i->sentries[start].discard_map = bitmap;
4969 bitmap += SIT_VBLOCK_MAP_SIZE;
4970 }
4971 }
4972
4973 sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
4974 if (!sit_i->tmp_map)
4975 return -ENOMEM;
4976
4977 if (__is_large_section(sbi)) {
4978 sit_i->sec_entries =
4979 f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry),
4980 MAIN_SECS(sbi)),
4981 GFP_KERNEL);
4982 if (!sit_i->sec_entries)
4983 return -ENOMEM;
4984 }
4985
4986 /* get information related with SIT */
4987 sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
4988
4989 /* setup SIT bitmap from ckeckpoint pack */
4990 sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
4991 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
4992
4993 sit_i->sit_bitmap = kmemdup(src_bitmap, sit_bitmap_size, GFP_KERNEL);
4994 if (!sit_i->sit_bitmap)
4995 return -ENOMEM;
4996
4997#ifdef CONFIG_F2FS_CHECK_FS
4998 sit_i->invalid_segmap = f2fs_kvzalloc(sbi,
4999 main_bitmap_size, GFP_KERNEL);
5000 if (!sit_i->invalid_segmap)
5001 return -ENOMEM;
5002#endif
5003
5004 sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
5005 sit_i->sit_blocks = SEGS_TO_BLKS(sbi, sit_segs);
5006 sit_i->written_valid_blocks = 0;
5007 sit_i->bitmap_size = sit_bitmap_size;
5008 sit_i->dirty_sentries = 0;
5009 sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
5010 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
5011 sit_i->mounted_time = ktime_get_boottime_seconds();
5012 init_rwsem(&sit_i->sentry_lock);
5013 return 0;
5014}
5015
5016static int build_free_segmap(struct f2fs_sb_info *sbi)
5017{
5018 struct free_segmap_info *free_i;
5019 unsigned int bitmap_size, sec_bitmap_size;
5020
5021 /* allocate memory for free segmap information */
5022 free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL);
5023 if (!free_i)
5024 return -ENOMEM;
5025
5026 SM_I(sbi)->free_info = free_i;
5027
5028 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
5029 free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL);
5030 if (!free_i->free_segmap)
5031 return -ENOMEM;
5032
5033 sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
5034 free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL);
5035 if (!free_i->free_secmap)
5036 return -ENOMEM;
5037
5038 /* set all segments as dirty temporarily */
5039 memset(free_i->free_segmap, 0xff, bitmap_size);
5040 memset(free_i->free_secmap, 0xff, sec_bitmap_size);
5041
5042 /* init free segmap information */
5043 free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));
5044 free_i->free_segments = 0;
5045 free_i->free_sections = 0;
5046 spin_lock_init(&free_i->segmap_lock);
5047 return 0;
5048}
5049
5050static int build_curseg(struct f2fs_sb_info *sbi)
5051{
5052 struct curseg_info *array;
5053 int i;
5054
5055 array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE,
5056 sizeof(*array)), GFP_KERNEL);
5057 if (!array)
5058 return -ENOMEM;
5059
5060 SM_I(sbi)->curseg_array = array;
5061
5062 for (i = 0; i < NO_CHECK_TYPE; i++) {
5063 mutex_init(&array[i].curseg_mutex);
5064 array[i].sum_blk = f2fs_kzalloc(sbi, sbi->sum_blocksize,
5065 GFP_KERNEL);
5066 if (!array[i].sum_blk)
5067 return -ENOMEM;
5068 init_rwsem(&array[i].journal_rwsem);
5069 array[i].journal = f2fs_kzalloc(sbi,
5070 sbi->sum_journal_size, GFP_KERNEL);
5071 if (!array[i].journal)
5072 return -ENOMEM;
5073 array[i].seg_type = log_type_to_seg_type(i);
5074 reset_curseg_fields(&array[i]);
5075 }
5076 return restore_curseg_summaries(sbi);
5077}
5078
5079static int build_sit_entries(struct f2fs_sb_info *sbi)
5080{
5081 struct sit_info *sit_i = SIT_I(sbi);
5082 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
5083 struct f2fs_journal *journal = curseg->journal;
5084 struct seg_entry *se;
5085 struct f2fs_sit_entry sit;
5086 int sit_blk_cnt = SIT_BLK_CNT(sbi);
5087 unsigned int i, start, end;
5088 unsigned int readed, start_blk = 0;
5089 int err = 0;
5090 block_t sit_valid_blocks[2] = {0, 0};
5091
5092 do {
5093 readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_VECS,
5094 META_SIT, true);
5095
5096 start = start_blk * sit_i->sents_per_block;
5097 end = (start_blk + readed) * sit_i->sents_per_block;
5098
5099 for (; start < end && start < MAIN_SEGS(sbi); start++) {
5100 struct f2fs_sit_block *sit_blk;
5101 struct folio *folio;
5102
5103 se = &sit_i->sentries[start];
5104 folio = get_current_sit_folio(sbi, start);
5105 if (IS_ERR(folio))
5106 return PTR_ERR(folio);
5107 sit_blk = folio_address(folio);
5108 sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
5109 f2fs_folio_put(folio, true);
5110
5111 err = check_block_count(sbi, start, &sit);
5112 if (err)
5113 return err;
5114 seg_info_from_raw_sit(se, &sit);
5115
5116 if (se->type >= NR_PERSISTENT_LOG) {
5117 f2fs_err(sbi, "Invalid segment type: %u, segno: %u",
5118 se->type, start);
5119 f2fs_handle_error(sbi,
5120 ERROR_INCONSISTENT_SUM_TYPE);
5121 return -EFSCORRUPTED;
5122 }
5123
5124 sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks;
5125
5126 if (!f2fs_block_unit_discard(sbi))
5127 goto init_discard_map_done;
5128
5129 /* build discard map only one time */
5130 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
5131 memset(se->discard_map, 0xff,
5132 SIT_VBLOCK_MAP_SIZE);
5133 goto init_discard_map_done;
5134 }
5135 memcpy(se->discard_map, se->cur_valid_map,
5136 SIT_VBLOCK_MAP_SIZE);
5137 sbi->discard_blks += BLKS_PER_SEG(sbi) -
5138 se->valid_blocks;
5139init_discard_map_done:
5140 if (__is_large_section(sbi))
5141 get_sec_entry(sbi, start)->valid_blocks +=
5142 se->valid_blocks;
5143 }
5144 start_blk += readed;
5145 } while (start_blk < sit_blk_cnt);
5146
5147 down_read(&curseg->journal_rwsem);
5148 for (i = 0; i < sits_in_cursum(journal); i++) {
5149 unsigned int old_valid_blocks;
5150
5151 start = le32_to_cpu(segno_in_journal(journal, i));
5152 if (start >= MAIN_SEGS(sbi)) {
5153 f2fs_err(sbi, "Wrong journal entry on segno %u",
5154 start);
5155 err = -EFSCORRUPTED;
5156 f2fs_handle_error(sbi, ERROR_CORRUPTED_JOURNAL);
5157 break;
5158 }
5159
5160 se = &sit_i->sentries[start];
5161 sit = sit_in_journal(journal, i);
5162
5163 old_valid_blocks = se->valid_blocks;
5164
5165 sit_valid_blocks[SE_PAGETYPE(se)] -= old_valid_blocks;
5166
5167 err = check_block_count(sbi, start, &sit);
5168 if (err)
5169 break;
5170 seg_info_from_raw_sit(se, &sit);
5171
5172 if (se->type >= NR_PERSISTENT_LOG) {
5173 f2fs_err(sbi, "Invalid segment type: %u, segno: %u",
5174 se->type, start);
5175 err = -EFSCORRUPTED;
5176 f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE);
5177 break;
5178 }
5179
5180 sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks;
5181
5182 if (f2fs_block_unit_discard(sbi)) {
5183 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
5184 memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE);
5185 } else {
5186 memcpy(se->discard_map, se->cur_valid_map,
5187 SIT_VBLOCK_MAP_SIZE);
5188 sbi->discard_blks += old_valid_blocks;
5189 sbi->discard_blks -= se->valid_blocks;
5190 }
5191 }
5192
5193 if (__is_large_section(sbi)) {
5194 get_sec_entry(sbi, start)->valid_blocks +=
5195 se->valid_blocks;
5196 get_sec_entry(sbi, start)->valid_blocks -=
5197 old_valid_blocks;
5198 }
5199 }
5200 up_read(&curseg->journal_rwsem);
5201
5202 /* update ckpt_valid_block */
5203 if (__is_large_section(sbi)) {
5204 unsigned int segno;
5205
5206 for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi))
5207 set_ckpt_valid_blocks(sbi, segno);
5208 }
5209
5210 if (err)
5211 return err;
5212
5213 if (sit_valid_blocks[NODE] != valid_node_count(sbi)) {
5214 f2fs_err(sbi, "SIT is corrupted node# %u vs %u",
5215 sit_valid_blocks[NODE], valid_node_count(sbi));
5216 f2fs_handle_error(sbi, ERROR_INCONSISTENT_NODE_COUNT);
5217 return -EFSCORRUPTED;
5218 }
5219
5220 if (sit_valid_blocks[DATA] + sit_valid_blocks[NODE] >
5221 valid_user_blocks(sbi)) {
5222 f2fs_err(sbi, "SIT is corrupted data# %u %u vs %u",
5223 sit_valid_blocks[DATA], sit_valid_blocks[NODE],
5224 valid_user_blocks(sbi));
5225 f2fs_handle_error(sbi, ERROR_INCONSISTENT_BLOCK_COUNT);
5226 return -EFSCORRUPTED;
5227 }
5228
5229 return 0;
5230}
5231
5232static void init_free_segmap(struct f2fs_sb_info *sbi)
5233{
5234 unsigned int start;
5235 int type;
5236 struct seg_entry *sentry;
5237
5238 for (start = 0; start < MAIN_SEGS(sbi); start++) {
5239 if (f2fs_usable_blks_in_seg(sbi, start) == 0)
5240 continue;
5241 sentry = get_seg_entry(sbi, start);
5242 if (!sentry->valid_blocks)
5243 __set_free(sbi, start);
5244 else
5245 SIT_I(sbi)->written_valid_blocks +=
5246 sentry->valid_blocks;
5247 }
5248
5249 /* set use the current segments */
5250 for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
5251 struct curseg_info *curseg_t = CURSEG_I(sbi, type);
5252
5253 __set_test_and_inuse(sbi, curseg_t->segno);
5254 }
5255}
5256
5257static void init_dirty_segmap(struct f2fs_sb_info *sbi)
5258{
5259 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5260 struct free_segmap_info *free_i = FREE_I(sbi);
5261 unsigned int segno = 0, offset = 0, secno;
5262 block_t valid_blocks, usable_blks_in_seg;
5263
5264 while (1) {
5265 /* find dirty segment based on free segmap */
5266 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);
5267 if (segno >= MAIN_SEGS(sbi))
5268 break;
5269 offset = segno + 1;
5270 valid_blocks = get_valid_blocks(sbi, segno, false);
5271 usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno);
5272 if (valid_blocks == usable_blks_in_seg || !valid_blocks)
5273 continue;
5274 if (valid_blocks > usable_blks_in_seg) {
5275 f2fs_bug_on(sbi, 1);
5276 continue;
5277 }
5278 mutex_lock(&dirty_i->seglist_lock);
5279 __locate_dirty_segment(sbi, segno, DIRTY);
5280 mutex_unlock(&dirty_i->seglist_lock);
5281 }
5282
5283 if (!__is_large_section(sbi))
5284 return;
5285
5286 mutex_lock(&dirty_i->seglist_lock);
5287 for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi)) {
5288 valid_blocks = get_valid_blocks(sbi, segno, true);
5289 secno = GET_SEC_FROM_SEG(sbi, segno);
5290
5291 if (!valid_blocks || valid_blocks == CAP_BLKS_PER_SEC(sbi))
5292 continue;
5293 if (is_cursec(sbi, secno))
5294 continue;
5295 set_bit(secno, dirty_i->dirty_secmap);
5296 }
5297 mutex_unlock(&dirty_i->seglist_lock);
5298}
5299
5300static int init_victim_secmap(struct f2fs_sb_info *sbi)
5301{
5302 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5303 unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
5304
5305 dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
5306 if (!dirty_i->victim_secmap)
5307 return -ENOMEM;
5308
5309 dirty_i->pinned_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
5310 if (!dirty_i->pinned_secmap)
5311 return -ENOMEM;
5312
5313 dirty_i->pinned_secmap_cnt = 0;
5314 dirty_i->enable_pin_section = true;
5315 return 0;
5316}
5317
5318static int build_dirty_segmap(struct f2fs_sb_info *sbi)
5319{
5320 struct dirty_seglist_info *dirty_i;
5321 unsigned int bitmap_size, i;
5322
5323 /* allocate memory for dirty segments list information */
5324 dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info),
5325 GFP_KERNEL);
5326 if (!dirty_i)
5327 return -ENOMEM;
5328
5329 SM_I(sbi)->dirty_info = dirty_i;
5330 mutex_init(&dirty_i->seglist_lock);
5331
5332 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
5333
5334 for (i = 0; i < NR_DIRTY_TYPE; i++) {
5335 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size,
5336 GFP_KERNEL);
5337 if (!dirty_i->dirty_segmap[i])
5338 return -ENOMEM;
5339 }
5340
5341 if (__is_large_section(sbi)) {
5342 bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
5343 dirty_i->dirty_secmap = f2fs_kvzalloc(sbi,
5344 bitmap_size, GFP_KERNEL);
5345 if (!dirty_i->dirty_secmap)
5346 return -ENOMEM;
5347 }
5348
5349 init_dirty_segmap(sbi);
5350 return init_victim_secmap(sbi);
5351}
5352
5353static int sanity_check_curseg(struct f2fs_sb_info *sbi)
5354{
5355 int i;
5356
5357 /*
5358 * In LFS/SSR curseg, .next_blkoff should point to an unused blkaddr;
5359 * In LFS curseg, all blkaddr after .next_blkoff should be unused.
5360 */
5361 for (i = 0; i < NR_PERSISTENT_LOG; i++) {
5362 struct curseg_info *curseg = CURSEG_I(sbi, i);
5363 struct seg_entry *se = get_seg_entry(sbi, curseg->segno);
5364 unsigned int blkofs = curseg->next_blkoff;
5365
5366 if (f2fs_sb_has_readonly(sbi) &&
5367 i != CURSEG_HOT_DATA && i != CURSEG_HOT_NODE)
5368 continue;
5369
5370 sanity_check_seg_type(sbi, curseg->seg_type);
5371
5372 if (curseg->alloc_type != LFS && curseg->alloc_type != SSR) {
5373 f2fs_err(sbi,
5374 "Current segment has invalid alloc_type:%d",
5375 curseg->alloc_type);
5376 f2fs_handle_error(sbi, ERROR_INVALID_CURSEG);
5377 return -EFSCORRUPTED;
5378 }
5379
5380 if (f2fs_test_bit(blkofs, se->cur_valid_map))
5381 goto out;
5382
5383 if (curseg->alloc_type == SSR)
5384 continue;
5385
5386 for (blkofs += 1; blkofs < BLKS_PER_SEG(sbi); blkofs++) {
5387 if (!f2fs_test_bit(blkofs, se->cur_valid_map))
5388 continue;
5389out:
5390 f2fs_err(sbi,
5391 "Current segment's next free block offset is inconsistent with bitmap, logtype:%u, segno:%u, type:%u, next_blkoff:%u, blkofs:%u",
5392 i, curseg->segno, curseg->alloc_type,
5393 curseg->next_blkoff, blkofs);
5394 f2fs_handle_error(sbi, ERROR_INVALID_CURSEG);
5395 return -EFSCORRUPTED;
5396 }
5397 }
5398 return 0;
5399}
5400
5401#ifdef CONFIG_BLK_DEV_ZONED
5402static int check_zone_write_pointer(struct f2fs_sb_info *sbi,
5403 struct f2fs_dev_info *fdev,
5404 struct blk_zone *zone)
5405{
5406 unsigned int zone_segno;
5407 block_t zone_block, valid_block_cnt;
5408 unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
5409 int ret;
5410 unsigned int nofs_flags;
5411
5412 if (zone->type != BLK_ZONE_TYPE_SEQWRITE_REQ)
5413 return 0;
5414
5415 zone_block = fdev->start_blk + (zone->start >> log_sectors_per_block);
5416 zone_segno = GET_SEGNO(sbi, zone_block);
5417
5418 /*
5419 * Skip check of zones cursegs point to, since
5420 * fix_curseg_write_pointer() checks them.
5421 */
5422 if (zone_segno >= MAIN_SEGS(sbi))
5423 return 0;
5424
5425 /*
5426 * Get # of valid block of the zone.
5427 */
5428 valid_block_cnt = get_valid_blocks(sbi, zone_segno, true);
5429 if (is_cursec(sbi, GET_SEC_FROM_SEG(sbi, zone_segno))) {
5430 f2fs_notice(sbi, "Open zones: valid block[0x%x,0x%x] cond[%s]",
5431 zone_segno, valid_block_cnt,
5432 blk_zone_cond_str(zone->cond));
5433 return 0;
5434 }
5435
5436 if ((!valid_block_cnt && zone->cond == BLK_ZONE_COND_EMPTY) ||
5437 (valid_block_cnt && zone->cond == BLK_ZONE_COND_FULL))
5438 return 0;
5439
5440 if (!valid_block_cnt) {
5441 f2fs_notice(sbi, "Zone without valid block has non-zero write "
5442 "pointer. Reset the write pointer: cond[%s]",
5443 blk_zone_cond_str(zone->cond));
5444 ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block,
5445 zone->len >> log_sectors_per_block);
5446 if (ret)
5447 f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
5448 fdev->path, ret);
5449 return ret;
5450 }
5451
5452 /*
5453 * If there are valid blocks and the write pointer doesn't match
5454 * with them, we need to report the inconsistency and fill
5455 * the zone till the end to close the zone. This inconsistency
5456 * does not cause write error because the zone will not be
5457 * selected for write operation until it get discarded.
5458 */
5459 f2fs_notice(sbi, "Valid blocks are not aligned with write "
5460 "pointer: valid block[0x%x,0x%x] cond[%s]",
5461 zone_segno, valid_block_cnt, blk_zone_cond_str(zone->cond));
5462
5463 nofs_flags = memalloc_nofs_save();
5464 ret = blkdev_zone_mgmt(fdev->bdev, REQ_OP_ZONE_FINISH,
5465 zone->start, zone->len);
5466 memalloc_nofs_restore(nofs_flags);
5467 if (ret == -EOPNOTSUPP) {
5468 ret = blkdev_issue_zeroout(fdev->bdev, zone->wp,
5469 zone->len - (zone->wp - zone->start),
5470 GFP_NOFS, 0);
5471 if (ret)
5472 f2fs_err(sbi, "Fill up zone failed: %s (errno=%d)",
5473 fdev->path, ret);
5474 } else if (ret) {
5475 f2fs_err(sbi, "Finishing zone failed: %s (errno=%d)",
5476 fdev->path, ret);
5477 }
5478
5479 return ret;
5480}
5481
5482static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi,
5483 block_t zone_blkaddr)
5484{
5485 int i;
5486
5487 for (i = 0; i < sbi->s_ndevs; i++) {
5488 if (!bdev_is_zoned(FDEV(i).bdev))
5489 continue;
5490 if (sbi->s_ndevs == 1 || (FDEV(i).start_blk <= zone_blkaddr &&
5491 zone_blkaddr <= FDEV(i).end_blk))
5492 return &FDEV(i);
5493 }
5494
5495 return NULL;
5496}
5497
5498static int report_one_zone_cb(struct blk_zone *zone, unsigned int idx,
5499 void *data)
5500{
5501 memcpy(data, zone, sizeof(struct blk_zone));
5502 return 0;
5503}
5504
5505static int do_fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type)
5506{
5507 struct curseg_info *cs = CURSEG_I(sbi, type);
5508 struct f2fs_dev_info *zbd;
5509 struct blk_zone zone;
5510 unsigned int cs_section, wp_segno, wp_blkoff, wp_sector_off;
5511 block_t cs_zone_block, wp_block;
5512 unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
5513 sector_t zone_sector;
5514 int err;
5515
5516 cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
5517 cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
5518
5519 zbd = get_target_zoned_dev(sbi, cs_zone_block);
5520 if (!zbd)
5521 return 0;
5522
5523 /* report zone for the sector the curseg points to */
5524 zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
5525 << log_sectors_per_block;
5526 err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
5527 report_one_zone_cb, &zone);
5528 if (err != 1) {
5529 f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
5530 zbd->path, err);
5531 return err;
5532 }
5533
5534 if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
5535 return 0;
5536
5537 /*
5538 * When safely unmounted in the previous mount, we could use current
5539 * segments. Otherwise, allocate new sections.
5540 */
5541 if (is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG)) {
5542 wp_block = zbd->start_blk + (zone.wp >> log_sectors_per_block);
5543 wp_segno = GET_SEGNO(sbi, wp_block);
5544 wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
5545 wp_sector_off = zone.wp & GENMASK(log_sectors_per_block - 1, 0);
5546
5547 if (cs->segno == wp_segno && cs->next_blkoff == wp_blkoff &&
5548 wp_sector_off == 0)
5549 return 0;
5550
5551 f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: "
5552 "curseg[0x%x,0x%x] wp[0x%x,0x%x]", type, cs->segno,
5553 cs->next_blkoff, wp_segno, wp_blkoff);
5554 }
5555
5556 /* Allocate a new section if it's not new. */
5557 if (cs->next_blkoff ||
5558 cs->segno != GET_SEG_FROM_SEC(sbi, GET_ZONE_FROM_SEC(sbi, cs_section))) {
5559 unsigned int old_segno = cs->segno, old_blkoff = cs->next_blkoff;
5560
5561 f2fs_allocate_new_section(sbi, type, true);
5562 f2fs_notice(sbi, "Assign new section to curseg[%d]: "
5563 "[0x%x,0x%x] -> [0x%x,0x%x]",
5564 type, old_segno, old_blkoff,
5565 cs->segno, cs->next_blkoff);
5566 }
5567
5568 /* check consistency of the zone curseg pointed to */
5569 if (check_zone_write_pointer(sbi, zbd, &zone))
5570 return -EIO;
5571
5572 /* check newly assigned zone */
5573 cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
5574 cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
5575
5576 zbd = get_target_zoned_dev(sbi, cs_zone_block);
5577 if (!zbd)
5578 return 0;
5579
5580 zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
5581 << log_sectors_per_block;
5582 err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
5583 report_one_zone_cb, &zone);
5584 if (err != 1) {
5585 f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
5586 zbd->path, err);
5587 return err;
5588 }
5589
5590 if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
5591 return 0;
5592
5593 if (zone.wp != zone.start) {
5594 f2fs_notice(sbi,
5595 "New zone for curseg[%d] is not yet discarded. "
5596 "Reset the zone: curseg[0x%x,0x%x]",
5597 type, cs->segno, cs->next_blkoff);
5598 err = __f2fs_issue_discard_zone(sbi, zbd->bdev, cs_zone_block,
5599 zone.len >> log_sectors_per_block);
5600 if (err) {
5601 f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
5602 zbd->path, err);
5603 return err;
5604 }
5605 }
5606
5607 return 0;
5608}
5609
5610static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
5611{
5612 int i, ret;
5613
5614 for (i = 0; i < NR_PERSISTENT_LOG; i++) {
5615 ret = do_fix_curseg_write_pointer(sbi, i);
5616 if (ret)
5617 return ret;
5618 }
5619
5620 return 0;
5621}
5622
5623struct check_zone_write_pointer_args {
5624 struct f2fs_sb_info *sbi;
5625 struct f2fs_dev_info *fdev;
5626};
5627
5628static int check_zone_write_pointer_cb(struct blk_zone *zone, unsigned int idx,
5629 void *data)
5630{
5631 struct check_zone_write_pointer_args *args;
5632
5633 args = (struct check_zone_write_pointer_args *)data;
5634
5635 return check_zone_write_pointer(args->sbi, args->fdev, zone);
5636}
5637
5638static int check_write_pointer(struct f2fs_sb_info *sbi)
5639{
5640 int i, ret;
5641 struct check_zone_write_pointer_args args;
5642
5643 for (i = 0; i < sbi->s_ndevs; i++) {
5644 if (!bdev_is_zoned(FDEV(i).bdev))
5645 continue;
5646
5647 args.sbi = sbi;
5648 args.fdev = &FDEV(i);
5649 ret = blkdev_report_zones(FDEV(i).bdev, 0, BLK_ALL_ZONES,
5650 check_zone_write_pointer_cb, &args);
5651 if (ret < 0)
5652 return ret;
5653 }
5654
5655 return 0;
5656}
5657
5658int f2fs_check_and_fix_write_pointer(struct f2fs_sb_info *sbi)
5659{
5660 int ret;
5661
5662 if (!f2fs_sb_has_blkzoned(sbi) || f2fs_readonly(sbi->sb) ||
5663 f2fs_hw_is_readonly(sbi))
5664 return 0;
5665
5666 f2fs_notice(sbi, "Checking entire write pointers");
5667 ret = fix_curseg_write_pointer(sbi);
5668 if (!ret)
5669 ret = check_write_pointer(sbi);
5670 return ret;
5671}
5672
5673/*
5674 * Return the number of usable blocks in a segment. The number of blocks
5675 * returned is always equal to the number of blocks in a segment for
5676 * segments fully contained within a sequential zone capacity or a
5677 * conventional zone. For segments partially contained in a sequential
5678 * zone capacity, the number of usable blocks up to the zone capacity
5679 * is returned. 0 is returned in all other cases.
5680 */
5681static inline unsigned int f2fs_usable_zone_blks_in_seg(
5682 struct f2fs_sb_info *sbi, unsigned int segno)
5683{
5684 block_t seg_start, sec_start_blkaddr, sec_cap_blkaddr;
5685 unsigned int secno;
5686
5687 if (!sbi->unusable_blocks_per_sec)
5688 return BLKS_PER_SEG(sbi);
5689
5690 secno = GET_SEC_FROM_SEG(sbi, segno);
5691 seg_start = START_BLOCK(sbi, segno);
5692 sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));
5693 sec_cap_blkaddr = sec_start_blkaddr + CAP_BLKS_PER_SEC(sbi);
5694
5695 /*
5696 * If segment starts before zone capacity and spans beyond
5697 * zone capacity, then usable blocks are from seg start to
5698 * zone capacity. If the segment starts after the zone capacity,
5699 * then there are no usable blocks.
5700 */
5701 if (seg_start >= sec_cap_blkaddr)
5702 return 0;
5703 if (seg_start + BLKS_PER_SEG(sbi) > sec_cap_blkaddr)
5704 return sec_cap_blkaddr - seg_start;
5705
5706 return BLKS_PER_SEG(sbi);
5707}
5708#else
5709int f2fs_check_and_fix_write_pointer(struct f2fs_sb_info *sbi)
5710{
5711 return 0;
5712}
5713
5714static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi,
5715 unsigned int segno)
5716{
5717 return 0;
5718}
5719
5720#endif
5721unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
5722 unsigned int segno)
5723{
5724 if (f2fs_sb_has_blkzoned(sbi))
5725 return f2fs_usable_zone_blks_in_seg(sbi, segno);
5726
5727 return BLKS_PER_SEG(sbi);
5728}
5729
5730unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi)
5731{
5732 if (f2fs_sb_has_blkzoned(sbi))
5733 return CAP_SEGS_PER_SEC(sbi);
5734
5735 return SEGS_PER_SEC(sbi);
5736}
5737
5738unsigned long long f2fs_get_section_mtime(struct f2fs_sb_info *sbi,
5739 unsigned int segno)
5740{
5741 unsigned int usable_segs_per_sec = f2fs_usable_segs_in_sec(sbi);
5742 unsigned int secno = 0, start = 0;
5743 unsigned int total_valid_blocks = 0;
5744 unsigned long long mtime = 0;
5745 unsigned int i = 0;
5746
5747 secno = GET_SEC_FROM_SEG(sbi, segno);
5748 start = GET_SEG_FROM_SEC(sbi, secno);
5749
5750 if (!__is_large_section(sbi)) {
5751 mtime = get_seg_entry(sbi, start + i)->mtime;
5752 goto out;
5753 }
5754
5755 for (i = 0; i < usable_segs_per_sec; i++) {
5756 /* for large section, only check the mtime of valid segments */
5757 struct seg_entry *se = get_seg_entry(sbi, start+i);
5758
5759 mtime += se->mtime * se->valid_blocks;
5760 total_valid_blocks += se->valid_blocks;
5761 }
5762
5763 if (total_valid_blocks == 0)
5764 return INVALID_MTIME;
5765
5766 mtime = div_u64(mtime, total_valid_blocks);
5767out:
5768 if (unlikely(mtime == INVALID_MTIME))
5769 mtime -= 1;
5770 return mtime;
5771}
5772
5773/*
5774 * Update min, max modified time for cost-benefit GC algorithm
5775 */
5776static void init_min_max_mtime(struct f2fs_sb_info *sbi)
5777{
5778 struct sit_info *sit_i = SIT_I(sbi);
5779 unsigned int segno;
5780
5781 down_write(&sit_i->sentry_lock);
5782
5783 sit_i->min_mtime = ULLONG_MAX;
5784
5785 for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi)) {
5786 unsigned long long mtime = 0;
5787
5788 mtime = f2fs_get_section_mtime(sbi, segno);
5789
5790 if (sit_i->min_mtime > mtime)
5791 sit_i->min_mtime = mtime;
5792 }
5793 sit_i->max_mtime = get_mtime(sbi, false);
5794 sit_i->dirty_max_mtime = 0;
5795 up_write(&sit_i->sentry_lock);
5796}
5797
5798int f2fs_build_segment_manager(struct f2fs_sb_info *sbi)
5799{
5800 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
5801 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
5802 struct f2fs_sm_info *sm_info;
5803 int err;
5804
5805 sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL);
5806 if (!sm_info)
5807 return -ENOMEM;
5808
5809 /* init sm info */
5810 sbi->sm_info = sm_info;
5811 sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
5812 sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
5813 sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
5814 sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
5815 sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
5816 sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
5817 sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
5818 sm_info->rec_prefree_segments = sm_info->main_segments *
5819 DEF_RECLAIM_PREFREE_SEGMENTS / 100;
5820 if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS)
5821 sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS;
5822
5823 if (!f2fs_lfs_mode(sbi))
5824 sm_info->ipu_policy = BIT(F2FS_IPU_FSYNC);
5825 sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
5826 sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
5827 sm_info->min_seq_blocks = BLKS_PER_SEG(sbi);
5828 sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS;
5829 sm_info->min_ssr_sections = reserved_sections(sbi);
5830
5831 INIT_LIST_HEAD(&sm_info->sit_entry_set);
5832
5833 init_f2fs_rwsem(&sm_info->curseg_lock);
5834
5835 err = f2fs_create_flush_cmd_control(sbi);
5836 if (err)
5837 return err;
5838
5839 err = create_discard_cmd_control(sbi);
5840 if (err)
5841 return err;
5842
5843 err = build_sit_info(sbi);
5844 if (err)
5845 return err;
5846 err = build_free_segmap(sbi);
5847 if (err)
5848 return err;
5849 err = build_curseg(sbi);
5850 if (err)
5851 return err;
5852
5853 /* reinit free segmap based on SIT */
5854 err = build_sit_entries(sbi);
5855 if (err)
5856 return err;
5857
5858 init_free_segmap(sbi);
5859 err = build_dirty_segmap(sbi);
5860 if (err)
5861 return err;
5862
5863 err = sanity_check_curseg(sbi);
5864 if (err)
5865 return err;
5866
5867 init_min_max_mtime(sbi);
5868 return 0;
5869}
5870
5871static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
5872 enum dirty_type dirty_type)
5873{
5874 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5875
5876 mutex_lock(&dirty_i->seglist_lock);
5877 kvfree(dirty_i->dirty_segmap[dirty_type]);
5878 dirty_i->nr_dirty[dirty_type] = 0;
5879 mutex_unlock(&dirty_i->seglist_lock);
5880}
5881
5882static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
5883{
5884 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5885
5886 kvfree(dirty_i->pinned_secmap);
5887 kvfree(dirty_i->victim_secmap);
5888}
5889
5890static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
5891{
5892 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5893 int i;
5894
5895 if (!dirty_i)
5896 return;
5897
5898 /* discard pre-free/dirty segments list */
5899 for (i = 0; i < NR_DIRTY_TYPE; i++)
5900 discard_dirty_segmap(sbi, i);
5901
5902 if (__is_large_section(sbi)) {
5903 mutex_lock(&dirty_i->seglist_lock);
5904 kvfree(dirty_i->dirty_secmap);
5905 mutex_unlock(&dirty_i->seglist_lock);
5906 }
5907
5908 destroy_victim_secmap(sbi);
5909 SM_I(sbi)->dirty_info = NULL;
5910 kfree(dirty_i);
5911}
5912
5913static void destroy_curseg(struct f2fs_sb_info *sbi)
5914{
5915 struct curseg_info *array = SM_I(sbi)->curseg_array;
5916 int i;
5917
5918 if (!array)
5919 return;
5920 SM_I(sbi)->curseg_array = NULL;
5921 for (i = 0; i < NR_CURSEG_TYPE; i++) {
5922 kfree(array[i].sum_blk);
5923 kfree(array[i].journal);
5924 }
5925 kfree(array);
5926}
5927
5928static void destroy_free_segmap(struct f2fs_sb_info *sbi)
5929{
5930 struct free_segmap_info *free_i = SM_I(sbi)->free_info;
5931
5932 if (!free_i)
5933 return;
5934 SM_I(sbi)->free_info = NULL;
5935 kvfree(free_i->free_segmap);
5936 kvfree(free_i->free_secmap);
5937 kfree(free_i);
5938}
5939
5940static void destroy_sit_info(struct f2fs_sb_info *sbi)
5941{
5942 struct sit_info *sit_i = SIT_I(sbi);
5943
5944 if (!sit_i)
5945 return;
5946
5947 if (sit_i->sentries)
5948 kvfree(sit_i->bitmap);
5949 kfree(sit_i->tmp_map);
5950
5951 kvfree(sit_i->sentries);
5952 kvfree(sit_i->sec_entries);
5953 kvfree(sit_i->dirty_sentries_bitmap);
5954
5955 SM_I(sbi)->sit_info = NULL;
5956 kfree(sit_i->sit_bitmap);
5957#ifdef CONFIG_F2FS_CHECK_FS
5958 kvfree(sit_i->invalid_segmap);
5959#endif
5960 kfree(sit_i);
5961}
5962
5963void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi)
5964{
5965 struct f2fs_sm_info *sm_info = SM_I(sbi);
5966
5967 if (!sm_info)
5968 return;
5969 f2fs_destroy_flush_cmd_control(sbi, true);
5970 destroy_discard_cmd_control(sbi);
5971 destroy_dirty_segmap(sbi);
5972 destroy_curseg(sbi);
5973 destroy_free_segmap(sbi);
5974 destroy_sit_info(sbi);
5975 sbi->sm_info = NULL;
5976 kfree(sm_info);
5977}
5978
5979int __init f2fs_create_segment_manager_caches(void)
5980{
5981 discard_entry_slab = f2fs_kmem_cache_create("f2fs_discard_entry",
5982 sizeof(struct discard_entry));
5983 if (!discard_entry_slab)
5984 goto fail;
5985
5986 discard_cmd_slab = f2fs_kmem_cache_create("f2fs_discard_cmd",
5987 sizeof(struct discard_cmd));
5988 if (!discard_cmd_slab)
5989 goto destroy_discard_entry;
5990
5991 sit_entry_set_slab = f2fs_kmem_cache_create("f2fs_sit_entry_set",
5992 sizeof(struct sit_entry_set));
5993 if (!sit_entry_set_slab)
5994 goto destroy_discard_cmd;
5995
5996 revoke_entry_slab = f2fs_kmem_cache_create("f2fs_revoke_entry",
5997 sizeof(struct revoke_entry));
5998 if (!revoke_entry_slab)
5999 goto destroy_sit_entry_set;
6000 return 0;
6001
6002destroy_sit_entry_set:
6003 kmem_cache_destroy(sit_entry_set_slab);
6004destroy_discard_cmd:
6005 kmem_cache_destroy(discard_cmd_slab);
6006destroy_discard_entry:
6007 kmem_cache_destroy(discard_entry_slab);
6008fail:
6009 return -ENOMEM;
6010}
6011
6012void f2fs_destroy_segment_manager_caches(void)
6013{
6014 kmem_cache_destroy(sit_entry_set_slab);
6015 kmem_cache_destroy(discard_cmd_slab);
6016 kmem_cache_destroy(discard_entry_slab);
6017 kmem_cache_destroy(revoke_entry_slab);
6018}