projects/linux·blob master · refs

2223 lines · 55.5 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
4 * All Rights Reserved.
5 */
6#include "xfs_platform.h"
7#include "xfs_fs.h"
8#include "xfs_shared.h"
9#include "xfs_format.h"
10#include "xfs_log_format.h"
11#include "xfs_trans_resv.h"
12#include "xfs_bit.h"
13#include "xfs_mount.h"
14#include "xfs_inode.h"
15#include "xfs_alloc.h"
16#include "xfs_bmap.h"
17#include "xfs_bmap_btree.h"
18#include "xfs_bmap_util.h"
19#include "xfs_trans.h"
20#include "xfs_trans_space.h"
21#include "xfs_icache.h"
22#include "xfs_rtalloc.h"
23#include "xfs_sb.h"
24#include "xfs_rtbitmap.h"
25#include "xfs_rtrmap_btree.h"
26#include "xfs_quota.h"
27#include "xfs_log_priv.h"
28#include "xfs_health.h"
29#include "xfs_da_format.h"
30#include "xfs_metafile.h"
31#include "xfs_rtgroup.h"
32#include "xfs_error.h"
33#include "xfs_trace.h"
34#include "xfs_rtrefcount_btree.h"
35#include "xfs_reflink.h"
36#include "xfs_zone_alloc.h"
37
38/*
39 * Return whether there are any free extents in the size range given
40 * by low and high, for the bitmap block bbno.
41 */
42STATIC int
43xfs_rtany_summary(
44 struct xfs_rtalloc_args *args,
45 int low, /* low log2 extent size */
46 int high, /* high log2 extent size */
47 xfs_fileoff_t bbno, /* bitmap block number */
48 int *maxlog) /* out: max log2 extent size free */
49{
50 uint8_t *rsum_cache = args->rtg->rtg_rsum_cache;
51 int error;
52 int log; /* loop counter, log2 of ext. size */
53 xfs_suminfo_t sum; /* summary data */
54
55 /* There are no extents at levels >= rsum_cache[bbno]. */
56 if (rsum_cache) {
57 high = min(high, rsum_cache[bbno] - 1);
58 if (low > high) {
59 *maxlog = -1;
60 return 0;
61 }
62 }
63
64 /*
65 * Loop over logs of extent sizes.
66 */
67 for (log = high; log >= low; log--) {
68 /*
69 * Get one summary datum.
70 */
71 error = xfs_rtget_summary(args, log, bbno, &sum);
72 if (error) {
73 return error;
74 }
75 /*
76 * If there are any, return success.
77 */
78 if (sum) {
79 *maxlog = log;
80 goto out;
81 }
82 }
83 /*
84 * Found nothing, return failure.
85 */
86 *maxlog = -1;
87out:
88 /* There were no extents at levels > log. */
89 if (rsum_cache && log + 1 < rsum_cache[bbno])
90 rsum_cache[bbno] = log + 1;
91 return 0;
92}
93
94/*
95 * Copy and transform the summary file, given the old and new
96 * parameters in the mount structures.
97 */
98STATIC int
99xfs_rtcopy_summary(
100 struct xfs_rtalloc_args *oargs,
101 struct xfs_rtalloc_args *nargs)
102{
103 xfs_fileoff_t bbno; /* bitmap block number */
104 int error;
105 int log; /* summary level number (log length) */
106 xfs_suminfo_t sum; /* summary data */
107
108 for (log = oargs->mp->m_rsumlevels - 1; log >= 0; log--) {
109 for (bbno = oargs->mp->m_sb.sb_rbmblocks - 1;
110 (xfs_srtblock_t)bbno >= 0;
111 bbno--) {
112 error = xfs_rtget_summary(oargs, log, bbno, &sum);
113 if (error)
114 goto out;
115 if (XFS_IS_CORRUPT(oargs->mp, sum < 0)) {
116 error = -EFSCORRUPTED;
117 goto out;
118 }
119 if (sum == 0)
120 continue;
121 error = xfs_rtmodify_summary(oargs, log, bbno, -sum);
122 if (error)
123 goto out;
124 error = xfs_rtmodify_summary(nargs, log, bbno, sum);
125 if (error)
126 goto out;
127 }
128 }
129 error = 0;
130out:
131 xfs_rtbuf_cache_relse(oargs);
132 return error;
133}
134/*
135 * Mark an extent specified by start and len allocated.
136 * Updates all the summary information as well as the bitmap.
137 */
138STATIC int
139xfs_rtallocate_range(
140 struct xfs_rtalloc_args *args,
141 xfs_rtxnum_t start, /* start rtext to allocate */
142 xfs_rtxlen_t len) /* in/out: summary block number */
143{
144 struct xfs_mount *mp = args->mp;
145 xfs_rtxnum_t end; /* end of the allocated rtext */
146 int error;
147 xfs_rtxnum_t postblock = 0; /* first rtext allocated > end */
148 xfs_rtxnum_t preblock = 0; /* first rtext allocated < start */
149
150 end = start + len - 1;
151 /*
152 * Assume we're allocating out of the middle of a free extent.
153 * We need to find the beginning and end of the extent so we can
154 * properly update the summary.
155 */
156 error = xfs_rtfind_back(args, start, &preblock);
157 if (error)
158 return error;
159
160 /*
161 * Find the next allocated block (end of free extent).
162 */
163 error = xfs_rtfind_forw(args, end, args->rtg->rtg_extents - 1,
164 &postblock);
165 if (error)
166 return error;
167
168 /*
169 * Decrement the summary information corresponding to the entire
170 * (old) free extent.
171 */
172 error = xfs_rtmodify_summary(args,
173 xfs_highbit64(postblock + 1 - preblock),
174 xfs_rtx_to_rbmblock(mp, preblock), -1);
175 if (error)
176 return error;
177
178 /*
179 * If there are blocks not being allocated at the front of the
180 * old extent, add summary data for them to be free.
181 */
182 if (preblock < start) {
183 error = xfs_rtmodify_summary(args,
184 xfs_highbit64(start - preblock),
185 xfs_rtx_to_rbmblock(mp, preblock), 1);
186 if (error)
187 return error;
188 }
189
190 /*
191 * If there are blocks not being allocated at the end of the
192 * old extent, add summary data for them to be free.
193 */
194 if (postblock > end) {
195 error = xfs_rtmodify_summary(args,
196 xfs_highbit64(postblock - end),
197 xfs_rtx_to_rbmblock(mp, end + 1), 1);
198 if (error)
199 return error;
200 }
201
202 /*
203 * Modify the bitmap to mark this extent allocated.
204 */
205 return xfs_rtmodify_range(args, start, len, 0);
206}
207
208/* Reduce @rtxlen until it is a multiple of @prod. */
209static inline xfs_rtxlen_t
210xfs_rtalloc_align_len(
211 xfs_rtxlen_t rtxlen,
212 xfs_rtxlen_t prod)
213{
214 if (unlikely(prod > 1))
215 return rounddown(rtxlen, prod);
216 return rtxlen;
217}
218
219/*
220 * Make sure we don't run off the end of the rt volume. Be careful that
221 * adjusting maxlen downwards doesn't cause us to fail the alignment checks.
222 */
223static inline xfs_rtxlen_t
224xfs_rtallocate_clamp_len(
225 struct xfs_rtgroup *rtg,
226 xfs_rtxnum_t startrtx,
227 xfs_rtxlen_t rtxlen,
228 xfs_rtxlen_t prod)
229{
230 xfs_rtxlen_t ret;
231
232 ret = min(rtg->rtg_extents, startrtx + rtxlen) - startrtx;
233 return xfs_rtalloc_align_len(ret, prod);
234}
235
236/*
237 * Attempt to allocate an extent minlen<=len<=maxlen starting from
238 * bitmap block bbno. If we don't get maxlen then use prod to trim
239 * the length, if given. Returns error; returns starting block in *rtx.
240 * The lengths are all in rtextents.
241 */
242STATIC int
243xfs_rtallocate_extent_block(
244 struct xfs_rtalloc_args *args,
245 xfs_fileoff_t bbno, /* bitmap block number */
246 xfs_rtxlen_t minlen, /* minimum length to allocate */
247 xfs_rtxlen_t maxlen, /* maximum length to allocate */
248 xfs_rtxlen_t *len, /* out: actual length allocated */
249 xfs_rtxnum_t *nextp, /* out: next rtext to try */
250 xfs_rtxlen_t prod, /* extent product factor */
251 xfs_rtxnum_t *rtx) /* out: start rtext allocated */
252{
253 struct xfs_mount *mp = args->mp;
254 xfs_rtxnum_t besti = -1; /* best rtext found so far */
255 xfs_rtxnum_t end; /* last rtext in chunk */
256 xfs_rtxnum_t i; /* current rtext trying */
257 xfs_rtxnum_t next; /* next rtext to try */
258 xfs_rtxlen_t scanlen; /* number of free rtx to look for */
259 xfs_rtxlen_t bestlen = 0; /* best length found so far */
260 int stat; /* status from internal calls */
261 int error;
262
263 /*
264 * Loop over all the extents starting in this bitmap block up to the
265 * end of the rt volume, looking for one that's long enough.
266 */
267 end = min(args->rtg->rtg_extents, xfs_rbmblock_to_rtx(mp, bbno + 1)) -
268 1;
269 for (i = xfs_rbmblock_to_rtx(mp, bbno); i <= end; i++) {
270 /* Make sure we don't scan off the end of the rt volume. */
271 scanlen = xfs_rtallocate_clamp_len(args->rtg, i, maxlen, prod);
272 if (scanlen < minlen)
273 break;
274
275 /*
276 * See if there's a free extent of scanlen starting at i.
277 * If it's not so then next will contain the first non-free.
278 */
279 error = xfs_rtcheck_range(args, i, scanlen, 1, &next, &stat);
280 if (error)
281 return error;
282 if (stat) {
283 /*
284 * i to scanlen is all free, allocate and return that.
285 */
286 *len = scanlen;
287 *rtx = i;
288 return 0;
289 }
290
291 /*
292 * In the case where we have a variable-sized allocation
293 * request, figure out how big this free piece is,
294 * and if it's big enough for the minimum, and the best
295 * so far, remember it.
296 */
297 if (minlen < maxlen) {
298 xfs_rtxnum_t thislen; /* this extent size */
299
300 thislen = next - i;
301 if (thislen >= minlen && thislen > bestlen) {
302 besti = i;
303 bestlen = thislen;
304 }
305 }
306 /*
307 * If not done yet, find the start of the next free space.
308 */
309 if (next >= end)
310 break;
311 error = xfs_rtfind_forw(args, next, end, &i);
312 if (error)
313 return error;
314 }
315
316 /* Searched the whole thing & didn't find a maxlen free extent. */
317 if (besti == -1)
318 goto nospace;
319
320 /*
321 * Ensure bestlen is a multiple of prod, but don't return a too-short
322 * extent.
323 */
324 bestlen = xfs_rtalloc_align_len(bestlen, prod);
325 if (bestlen < minlen)
326 goto nospace;
327
328 /*
329 * Pick besti for bestlen & return that.
330 */
331 *len = bestlen;
332 *rtx = besti;
333 return 0;
334nospace:
335 /* Allocation failed. Set *nextp to the next block to try. */
336 *nextp = next;
337 return -ENOSPC;
338}
339
340/*
341 * Allocate an extent of length minlen<=len<=maxlen, starting at block
342 * bno. If we don't get maxlen then use prod to trim the length, if given.
343 * Returns error; returns starting block in *rtx.
344 * The lengths are all in rtextents.
345 */
346STATIC int
347xfs_rtallocate_extent_exact(
348 struct xfs_rtalloc_args *args,
349 xfs_rtxnum_t start, /* starting rtext number to allocate */
350 xfs_rtxlen_t minlen, /* minimum length to allocate */
351 xfs_rtxlen_t maxlen, /* maximum length to allocate */
352 xfs_rtxlen_t *len, /* out: actual length allocated */
353 xfs_rtxlen_t prod, /* extent product factor */
354 xfs_rtxnum_t *rtx) /* out: start rtext allocated */
355{
356 xfs_rtxnum_t next; /* next rtext to try (dummy) */
357 xfs_rtxlen_t alloclen; /* candidate length */
358 xfs_rtxlen_t scanlen; /* number of free rtx to look for */
359 int isfree; /* extent is free */
360 int error;
361
362 ASSERT(minlen % prod == 0);
363 ASSERT(maxlen % prod == 0);
364
365 /* Make sure we don't run off the end of the rt volume. */
366 scanlen = xfs_rtallocate_clamp_len(args->rtg, start, maxlen, prod);
367 if (scanlen < minlen)
368 return -ENOSPC;
369
370 /* Check if the range in question (for scanlen) is free. */
371 error = xfs_rtcheck_range(args, start, scanlen, 1, &next, &isfree);
372 if (error)
373 return error;
374
375 if (isfree) {
376 /* start to scanlen is all free; allocate it. */
377 *len = scanlen;
378 *rtx = start;
379 return 0;
380 }
381
382 /*
383 * If not, allocate what there is, if it's at least minlen.
384 */
385 alloclen = next - start;
386 if (alloclen < minlen)
387 return -ENOSPC;
388
389 /* Ensure alloclen is a multiple of prod. */
390 alloclen = xfs_rtalloc_align_len(alloclen, prod);
391 if (alloclen < minlen)
392 return -ENOSPC;
393
394 *len = alloclen;
395 *rtx = start;
396 return 0;
397}
398
399/*
400 * Allocate an extent of length minlen<=len<=maxlen, starting as near
401 * to start as possible. If we don't get maxlen then use prod to trim
402 * the length, if given. The lengths are all in rtextents.
403 */
404STATIC int
405xfs_rtallocate_extent_near(
406 struct xfs_rtalloc_args *args,
407 xfs_rtxnum_t start, /* starting rtext number to allocate */
408 xfs_rtxlen_t minlen, /* minimum length to allocate */
409 xfs_rtxlen_t maxlen, /* maximum length to allocate */
410 xfs_rtxlen_t *len, /* out: actual length allocated */
411 xfs_rtxlen_t prod, /* extent product factor */
412 xfs_rtxnum_t *rtx) /* out: start rtext allocated */
413{
414 struct xfs_mount *mp = args->mp;
415 int maxlog; /* max useful extent from summary */
416 xfs_fileoff_t bbno; /* bitmap block number */
417 int error;
418 int i; /* bitmap block offset (loop control) */
419 int j; /* secondary loop control */
420 int log2len; /* log2 of minlen */
421 xfs_rtxnum_t n; /* next rtext to try */
422
423 ASSERT(minlen % prod == 0);
424 ASSERT(maxlen % prod == 0);
425
426 /*
427 * If the block number given is off the end, silently set it to the last
428 * block.
429 */
430 start = min(start, args->rtg->rtg_extents - 1);
431
432 /*
433 * Try the exact allocation first.
434 */
435 error = xfs_rtallocate_extent_exact(args, start, minlen, maxlen, len,
436 prod, rtx);
437 if (error != -ENOSPC)
438 return error;
439
440 bbno = xfs_rtx_to_rbmblock(mp, start);
441 i = 0;
442 j = -1;
443 ASSERT(minlen != 0);
444 log2len = xfs_highbit32(minlen);
445 /*
446 * Loop over all bitmap blocks (bbno + i is current block).
447 */
448 for (;;) {
449 /*
450 * Get summary information of extents of all useful levels
451 * starting in this bitmap block.
452 */
453 error = xfs_rtany_summary(args, log2len, mp->m_rsumlevels - 1,
454 bbno + i, &maxlog);
455 if (error)
456 return error;
457
458 /*
459 * If there are any useful extents starting here, try
460 * allocating one.
461 */
462 if (maxlog >= 0) {
463 xfs_extlen_t maxavail =
464 min_t(xfs_rtblock_t, maxlen,
465 (1ULL << (maxlog + 1)) - 1);
466 /*
467 * On the positive side of the starting location.
468 */
469 if (i >= 0) {
470 /*
471 * Try to allocate an extent starting in
472 * this block.
473 */
474 error = xfs_rtallocate_extent_block(args,
475 bbno + i, minlen, maxavail, len,
476 &n, prod, rtx);
477 if (error != -ENOSPC)
478 return error;
479 }
480 /*
481 * On the negative side of the starting location.
482 */
483 else { /* i < 0 */
484 int maxblocks;
485
486 /*
487 * Loop backwards to find the end of the extent
488 * we found in the realtime summary.
489 *
490 * maxblocks is the maximum possible number of
491 * bitmap blocks from the start of the extent
492 * to the end of the extent.
493 */
494 if (maxlog == 0)
495 maxblocks = 0;
496 else if (maxlog < mp->m_blkbit_log)
497 maxblocks = 1;
498 else
499 maxblocks = 2 << (maxlog - mp->m_blkbit_log);
500
501 /*
502 * We need to check bbno + i + maxblocks down to
503 * bbno + i. We already checked bbno down to
504 * bbno + j + 1, so we don't need to check those
505 * again.
506 */
507 j = min(i + maxblocks, j);
508 for (; j >= i; j--) {
509 error = xfs_rtallocate_extent_block(args,
510 bbno + j, minlen,
511 maxavail, len, &n, prod,
512 rtx);
513 if (error != -ENOSPC)
514 return error;
515 }
516 }
517 }
518 /*
519 * Loop control. If we were on the positive side, and there's
520 * still more blocks on the negative side, go there.
521 */
522 if (i > 0 && (int)bbno - i >= 0)
523 i = -i;
524 /*
525 * If positive, and no more negative, but there are more
526 * positive, go there.
527 */
528 else if (i > 0 && (int)bbno + i < mp->m_sb.sb_rbmblocks - 1)
529 i++;
530 /*
531 * If negative or 0 (just started), and there are positive
532 * blocks to go, go there. The 0 case moves to block 1.
533 */
534 else if (i <= 0 && (int)bbno - i < mp->m_sb.sb_rbmblocks - 1)
535 i = 1 - i;
536 /*
537 * If negative or 0 and there are more negative blocks,
538 * go there.
539 */
540 else if (i <= 0 && (int)bbno + i > 0)
541 i--;
542 /*
543 * Must be done. Return failure.
544 */
545 else
546 break;
547 }
548 return -ENOSPC;
549}
550
551static int
552xfs_rtalloc_sumlevel(
553 struct xfs_rtalloc_args *args,
554 int l, /* level number */
555 xfs_rtxlen_t minlen, /* minimum length to allocate */
556 xfs_rtxlen_t maxlen, /* maximum length to allocate */
557 xfs_rtxlen_t prod, /* extent product factor */
558 xfs_rtxlen_t *len, /* out: actual length allocated */
559 xfs_rtxnum_t *rtx) /* out: start rtext allocated */
560{
561 xfs_fileoff_t i; /* bitmap block number */
562 int error;
563
564 for (i = 0; i < args->mp->m_sb.sb_rbmblocks; i++) {
565 xfs_suminfo_t sum; /* summary information for extents */
566 xfs_rtxnum_t n; /* next rtext to be tried */
567
568 error = xfs_rtget_summary(args, l, i, &sum);
569 if (error)
570 return error;
571
572 /*
573 * Nothing there, on to the next block.
574 */
575 if (!sum)
576 continue;
577
578 /*
579 * Try allocating the extent.
580 */
581 error = xfs_rtallocate_extent_block(args, i, minlen, maxlen,
582 len, &n, prod, rtx);
583 if (error != -ENOSPC)
584 return error;
585
586 /*
587 * If the "next block to try" returned from the allocator is
588 * beyond the next bitmap block, skip to that bitmap block.
589 */
590 if (xfs_rtx_to_rbmblock(args->mp, n) > i + 1)
591 i = xfs_rtx_to_rbmblock(args->mp, n) - 1;
592 }
593
594 return -ENOSPC;
595}
596
597/*
598 * Allocate an extent of length minlen<=len<=maxlen, with no position
599 * specified. If we don't get maxlen then use prod to trim
600 * the length, if given. The lengths are all in rtextents.
601 */
602static int
603xfs_rtallocate_extent_size(
604 struct xfs_rtalloc_args *args,
605 xfs_rtxlen_t minlen, /* minimum length to allocate */
606 xfs_rtxlen_t maxlen, /* maximum length to allocate */
607 xfs_rtxlen_t *len, /* out: actual length allocated */
608 xfs_rtxlen_t prod, /* extent product factor */
609 xfs_rtxnum_t *rtx) /* out: start rtext allocated */
610{
611 int error;
612 int l; /* level number (loop control) */
613
614 ASSERT(minlen % prod == 0);
615 ASSERT(maxlen % prod == 0);
616 ASSERT(maxlen != 0);
617
618 /*
619 * Loop over all the levels starting with maxlen.
620 *
621 * At each level, look at all the bitmap blocks, to see if there are
622 * extents starting there that are long enough (>= maxlen).
623 *
624 * Note, only on the initial level can the allocation fail if the
625 * summary says there's an extent.
626 */
627 for (l = xfs_highbit32(maxlen); l < args->mp->m_rsumlevels; l++) {
628 error = xfs_rtalloc_sumlevel(args, l, minlen, maxlen, prod, len,
629 rtx);
630 if (error != -ENOSPC)
631 return error;
632 }
633
634 /*
635 * Didn't find any maxlen blocks. Try smaller ones, unless we are
636 * looking for a fixed size extent.
637 */
638 if (minlen > --maxlen)
639 return -ENOSPC;
640 ASSERT(minlen != 0);
641 ASSERT(maxlen != 0);
642
643 /*
644 * Loop over sizes, from maxlen down to minlen.
645 *
646 * This time, when we do the allocations, allow smaller ones to succeed,
647 * but make sure the specified minlen/maxlen are in the possible range
648 * for this summary level.
649 */
650 for (l = xfs_highbit32(maxlen); l >= xfs_highbit32(minlen); l--) {
651 error = xfs_rtalloc_sumlevel(args, l,
652 max_t(xfs_rtxlen_t, minlen, 1 << l),
653 min_t(xfs_rtxlen_t, maxlen, (1 << (l + 1)) - 1),
654 prod, len, rtx);
655 if (error != -ENOSPC)
656 return error;
657 }
658
659 return -ENOSPC;
660}
661
662static void
663xfs_rtunmount_rtg(
664 struct xfs_rtgroup *rtg)
665{
666 int i;
667
668 for (i = 0; i < XFS_RTGI_MAX; i++)
669 xfs_rtginode_irele(&rtg->rtg_inodes[i]);
670 if (!xfs_has_zoned(rtg_mount(rtg)))
671 kvfree(rtg->rtg_rsum_cache);
672}
673
674static int
675xfs_alloc_rsum_cache(
676 struct xfs_rtgroup *rtg,
677 xfs_extlen_t rbmblocks)
678{
679 /*
680 * The rsum cache is initialized to the maximum value, which is
681 * trivially an upper bound on the maximum level with any free extents.
682 */
683 rtg->rtg_rsum_cache = kvmalloc(rbmblocks, GFP_KERNEL);
684 if (!rtg->rtg_rsum_cache)
685 return -ENOMEM;
686 memset(rtg->rtg_rsum_cache, -1, rbmblocks);
687 return 0;
688}
689
690/*
691 * If we changed the rt extent size (meaning there was no rt volume previously)
692 * and the root directory had EXTSZINHERIT and RTINHERIT set, it's possible
693 * that the extent size hint on the root directory is no longer congruent with
694 * the new rt extent size. Log the rootdir inode to fix this.
695 */
696static int
697xfs_growfs_rt_fixup_extsize(
698 struct xfs_mount *mp)
699{
700 struct xfs_inode *ip = mp->m_rootip;
701 struct xfs_trans *tp;
702 int error = 0;
703
704 xfs_ilock(ip, XFS_IOLOCK_EXCL);
705 if (!(ip->i_diflags & XFS_DIFLAG_RTINHERIT) ||
706 !(ip->i_diflags & XFS_DIFLAG_EXTSZINHERIT))
707 goto out_iolock;
708
709 error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_ichange, 0, 0, false,
710 &tp);
711 if (error)
712 goto out_iolock;
713
714 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
715 error = xfs_trans_commit(tp);
716 xfs_iunlock(ip, XFS_ILOCK_EXCL);
717
718out_iolock:
719 xfs_iunlock(ip, XFS_IOLOCK_EXCL);
720 return error;
721}
722
723/* Ensure that the rtgroup metadata inode is loaded, creating it if neeeded. */
724static int
725xfs_rtginode_ensure(
726 struct xfs_rtgroup *rtg,
727 enum xfs_rtg_inodes type)
728{
729 struct xfs_trans *tp;
730 int error;
731
732 if (rtg->rtg_inodes[type])
733 return 0;
734
735 tp = xfs_trans_alloc_empty(rtg_mount(rtg));
736 error = xfs_rtginode_load(rtg, type, tp);
737 xfs_trans_cancel(tp);
738
739 if (error != -ENOENT)
740 return error;
741 return xfs_rtginode_create(rtg, type, true);
742}
743
744static struct xfs_mount *
745xfs_growfs_rt_alloc_fake_mount(
746 const struct xfs_mount *mp,
747 xfs_rfsblock_t rblocks,
748 xfs_agblock_t rextsize)
749{
750 struct xfs_mount *nmp;
751
752 nmp = kmemdup(mp, sizeof(*mp), GFP_KERNEL);
753 if (!nmp)
754 return NULL;
755 xfs_mount_sb_set_rextsize(nmp, &nmp->m_sb, rextsize);
756 nmp->m_sb.sb_rblocks = rblocks;
757 nmp->m_sb.sb_rextents = xfs_blen_to_rtbxlen(nmp, nmp->m_sb.sb_rblocks);
758 nmp->m_sb.sb_rbmblocks = xfs_rtbitmap_blockcount(nmp);
759 nmp->m_sb.sb_rextslog = xfs_compute_rextslog(nmp->m_sb.sb_rextents);
760 if (xfs_has_rtgroups(nmp))
761 nmp->m_sb.sb_rgcount = howmany_64(nmp->m_sb.sb_rextents,
762 nmp->m_sb.sb_rgextents);
763 else
764 nmp->m_sb.sb_rgcount = 1;
765 nmp->m_rsumblocks = xfs_rtsummary_blockcount(nmp, &nmp->m_rsumlevels);
766
767 if (rblocks > 0)
768 nmp->m_features |= XFS_FEAT_REALTIME;
769
770 /* recompute growfsrt reservation from new rsumsize */
771 xfs_trans_resv_calc(nmp, &nmp->m_resv);
772 return nmp;
773}
774
775/* Free all the new space and return the number of extents actually freed. */
776static int
777xfs_growfs_rt_free_new(
778 struct xfs_rtgroup *rtg,
779 struct xfs_rtalloc_args *nargs,
780 xfs_rtbxlen_t *freed_rtx)
781{
782 struct xfs_mount *mp = rtg_mount(rtg);
783 xfs_rgnumber_t rgno = rtg_rgno(rtg);
784 xfs_rtxnum_t start_rtx = 0, end_rtx;
785
786 if (rgno < mp->m_sb.sb_rgcount)
787 start_rtx = xfs_rtgroup_extents(mp, rgno);
788 end_rtx = xfs_rtgroup_extents(nargs->mp, rgno);
789
790 /*
791 * Compute the first new extent that we want to free, being careful to
792 * skip past a realtime superblock at the start of the realtime volume.
793 */
794 if (xfs_has_rtsb(nargs->mp) && rgno == 0 && start_rtx == 0)
795 start_rtx++;
796 *freed_rtx = end_rtx - start_rtx;
797 return xfs_rtfree_range(nargs, start_rtx, *freed_rtx);
798}
799
800static xfs_rfsblock_t
801xfs_growfs_rt_nrblocks(
802 struct xfs_rtgroup *rtg,
803 xfs_rfsblock_t nrblocks,
804 xfs_agblock_t rextsize,
805 xfs_fileoff_t bmbno)
806{
807 struct xfs_mount *mp = rtg_mount(rtg);
808 xfs_rfsblock_t step;
809
810 step = (bmbno + 1) * mp->m_rtx_per_rbmblock * rextsize;
811 if (xfs_has_rtgroups(mp)) {
812 xfs_rfsblock_t rgblocks = mp->m_sb.sb_rgextents * rextsize;
813
814 step = min(rgblocks, step) + rgblocks * rtg_rgno(rtg);
815 }
816
817 return min(nrblocks, step);
818}
819
820/*
821 * If the post-grow filesystem will have an rtsb; we're initializing the first
822 * rtgroup; and the filesystem didn't have a realtime section, write the rtsb
823 * now, and attach the rtsb buffer to the real mount.
824 */
825static int
826xfs_growfs_rt_init_rtsb(
827 const struct xfs_rtalloc_args *nargs,
828 const struct xfs_rtgroup *rtg,
829 const struct xfs_rtalloc_args *args)
830{
831 struct xfs_mount *mp = args->mp;
832 struct xfs_buf *rtsb_bp;
833 int error;
834
835 if (!xfs_has_rtsb(nargs->mp))
836 return 0;
837 if (rtg_rgno(rtg) > 0)
838 return 0;
839 if (mp->m_sb.sb_rblocks)
840 return 0;
841
842 error = xfs_buf_get_uncached(mp->m_rtdev_targp, XFS_FSB_TO_BB(mp, 1),
843 &rtsb_bp);
844 if (error)
845 return error;
846
847 rtsb_bp->b_maps[0].bm_bn = XFS_RTSB_DADDR;
848 rtsb_bp->b_ops = &xfs_rtsb_buf_ops;
849
850 xfs_update_rtsb(rtsb_bp, mp->m_sb_bp);
851 mp->m_rtsb_bp = rtsb_bp;
852 error = xfs_bwrite(rtsb_bp);
853 xfs_buf_unlock(rtsb_bp);
854 if (error)
855 return error;
856
857 /* Initialize the rtrmap to reflect the rtsb. */
858 if (rtg_rmap(args->rtg) != NULL)
859 error = xfs_rtrmapbt_init_rtsb(nargs->mp, args->rtg, args->tp);
860
861 return error;
862}
863
864static void
865xfs_growfs_rt_sb_fields(
866 struct xfs_trans *tp,
867 const struct xfs_mount *nmp)
868{
869 struct xfs_mount *mp = tp->t_mountp;
870
871 if (nmp->m_sb.sb_rextsize != mp->m_sb.sb_rextsize)
872 xfs_trans_mod_sb(tp, XFS_TRANS_SB_REXTSIZE,
873 nmp->m_sb.sb_rextsize - mp->m_sb.sb_rextsize);
874 if (nmp->m_sb.sb_rbmblocks != mp->m_sb.sb_rbmblocks)
875 xfs_trans_mod_sb(tp, XFS_TRANS_SB_RBMBLOCKS,
876 nmp->m_sb.sb_rbmblocks - mp->m_sb.sb_rbmblocks);
877 if (nmp->m_sb.sb_rblocks != mp->m_sb.sb_rblocks)
878 xfs_trans_mod_sb(tp, XFS_TRANS_SB_RBLOCKS,
879 nmp->m_sb.sb_rblocks - mp->m_sb.sb_rblocks);
880 if (nmp->m_sb.sb_rextents != mp->m_sb.sb_rextents)
881 xfs_trans_mod_sb(tp, XFS_TRANS_SB_REXTENTS,
882 nmp->m_sb.sb_rextents - mp->m_sb.sb_rextents);
883 if (nmp->m_sb.sb_rextslog != mp->m_sb.sb_rextslog)
884 xfs_trans_mod_sb(tp, XFS_TRANS_SB_REXTSLOG,
885 nmp->m_sb.sb_rextslog - mp->m_sb.sb_rextslog);
886 if (nmp->m_sb.sb_rgcount != mp->m_sb.sb_rgcount)
887 xfs_trans_mod_sb(tp, XFS_TRANS_SB_RGCOUNT,
888 nmp->m_sb.sb_rgcount - mp->m_sb.sb_rgcount);
889}
890
891static int
892xfs_growfs_rt_zoned(
893 struct xfs_rtgroup *rtg)
894{
895 struct xfs_mount *mp = rtg_mount(rtg);
896 struct xfs_mount *nmp;
897 struct xfs_trans *tp;
898 xfs_rtbxlen_t freed_rtx;
899 int error;
900
901 /*
902 * Calculate new sb and mount fields for this round. Also ensure the
903 * rtg_extents value is uptodate as the rtbitmap code relies on it.
904 */
905 nmp = xfs_growfs_rt_alloc_fake_mount(mp,
906 xfs_rtgs_to_rfsbs(mp, rtg_rgno(rtg) + 1),
907 mp->m_sb.sb_rextsize);
908 if (!nmp)
909 return -ENOMEM;
910 freed_rtx = nmp->m_sb.sb_rextents - mp->m_sb.sb_rextents;
911
912 xfs_rtgroup_calc_geometry(nmp, rtg, rtg_rgno(rtg),
913 nmp->m_sb.sb_rgcount, nmp->m_sb.sb_rextents);
914
915 error = xfs_trans_alloc(mp, &M_RES(nmp)->tr_growrtfree, 0, 0, 0, &tp);
916 if (error)
917 goto out_free;
918
919 xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP);
920 xfs_rtgroup_trans_join(tp, rtg, XFS_RTGLOCK_RMAP);
921
922 xfs_growfs_rt_sb_fields(tp, nmp);
923 xfs_trans_mod_sb(tp, XFS_TRANS_SB_FREXTENTS, freed_rtx);
924
925 error = xfs_trans_commit(tp);
926 if (error)
927 goto out_free;
928
929 /*
930 * Ensure the mount RT feature flag is now set, and compute new
931 * maxlevels for rt btrees.
932 */
933 mp->m_features |= XFS_FEAT_REALTIME;
934 xfs_rtrmapbt_compute_maxlevels(mp);
935 xfs_rtrefcountbt_compute_maxlevels(mp);
936
937 /*
938 * Finally add the newly added zone to the freelist and add the space
939 * to the available counter. The order is important here: only add
940 * the available space after the zones, as available space guarantees
941 * that zones to back it are available.
942 */
943 xfs_zone_mark_free(rtg);
944 xfs_zoned_add_available(mp, freed_rtx);
945out_free:
946 kfree(nmp);
947 return error;
948}
949
950static int
951xfs_growfs_rt_bmblock(
952 struct xfs_rtgroup *rtg,
953 xfs_rfsblock_t nrblocks,
954 xfs_agblock_t rextsize,
955 xfs_fileoff_t bmbno)
956{
957 struct xfs_mount *mp = rtg_mount(rtg);
958 struct xfs_inode *rbmip = rtg_bitmap(rtg);
959 struct xfs_inode *rsumip = rtg_summary(rtg);
960 struct xfs_rtalloc_args args = {
961 .mp = mp,
962 .rtg = rtg,
963 };
964 struct xfs_rtalloc_args nargs = {
965 .rtg = rtg,
966 };
967 struct xfs_mount *nmp;
968 xfs_rtbxlen_t freed_rtx;
969 int error;
970
971 /*
972 * Calculate new sb and mount fields for this round. Also ensure the
973 * rtg_extents value is uptodate as the rtbitmap code relies on it.
974 */
975 nmp = nargs.mp = xfs_growfs_rt_alloc_fake_mount(mp,
976 xfs_growfs_rt_nrblocks(rtg, nrblocks, rextsize, bmbno),
977 rextsize);
978 if (!nmp)
979 return -ENOMEM;
980
981 xfs_rtgroup_calc_geometry(nmp, rtg, rtg_rgno(rtg),
982 nmp->m_sb.sb_rgcount, nmp->m_sb.sb_rextents);
983
984 /*
985 * Recompute the growfsrt reservation from the new rsumsize, so that the
986 * transaction below use the new, potentially larger value.
987 * */
988 xfs_trans_resv_calc(nmp, &nmp->m_resv);
989 error = xfs_trans_alloc(mp, &M_RES(nmp)->tr_growrtfree, 0, 0, 0,
990 &args.tp);
991 if (error)
992 goto out_free;
993 nargs.tp = args.tp;
994
995 xfs_rtgroup_lock(args.rtg, XFS_RTGLOCK_BITMAP | XFS_RTGLOCK_RMAP);
996 xfs_rtgroup_trans_join(args.tp, args.rtg,
997 XFS_RTGLOCK_BITMAP | XFS_RTGLOCK_RMAP);
998
999 /*
1000 * Update the bitmap inode's size ondisk and incore. We need to update
1001 * the incore size so that inode inactivation won't punch what it thinks
1002 * are "posteof" blocks.
1003 */
1004 rbmip->i_disk_size = nmp->m_sb.sb_rbmblocks * nmp->m_sb.sb_blocksize;
1005 i_size_write(VFS_I(rbmip), rbmip->i_disk_size);
1006 xfs_trans_log_inode(args.tp, rbmip, XFS_ILOG_CORE);
1007
1008 /*
1009 * Update the summary inode's size. We need to update the incore size
1010 * so that inode inactivation won't punch what it thinks are "posteof"
1011 * blocks.
1012 */
1013 rsumip->i_disk_size = nmp->m_rsumblocks * nmp->m_sb.sb_blocksize;
1014 i_size_write(VFS_I(rsumip), rsumip->i_disk_size);
1015 xfs_trans_log_inode(args.tp, rsumip, XFS_ILOG_CORE);
1016
1017 /*
1018 * Copy summary data from old to new sizes when the real size (not
1019 * block-aligned) changes.
1020 */
1021 if (mp->m_sb.sb_rbmblocks != nmp->m_sb.sb_rbmblocks ||
1022 mp->m_rsumlevels != nmp->m_rsumlevels) {
1023 error = xfs_rtcopy_summary(&args, &nargs);
1024 if (error)
1025 goto out_cancel;
1026 }
1027
1028 error = xfs_growfs_rt_init_rtsb(&nargs, rtg, &args);
1029 if (error)
1030 goto out_cancel;
1031
1032 /*
1033 * Update superblock fields.
1034 */
1035 xfs_growfs_rt_sb_fields(args.tp, nmp);
1036
1037 /*
1038 * Free the new extent.
1039 */
1040 error = xfs_growfs_rt_free_new(rtg, &nargs, &freed_rtx);
1041 xfs_rtbuf_cache_relse(&nargs);
1042 if (error)
1043 goto out_cancel;
1044
1045 /*
1046 * Mark more blocks free in the superblock.
1047 */
1048 xfs_trans_mod_sb(args.tp, XFS_TRANS_SB_FREXTENTS, freed_rtx);
1049
1050 /*
1051 * Update the calculated values in the real mount structure.
1052 */
1053 mp->m_rsumlevels = nmp->m_rsumlevels;
1054 mp->m_rsumblocks = nmp->m_rsumblocks;
1055
1056 /*
1057 * Recompute the growfsrt reservation from the new rsumsize.
1058 */
1059 xfs_trans_resv_calc(mp, &mp->m_resv);
1060
1061 /*
1062 * Sync sb counters now to reflect the updated values. Lazy counters are
1063 * not always updated and in order to avoid inconsistencies between
1064 * frextents and rtextents, it is better to sync the counters.
1065 */
1066
1067 if (xfs_has_lazysbcount(mp))
1068 xfs_log_sb(args.tp);
1069
1070 error = xfs_trans_commit(args.tp);
1071 if (error)
1072 goto out_free;
1073
1074 /*
1075 * Ensure the mount RT feature flag is now set, and compute new
1076 * maxlevels for rt btrees.
1077 */
1078 mp->m_features |= XFS_FEAT_REALTIME;
1079 xfs_rtrmapbt_compute_maxlevels(mp);
1080 xfs_rtrefcountbt_compute_maxlevels(mp);
1081
1082 kfree(nmp);
1083 return 0;
1084
1085out_cancel:
1086 xfs_trans_cancel(args.tp);
1087out_free:
1088 kfree(nmp);
1089 return error;
1090}
1091
1092static xfs_rtxnum_t
1093xfs_last_rtgroup_extents(
1094 struct xfs_mount *mp)
1095{
1096 return mp->m_sb.sb_rextents -
1097 ((xfs_rtxnum_t)(mp->m_sb.sb_rgcount - 1) *
1098 mp->m_sb.sb_rgextents);
1099}
1100
1101/*
1102 * This will return the bitmap block number (indexed at 0) that will be
1103 * extended/modified. There are 2 cases here:
1104 * 1. The size of the rtg is such that it is a multiple of
1105 * xfs_rtbitmap_rtx_per_rbmblock() i.e, an integral number of bitmap blocks
1106 * are completely filled up. In this case, we should return
1107 * 1 + (the last used bitmap block number).
1108 * 2. The size of the rtg is not an multiple of xfs_rtbitmap_rtx_per_rbmblock().
1109 * Here we will return the block number of last used block number. In this
1110 * case, we will modify the last used bitmap block to extend the size of the
1111 * rtgroup.
1112 *
1113 * This also deals with the case where there were no rtextents before.
1114 */
1115static xfs_fileoff_t
1116xfs_last_rt_bmblock_to_extend(
1117 struct xfs_rtgroup *rtg)
1118{
1119 struct xfs_mount *mp = rtg_mount(rtg);
1120 xfs_rgnumber_t rgno = rtg_rgno(rtg);
1121 xfs_fileoff_t bmbno = 0;
1122 unsigned int mod = 0;
1123
1124 ASSERT(!mp->m_sb.sb_rgcount || rgno >= mp->m_sb.sb_rgcount - 1);
1125
1126 if (mp->m_sb.sb_rgcount && rgno == mp->m_sb.sb_rgcount - 1) {
1127 xfs_rtxnum_t nrext = xfs_last_rtgroup_extents(mp);
1128
1129 /* Also fill up the previous block if not entirely full. */
1130 /* We are doing a -1 to convert it to a 0 based index */
1131 bmbno = xfs_rtbitmap_blockcount_len(mp, nrext) - 1;
1132 div_u64_rem(nrext, xfs_rtbitmap_rtx_per_rbmblock(mp), &mod);
1133 /*
1134 * mod = 0 means that all the current blocks are full. So
1135 * return the next block number to be used for the rtgroup
1136 * growth.
1137 */
1138 if (mod == 0)
1139 bmbno++;
1140 }
1141
1142 return bmbno;
1143}
1144
1145/*
1146 * Allocate space to the bitmap and summary files, as necessary.
1147 */
1148static int
1149xfs_growfs_rt_alloc_blocks(
1150 struct xfs_rtgroup *rtg,
1151 xfs_rfsblock_t nrblocks,
1152 xfs_agblock_t rextsize,
1153 xfs_extlen_t *nrbmblocks)
1154{
1155 struct xfs_mount *mp = rtg_mount(rtg);
1156 struct xfs_inode *rbmip = rtg_bitmap(rtg);
1157 struct xfs_inode *rsumip = rtg_summary(rtg);
1158 xfs_extlen_t orbmblocks = 0;
1159 xfs_extlen_t orsumblocks = 0;
1160 struct xfs_mount *nmp;
1161 int error = 0;
1162
1163 nmp = xfs_growfs_rt_alloc_fake_mount(mp, nrblocks, rextsize);
1164 if (!nmp)
1165 return -ENOMEM;
1166 *nrbmblocks = nmp->m_sb.sb_rbmblocks;
1167
1168 if (xfs_has_rtgroups(mp)) {
1169 /*
1170 * For file systems with the rtgroups feature, the RT bitmap and
1171 * summary are always fully allocated, which means that we never
1172 * need to grow the existing files.
1173 *
1174 * But we have to be careful to only fill the bitmap until the
1175 * end of the actually used range.
1176 */
1177 if (rtg_rgno(rtg) == nmp->m_sb.sb_rgcount - 1)
1178 *nrbmblocks = xfs_rtbitmap_blockcount_len(nmp,
1179 xfs_last_rtgroup_extents(nmp));
1180
1181 if (mp->m_sb.sb_rgcount &&
1182 rtg_rgno(rtg) == mp->m_sb.sb_rgcount - 1)
1183 goto out_free;
1184 } else {
1185 /*
1186 * Get the old block counts for bitmap and summary inodes.
1187 * These can't change since other growfs callers are locked out.
1188 */
1189 orbmblocks = XFS_B_TO_FSB(mp, rbmip->i_disk_size);
1190 orsumblocks = XFS_B_TO_FSB(mp, rsumip->i_disk_size);
1191 }
1192
1193 error = xfs_rtfile_initialize_blocks(rtg, XFS_RTGI_BITMAP, orbmblocks,
1194 nmp->m_sb.sb_rbmblocks, NULL);
1195 if (error)
1196 goto out_free;
1197 error = xfs_rtfile_initialize_blocks(rtg, XFS_RTGI_SUMMARY, orsumblocks,
1198 nmp->m_rsumblocks, NULL);
1199out_free:
1200 kfree(nmp);
1201 return error;
1202}
1203
1204static int
1205xfs_growfs_rtg(
1206 struct xfs_mount *mp,
1207 xfs_rgnumber_t rgno,
1208 xfs_rfsblock_t nrblocks,
1209 xfs_agblock_t rextsize)
1210{
1211 uint8_t *old_rsum_cache = NULL;
1212 xfs_extlen_t bmblocks;
1213 xfs_fileoff_t bmbno;
1214 struct xfs_rtgroup *rtg;
1215 unsigned int i;
1216 int error;
1217
1218 rtg = xfs_rtgroup_grab(mp, rgno);
1219 if (!rtg)
1220 return -EINVAL;
1221
1222 for (i = 0; i < XFS_RTGI_MAX; i++) {
1223 error = xfs_rtginode_ensure(rtg, i);
1224 if (error)
1225 goto out_rele;
1226 }
1227
1228 if (xfs_has_zoned(mp)) {
1229 error = xfs_growfs_rt_zoned(rtg);
1230 goto out_rele;
1231 }
1232
1233 error = xfs_growfs_rt_alloc_blocks(rtg, nrblocks, rextsize, &bmblocks);
1234 if (error)
1235 goto out_rele;
1236
1237 if (bmblocks != rtg_mount(rtg)->m_sb.sb_rbmblocks) {
1238 old_rsum_cache = rtg->rtg_rsum_cache;
1239 error = xfs_alloc_rsum_cache(rtg, bmblocks);
1240 if (error)
1241 goto out_rele;
1242 }
1243
1244 for (bmbno = xfs_last_rt_bmblock_to_extend(rtg); bmbno < bmblocks;
1245 bmbno++) {
1246 error = xfs_growfs_rt_bmblock(rtg, nrblocks, rextsize, bmbno);
1247 if (error)
1248 goto out_error;
1249 }
1250
1251 kvfree(old_rsum_cache);
1252 goto out_rele;
1253
1254out_error:
1255 /*
1256 * Reset rtg_extents to the old value if adding more blocks failed.
1257 */
1258 xfs_rtgroup_calc_geometry(mp, rtg, rtg_rgno(rtg), mp->m_sb.sb_rgcount,
1259 mp->m_sb.sb_rextents);
1260 if (old_rsum_cache) {
1261 kvfree(rtg->rtg_rsum_cache);
1262 rtg->rtg_rsum_cache = old_rsum_cache;
1263 }
1264out_rele:
1265 xfs_rtgroup_rele(rtg);
1266 return error;
1267}
1268
1269int
1270xfs_growfs_check_rtgeom(
1271 const struct xfs_mount *mp,
1272 xfs_rfsblock_t dblocks,
1273 xfs_rfsblock_t rblocks,
1274 xfs_extlen_t rextsize)
1275{
1276 xfs_extlen_t min_logfsbs;
1277 struct xfs_mount *nmp;
1278
1279 nmp = xfs_growfs_rt_alloc_fake_mount(mp, rblocks, rextsize);
1280 if (!nmp)
1281 return -ENOMEM;
1282 nmp->m_sb.sb_dblocks = dblocks;
1283
1284 xfs_rtrmapbt_compute_maxlevels(nmp);
1285 xfs_rtrefcountbt_compute_maxlevels(nmp);
1286 xfs_trans_resv_calc(nmp, M_RES(nmp));
1287
1288 /*
1289 * New summary size can't be more than half the size of the log. This
1290 * prevents us from getting a log overflow, since we'll log basically
1291 * the whole summary file at once.
1292 */
1293 min_logfsbs = min_t(xfs_extlen_t, xfs_log_calc_minimum_size(nmp),
1294 nmp->m_rsumblocks * 2);
1295
1296 trace_xfs_growfs_check_rtgeom(mp, min_logfsbs);
1297
1298 if (min_logfsbs > mp->m_sb.sb_logblocks)
1299 goto out_inval;
1300
1301 if (xfs_has_zoned(mp)) {
1302 uint32_t gblocks = mp->m_groups[XG_TYPE_RTG].blocks;
1303 uint32_t rem;
1304
1305 if (rextsize != 1)
1306 goto out_inval;
1307 div_u64_rem(nmp->m_sb.sb_rblocks, gblocks, &rem);
1308 if (rem) {
1309 xfs_warn(mp,
1310"new RT volume size (%lld) not aligned to RT group size (%d)",
1311 nmp->m_sb.sb_rblocks, gblocks);
1312 goto out_inval;
1313 }
1314 }
1315
1316 kfree(nmp);
1317 return 0;
1318out_inval:
1319 kfree(nmp);
1320 return -EINVAL;
1321}
1322
1323/*
1324 * Compute the new number of rt groups and ensure that /rtgroups exists.
1325 *
1326 * Changing the rtgroup size is not allowed (even if the rt volume hasn't yet
1327 * been initialized) because the userspace ABI doesn't support it.
1328 */
1329static int
1330xfs_growfs_rt_prep_groups(
1331 struct xfs_mount *mp,
1332 xfs_rfsblock_t rblocks,
1333 xfs_extlen_t rextsize,
1334 xfs_rgnumber_t *new_rgcount)
1335{
1336 int error;
1337
1338 *new_rgcount = howmany_64(rblocks, mp->m_sb.sb_rgextents * rextsize);
1339 if (*new_rgcount > XFS_MAX_RGNUMBER)
1340 return -EINVAL;
1341
1342 /* Make sure the /rtgroups dir has been created */
1343 if (!mp->m_rtdirip) {
1344 struct xfs_trans *tp;
1345
1346 tp = xfs_trans_alloc_empty(mp);
1347 error = xfs_rtginode_load_parent(tp);
1348 xfs_trans_cancel(tp);
1349
1350 if (error == -ENOENT)
1351 error = xfs_rtginode_mkdir_parent(mp);
1352 if (error)
1353 return error;
1354 }
1355
1356 return 0;
1357}
1358
1359static bool
1360xfs_grow_last_rtg(
1361 struct xfs_mount *mp)
1362{
1363 if (!xfs_has_rtgroups(mp))
1364 return true;
1365 if (mp->m_sb.sb_rgcount == 0)
1366 return false;
1367 return xfs_rtgroup_extents(mp, mp->m_sb.sb_rgcount - 1) <
1368 mp->m_sb.sb_rgextents;
1369}
1370
1371/*
1372 * Read in the last block of the RT device to make sure it is accessible.
1373 */
1374static int
1375xfs_rt_check_size(
1376 struct xfs_mount *mp,
1377 xfs_rfsblock_t last_block)
1378{
1379 xfs_daddr_t daddr = XFS_FSB_TO_BB(mp, last_block);
1380 struct xfs_buf *bp;
1381 int error;
1382
1383 if (XFS_BB_TO_FSB(mp, daddr) != last_block) {
1384 xfs_warn(mp, "RT device size overflow: %llu != %llu",
1385 XFS_BB_TO_FSB(mp, daddr), last_block);
1386 return -EFBIG;
1387 }
1388
1389 error = xfs_buf_read_uncached(mp->m_rtdev_targp,
1390 XFS_FSB_TO_BB(mp, mp->m_sb.sb_rtstart) + daddr,
1391 XFS_FSB_TO_BB(mp, 1), &bp, NULL);
1392 if (error)
1393 xfs_warn(mp, "cannot read last RT device sector (%lld)",
1394 last_block);
1395 else
1396 xfs_buf_relse(bp);
1397 return error;
1398}
1399
1400/*
1401 * Grow the realtime area of the filesystem.
1402 */
1403int
1404xfs_growfs_rt(
1405 struct xfs_mount *mp,
1406 struct xfs_growfs_rt *in)
1407{
1408 xfs_rgnumber_t old_rgcount = mp->m_sb.sb_rgcount;
1409 xfs_rgnumber_t new_rgcount = 1;
1410 xfs_rgnumber_t rgno;
1411 xfs_agblock_t old_rextsize = mp->m_sb.sb_rextsize;
1412 int error;
1413
1414 if (!capable(CAP_SYS_ADMIN))
1415 return -EPERM;
1416
1417 /* Needs to have been mounted with an rt device. */
1418 if (!XFS_IS_REALTIME_MOUNT(mp))
1419 return -EINVAL;
1420
1421 if (!mutex_trylock(&mp->m_growlock))
1422 return -EWOULDBLOCK;
1423
1424 /* Shrink not supported. */
1425 error = -EINVAL;
1426 if (in->newblocks <= mp->m_sb.sb_rblocks)
1427 goto out_unlock;
1428 /* Can only change rt extent size when adding rt volume. */
1429 if (mp->m_sb.sb_rblocks > 0 && in->extsize != mp->m_sb.sb_rextsize)
1430 goto out_unlock;
1431
1432 /* Range check the extent size. */
1433 if (XFS_FSB_TO_B(mp, in->extsize) > XFS_MAX_RTEXTSIZE ||
1434 XFS_FSB_TO_B(mp, in->extsize) < XFS_MIN_RTEXTSIZE)
1435 goto out_unlock;
1436
1437 /* Check for features supported only on rtgroups filesystems. */
1438 error = -EOPNOTSUPP;
1439 if (!xfs_has_rtgroups(mp)) {
1440 if (xfs_has_rmapbt(mp))
1441 goto out_unlock;
1442 if (xfs_has_quota(mp))
1443 goto out_unlock;
1444 if (xfs_has_reflink(mp))
1445 goto out_unlock;
1446 } else if (xfs_has_reflink(mp) &&
1447 !xfs_reflink_supports_rextsize(mp, in->extsize))
1448 goto out_unlock;
1449
1450 error = xfs_sb_validate_fsb_count(&mp->m_sb, in->newblocks);
1451 if (error)
1452 goto out_unlock;
1453
1454 error = xfs_rt_check_size(mp, in->newblocks - 1);
1455 if (error)
1456 goto out_unlock;
1457
1458 /*
1459 * Calculate new parameters. These are the final values to be reached.
1460 */
1461 error = -EINVAL;
1462 if (in->newblocks < in->extsize)
1463 goto out_unlock;
1464
1465 /* Make sure the new fs size won't cause problems with the log. */
1466 error = xfs_growfs_check_rtgeom(mp, mp->m_sb.sb_dblocks, in->newblocks,
1467 in->extsize);
1468 if (error)
1469 goto out_unlock;
1470
1471 if (xfs_has_rtgroups(mp)) {
1472 error = xfs_growfs_rt_prep_groups(mp, in->newblocks,
1473 in->extsize, &new_rgcount);
1474 if (error)
1475 goto out_unlock;
1476 }
1477
1478 if (xfs_grow_last_rtg(mp)) {
1479 error = xfs_growfs_rtg(mp, old_rgcount - 1, in->newblocks,
1480 in->extsize);
1481 if (error)
1482 goto out_unlock;
1483 }
1484
1485 for (rgno = old_rgcount; rgno < new_rgcount; rgno++) {
1486 xfs_rtbxlen_t rextents = div_u64(in->newblocks, in->extsize);
1487
1488 error = xfs_rtgroup_alloc(mp, rgno, new_rgcount, rextents);
1489 if (error)
1490 goto out_unlock;
1491
1492 error = xfs_growfs_rtg(mp, rgno, in->newblocks, in->extsize);
1493 if (error) {
1494 struct xfs_rtgroup *rtg;
1495
1496 rtg = xfs_rtgroup_grab(mp, rgno);
1497 if (!WARN_ON_ONCE(!rtg)) {
1498 xfs_rtunmount_rtg(rtg);
1499 xfs_rtgroup_rele(rtg);
1500 xfs_rtgroup_free(mp, rgno);
1501 }
1502 break;
1503 }
1504 }
1505
1506 if (!error && old_rextsize != in->extsize)
1507 error = xfs_growfs_rt_fixup_extsize(mp);
1508
1509 /*
1510 * Update secondary superblocks now the physical grow has completed.
1511 *
1512 * Also do this in case of an error as we might have already
1513 * successfully updated one or more RTGs and incremented sb_rgcount.
1514 */
1515 if (!xfs_is_shutdown(mp)) {
1516 int error2 = xfs_update_secondary_sbs(mp);
1517
1518 if (!error)
1519 error = error2;
1520
1521 /* Reset the rt metadata btree space reservations. */
1522 error2 = xfs_metafile_resv_init(mp);
1523 if (error2 && error2 != -ENOSPC)
1524 error = error2;
1525 }
1526
1527out_unlock:
1528 mutex_unlock(&mp->m_growlock);
1529 return error;
1530}
1531
1532/* Read the realtime superblock and attach it to the mount. */
1533int
1534xfs_rtmount_readsb(
1535 struct xfs_mount *mp)
1536{
1537 struct xfs_buf *bp;
1538 int error;
1539
1540 if (!xfs_has_rtsb(mp))
1541 return 0;
1542 if (mp->m_sb.sb_rblocks == 0)
1543 return 0;
1544 if (mp->m_rtdev_targp == NULL) {
1545 xfs_warn(mp,
1546 "Filesystem has a realtime volume, use rtdev=device option");
1547 return -ENODEV;
1548 }
1549
1550 /* m_blkbb_log is not set up yet */
1551 error = xfs_buf_read_uncached(mp->m_rtdev_targp, XFS_RTSB_DADDR,
1552 mp->m_sb.sb_blocksize >> BBSHIFT, &bp,
1553 &xfs_rtsb_buf_ops);
1554 if (error) {
1555 xfs_warn(mp, "rt sb validate failed with error %d.", error);
1556 /* bad CRC means corrupted metadata */
1557 if (error == -EFSBADCRC)
1558 error = -EFSCORRUPTED;
1559 return error;
1560 }
1561
1562 mp->m_rtsb_bp = bp;
1563 xfs_buf_unlock(bp);
1564 return 0;
1565}
1566
1567/* Detach the realtime superblock from the mount and free it. */
1568void
1569xfs_rtmount_freesb(
1570 struct xfs_mount *mp)
1571{
1572 struct xfs_buf *bp = mp->m_rtsb_bp;
1573
1574 if (!bp)
1575 return;
1576
1577 xfs_buf_lock(bp);
1578 mp->m_rtsb_bp = NULL;
1579 xfs_buf_relse(bp);
1580}
1581
1582/*
1583 * Initialize realtime fields in the mount structure.
1584 */
1585int /* error */
1586xfs_rtmount_init(
1587 struct xfs_mount *mp) /* file system mount structure */
1588{
1589 if (mp->m_sb.sb_rblocks == 0)
1590 return 0;
1591 if (mp->m_rtdev_targp == NULL) {
1592 xfs_warn(mp,
1593 "Filesystem has a realtime volume, use rtdev=device option");
1594 return -ENODEV;
1595 }
1596
1597 mp->m_rsumblocks = xfs_rtsummary_blockcount(mp, &mp->m_rsumlevels);
1598
1599 return xfs_rt_check_size(mp, mp->m_sb.sb_rblocks - 1);
1600}
1601
1602static int
1603xfs_rtalloc_count_frextent(
1604 struct xfs_rtgroup *rtg,
1605 struct xfs_trans *tp,
1606 const struct xfs_rtalloc_rec *rec,
1607 void *priv)
1608{
1609 uint64_t *valp = priv;
1610
1611 *valp += rec->ar_extcount;
1612 return 0;
1613}
1614
1615/*
1616 * Reinitialize the number of free realtime extents from the realtime bitmap.
1617 * Callers must ensure that there is no other activity in the filesystem.
1618 */
1619int
1620xfs_rtalloc_reinit_frextents(
1621 struct xfs_mount *mp)
1622{
1623 uint64_t val = 0;
1624 int error;
1625
1626 struct xfs_rtgroup *rtg = NULL;
1627
1628 while ((rtg = xfs_rtgroup_next(mp, rtg))) {
1629 xfs_rtgroup_lock(rtg, XFS_RTGLOCK_BITMAP_SHARED);
1630 error = xfs_rtalloc_query_all(rtg, NULL,
1631 xfs_rtalloc_count_frextent, &val);
1632 xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_BITMAP_SHARED);
1633 if (error) {
1634 xfs_rtgroup_rele(rtg);
1635 return error;
1636 }
1637 }
1638
1639 spin_lock(&mp->m_sb_lock);
1640 mp->m_sb.sb_frextents = val;
1641 spin_unlock(&mp->m_sb_lock);
1642 xfs_set_freecounter(mp, XC_FREE_RTEXTENTS, mp->m_sb.sb_frextents);
1643 return 0;
1644}
1645
1646/*
1647 * Read in the bmbt of an rt metadata inode so that we never have to load them
1648 * at runtime. This enables the use of shared ILOCKs for rtbitmap scans. Use
1649 * an empty transaction to avoid deadlocking on loops in the bmbt.
1650 */
1651static inline int
1652xfs_rtmount_iread_extents(
1653 struct xfs_trans *tp,
1654 struct xfs_inode *ip)
1655{
1656 int error;
1657
1658 xfs_ilock(ip, XFS_ILOCK_EXCL);
1659
1660 error = xfs_iread_extents(tp, ip, XFS_DATA_FORK);
1661 if (error)
1662 goto out_unlock;
1663
1664 if (xfs_inode_has_attr_fork(ip)) {
1665 error = xfs_iread_extents(tp, ip, XFS_ATTR_FORK);
1666 if (error)
1667 goto out_unlock;
1668 }
1669
1670out_unlock:
1671 xfs_iunlock(ip, XFS_ILOCK_EXCL);
1672 return error;
1673}
1674
1675static int
1676xfs_rtmount_rtg(
1677 struct xfs_mount *mp,
1678 struct xfs_trans *tp,
1679 struct xfs_rtgroup *rtg)
1680{
1681 int error, i;
1682
1683 for (i = 0; i < XFS_RTGI_MAX; i++) {
1684 error = xfs_rtginode_load(rtg, i, tp);
1685 if (error)
1686 return error;
1687
1688 if (rtg->rtg_inodes[i]) {
1689 error = xfs_rtmount_iread_extents(tp,
1690 rtg->rtg_inodes[i]);
1691 if (error)
1692 return error;
1693 }
1694 }
1695
1696 if (xfs_has_zoned(mp))
1697 return 0;
1698 return xfs_alloc_rsum_cache(rtg, mp->m_sb.sb_rbmblocks);
1699}
1700
1701/*
1702 * Get the bitmap and summary inodes and the summary cache into the mount
1703 * structure at mount time.
1704 */
1705int
1706xfs_rtmount_inodes(
1707 struct xfs_mount *mp)
1708{
1709 struct xfs_trans *tp;
1710 struct xfs_rtgroup *rtg = NULL;
1711 int error;
1712
1713 tp = xfs_trans_alloc_empty(mp);
1714 if (xfs_has_rtgroups(mp) && mp->m_sb.sb_rgcount > 0) {
1715 error = xfs_rtginode_load_parent(tp);
1716 if (error)
1717 goto out_cancel;
1718 }
1719
1720 while ((rtg = xfs_rtgroup_next(mp, rtg))) {
1721 error = xfs_rtmount_rtg(mp, tp, rtg);
1722 if (error) {
1723 xfs_rtgroup_rele(rtg);
1724 xfs_rtunmount_inodes(mp);
1725 break;
1726 }
1727 }
1728
1729out_cancel:
1730 xfs_trans_cancel(tp);
1731 return error;
1732}
1733
1734void
1735xfs_rtunmount_inodes(
1736 struct xfs_mount *mp)
1737{
1738 struct xfs_rtgroup *rtg = NULL;
1739
1740 while ((rtg = xfs_rtgroup_next(mp, rtg)))
1741 xfs_rtunmount_rtg(rtg);
1742 xfs_rtginode_irele(&mp->m_rtdirip);
1743}
1744
1745/*
1746 * Pick an extent for allocation at the start of a new realtime file.
1747 * Use the sequence number stored in the atime field of the bitmap inode.
1748 * Translate this to a fraction of the rtextents, and return the product
1749 * of rtextents and the fraction.
1750 * The fraction sequence is 0, 1/2, 1/4, 3/4, 1/8, ..., 7/8, 1/16, ...
1751 */
1752static xfs_rtxnum_t
1753xfs_rtpick_extent(
1754 struct xfs_rtgroup *rtg,
1755 struct xfs_trans *tp,
1756 xfs_rtxlen_t len) /* allocation length (rtextents) */
1757{
1758 struct xfs_mount *mp = rtg_mount(rtg);
1759 struct xfs_inode *rbmip = rtg_bitmap(rtg);
1760 xfs_rtxnum_t b = 0; /* result rtext */
1761 int log2; /* log of sequence number */
1762 uint64_t resid; /* residual after log removed */
1763 uint64_t seq; /* sequence number of file creation */
1764 struct timespec64 ts; /* timespec in inode */
1765
1766 xfs_assert_ilocked(rbmip, XFS_ILOCK_EXCL);
1767
1768 ts = inode_get_atime(VFS_I(rbmip));
1769 if (!(rbmip->i_diflags & XFS_DIFLAG_NEWRTBM)) {
1770 rbmip->i_diflags |= XFS_DIFLAG_NEWRTBM;
1771 seq = 0;
1772 } else {
1773 seq = ts.tv_sec;
1774 }
1775 log2 = xfs_highbit64(seq);
1776 if (log2 != -1) {
1777 resid = seq - (1ULL << log2);
1778 b = (mp->m_sb.sb_rextents * ((resid << 1) + 1ULL)) >>
1779 (log2 + 1);
1780 if (b >= mp->m_sb.sb_rextents)
1781 div64_u64_rem(b, mp->m_sb.sb_rextents, &b);
1782 if (b + len > mp->m_sb.sb_rextents)
1783 b = mp->m_sb.sb_rextents - len;
1784 }
1785 ts.tv_sec = seq + 1;
1786 inode_set_atime_to_ts(VFS_I(rbmip), ts);
1787 xfs_trans_log_inode(tp, rbmip, XFS_ILOG_CORE);
1788 return b;
1789}
1790
1791static void
1792xfs_rtalloc_align_minmax(
1793 xfs_rtxlen_t *raminlen,
1794 xfs_rtxlen_t *ramaxlen,
1795 xfs_rtxlen_t *prod)
1796{
1797 xfs_rtxlen_t newmaxlen = *ramaxlen;
1798 xfs_rtxlen_t newminlen = *raminlen;
1799 xfs_rtxlen_t slack;
1800
1801 slack = newmaxlen % *prod;
1802 if (slack)
1803 newmaxlen -= slack;
1804 slack = newminlen % *prod;
1805 if (slack)
1806 newminlen += *prod - slack;
1807
1808 /*
1809 * If adjusting for extent size hint alignment produces an invalid
1810 * min/max len combination, go ahead without it.
1811 */
1812 if (newmaxlen < newminlen) {
1813 *prod = 1;
1814 return;
1815 }
1816 *ramaxlen = newmaxlen;
1817 *raminlen = newminlen;
1818}
1819
1820/* Given a free extent, find any part of it that isn't busy, if possible. */
1821STATIC bool
1822xfs_rtalloc_check_busy(
1823 struct xfs_rtalloc_args *args,
1824 xfs_rtxnum_t start,
1825 xfs_rtxlen_t minlen_rtx,
1826 xfs_rtxlen_t maxlen_rtx,
1827 xfs_rtxlen_t len_rtx,
1828 xfs_rtxlen_t prod,
1829 xfs_rtxnum_t rtx,
1830 xfs_rtxlen_t *reslen,
1831 xfs_rtxnum_t *resrtx,
1832 unsigned *busy_gen)
1833{
1834 struct xfs_rtgroup *rtg = args->rtg;
1835 struct xfs_mount *mp = rtg_mount(rtg);
1836 xfs_agblock_t rgbno = xfs_rtx_to_rgbno(rtg, rtx);
1837 xfs_rgblock_t min_rgbno = xfs_rtx_to_rgbno(rtg, start);
1838 xfs_extlen_t minlen = xfs_rtxlen_to_extlen(mp, minlen_rtx);
1839 xfs_extlen_t len = xfs_rtxlen_to_extlen(mp, len_rtx);
1840 xfs_extlen_t diff;
1841 bool busy;
1842
1843 busy = xfs_extent_busy_trim(rtg_group(rtg), minlen,
1844 xfs_rtxlen_to_extlen(mp, maxlen_rtx), &rgbno, &len,
1845 busy_gen);
1846
1847 /*
1848 * If we have a largish extent that happens to start before min_rgbno,
1849 * see if we can shift it into range...
1850 */
1851 if (rgbno < min_rgbno && rgbno + len > min_rgbno) {
1852 diff = min_rgbno - rgbno;
1853 if (len > diff) {
1854 rgbno += diff;
1855 len -= diff;
1856 }
1857 }
1858
1859 if (prod > 1 && len >= minlen) {
1860 xfs_rgblock_t aligned_rgbno = roundup(rgbno, prod);
1861
1862 diff = aligned_rgbno - rgbno;
1863
1864 *resrtx = xfs_rgbno_to_rtx(mp, aligned_rgbno);
1865 *reslen = xfs_extlen_to_rtxlen(mp,
1866 diff >= len ? 0 : len - diff);
1867 } else {
1868 *resrtx = xfs_rgbno_to_rtx(mp, rgbno);
1869 *reslen = xfs_extlen_to_rtxlen(mp, len);
1870 }
1871
1872 return busy;
1873}
1874
1875/*
1876 * Adjust the given free extent so that it isn't busy, or flush the log and
1877 * wait for the space to become unbusy. Only needed for rtgroups.
1878 */
1879STATIC int
1880xfs_rtallocate_adjust_for_busy(
1881 struct xfs_rtalloc_args *args,
1882 xfs_rtxnum_t start,
1883 xfs_rtxlen_t minlen,
1884 xfs_rtxlen_t maxlen,
1885 xfs_rtxlen_t *len,
1886 xfs_rtxlen_t prod,
1887 xfs_rtxnum_t *rtx)
1888{
1889 xfs_rtxnum_t resrtx;
1890 xfs_rtxlen_t reslen;
1891 unsigned busy_gen;
1892 bool busy;
1893 int error;
1894
1895again:
1896 busy = xfs_rtalloc_check_busy(args, start, minlen, maxlen, *len, prod,
1897 *rtx, &reslen, &resrtx, &busy_gen);
1898 if (!busy)
1899 return 0;
1900
1901 if (reslen < minlen || (start != 0 && resrtx != *rtx)) {
1902 /*
1903 * Enough of the extent was busy that we cannot satisfy the
1904 * allocation, or this is a near allocation and the start of
1905 * the extent is busy. Flush the log and wait for the busy
1906 * situation to resolve.
1907 */
1908 trace_xfs_rtalloc_extent_busy(args->rtg, start, minlen, maxlen,
1909 *len, prod, *rtx, busy_gen);
1910
1911 error = xfs_extent_busy_flush(args->tp, rtg_group(args->rtg),
1912 busy_gen, 0);
1913 if (error)
1914 return error;
1915
1916 goto again;
1917 }
1918
1919 /* Some of the free space wasn't busy, hand that back to the caller. */
1920 trace_xfs_rtalloc_extent_busy_trim(args->rtg, *rtx, *len, resrtx,
1921 reslen);
1922 *len = reslen;
1923 *rtx = resrtx;
1924
1925 return 0;
1926}
1927
1928static int
1929xfs_rtallocate_rtg(
1930 struct xfs_trans *tp,
1931 xfs_rgnumber_t rgno,
1932 xfs_rtblock_t bno_hint,
1933 xfs_rtxlen_t minlen,
1934 xfs_rtxlen_t maxlen,
1935 xfs_rtxlen_t prod,
1936 bool wasdel,
1937 bool initial_user_data,
1938 bool *rtlocked,
1939 xfs_rtblock_t *bno,
1940 xfs_extlen_t *blen)
1941{
1942 struct xfs_rtalloc_args args = {
1943 .mp = tp->t_mountp,
1944 .tp = tp,
1945 };
1946 xfs_rtxnum_t start = 0;
1947 xfs_rtxnum_t rtx;
1948 xfs_rtxlen_t len = 0;
1949 int error = 0;
1950
1951 args.rtg = xfs_rtgroup_grab(args.mp, rgno);
1952 if (!args.rtg)
1953 return -ENOSPC;
1954
1955 /*
1956 * We need to lock out modifications to both the RT bitmap and summary
1957 * inodes for finding free space in xfs_rtallocate_extent_{near,size}
1958 * and join the bitmap and summary inodes for the actual allocation
1959 * down in xfs_rtallocate_range.
1960 *
1961 * For RTG-enabled file system we don't want to join the inodes to the
1962 * transaction until we are committed to allocate to allocate from this
1963 * RTG so that only one inode of each type is locked at a time.
1964 *
1965 * But for pre-RTG file systems we need to already to join the bitmap
1966 * inode to the transaction for xfs_rtpick_extent, which bumps the
1967 * sequence number in it, so we'll have to join the inode to the
1968 * transaction early here.
1969 *
1970 * This is all a bit messy, but at least the mess is contained in
1971 * this function.
1972 */
1973 if (!*rtlocked) {
1974 xfs_rtgroup_lock(args.rtg, XFS_RTGLOCK_BITMAP);
1975 if (!xfs_has_rtgroups(args.mp))
1976 xfs_rtgroup_trans_join(tp, args.rtg,
1977 XFS_RTGLOCK_BITMAP);
1978 *rtlocked = true;
1979 }
1980
1981 /*
1982 * For an allocation to an empty file at offset 0, pick an extent that
1983 * will space things out in the rt area.
1984 */
1985 if (bno_hint != NULLFSBLOCK)
1986 start = xfs_rtb_to_rtx(args.mp, bno_hint);
1987 else if (!xfs_has_rtgroups(args.mp) && initial_user_data)
1988 start = xfs_rtpick_extent(args.rtg, tp, maxlen);
1989
1990 if (start) {
1991 error = xfs_rtallocate_extent_near(&args, start, minlen, maxlen,
1992 &len, prod, &rtx);
1993 /*
1994 * If we can't allocate near a specific rt extent, try again
1995 * without locality criteria.
1996 */
1997 if (error == -ENOSPC) {
1998 xfs_rtbuf_cache_relse(&args);
1999 error = 0;
2000 }
2001 }
2002
2003 if (!error) {
2004 error = xfs_rtallocate_extent_size(&args, minlen, maxlen, &len,
2005 prod, &rtx);
2006 }
2007
2008 if (error) {
2009 if (xfs_has_rtgroups(args.mp))
2010 goto out_unlock;
2011 goto out_release;
2012 }
2013
2014 if (xfs_has_rtgroups(args.mp)) {
2015 error = xfs_rtallocate_adjust_for_busy(&args, start, minlen,
2016 maxlen, &len, prod, &rtx);
2017 if (error)
2018 goto out_unlock;
2019
2020 xfs_rtgroup_trans_join(tp, args.rtg, XFS_RTGLOCK_BITMAP);
2021 }
2022
2023 error = xfs_rtallocate_range(&args, rtx, len);
2024 if (error)
2025 goto out_release;
2026
2027 xfs_trans_mod_sb(tp, wasdel ?
2028 XFS_TRANS_SB_RES_FREXTENTS : XFS_TRANS_SB_FREXTENTS,
2029 -(long)len);
2030 *bno = xfs_rtx_to_rtb(args.rtg, rtx);
2031 *blen = xfs_rtxlen_to_extlen(args.mp, len);
2032
2033out_release:
2034 xfs_rtgroup_rele(args.rtg);
2035 xfs_rtbuf_cache_relse(&args);
2036 return error;
2037out_unlock:
2038 xfs_rtgroup_unlock(args.rtg, XFS_RTGLOCK_BITMAP);
2039 *rtlocked = false;
2040 goto out_release;
2041}
2042
2043int
2044xfs_rtallocate_rtgs(
2045 struct xfs_trans *tp,
2046 xfs_fsblock_t bno_hint,
2047 xfs_rtxlen_t minlen,
2048 xfs_rtxlen_t maxlen,
2049 xfs_rtxlen_t prod,
2050 bool wasdel,
2051 bool initial_user_data,
2052 xfs_rtblock_t *bno,
2053 xfs_extlen_t *blen)
2054{
2055 struct xfs_mount *mp = tp->t_mountp;
2056 xfs_rgnumber_t start_rgno, rgno;
2057 int error;
2058
2059 /*
2060 * For now this just blindly iterates over the RTGs for an initial
2061 * allocation. We could try to keep an in-memory rtg_longest member
2062 * to avoid the locking when just looking for big enough free space,
2063 * but for now this keeps things simple.
2064 */
2065 if (bno_hint != NULLFSBLOCK)
2066 start_rgno = xfs_rtb_to_rgno(mp, bno_hint);
2067 else
2068 start_rgno = (atomic_inc_return(&mp->m_rtgrotor) - 1) %
2069 mp->m_sb.sb_rgcount;
2070
2071 rgno = start_rgno;
2072 do {
2073 bool rtlocked = false;
2074
2075 error = xfs_rtallocate_rtg(tp, rgno, bno_hint, minlen, maxlen,
2076 prod, wasdel, initial_user_data, &rtlocked,
2077 bno, blen);
2078 if (error != -ENOSPC)
2079 return error;
2080 ASSERT(!rtlocked);
2081
2082 if (++rgno == mp->m_sb.sb_rgcount)
2083 rgno = 0;
2084 bno_hint = NULLFSBLOCK;
2085 } while (rgno != start_rgno);
2086
2087 return -ENOSPC;
2088}
2089
2090static int
2091xfs_rtallocate_align(
2092 struct xfs_bmalloca *ap,
2093 xfs_rtxlen_t *ralen,
2094 xfs_rtxlen_t *raminlen,
2095 xfs_rtxlen_t *prod,
2096 bool *noalign)
2097{
2098 struct xfs_mount *mp = ap->ip->i_mount;
2099 xfs_fileoff_t orig_offset = ap->offset;
2100 xfs_extlen_t minlen = mp->m_sb.sb_rextsize;
2101 xfs_extlen_t align; /* minimum allocation alignment */
2102 xfs_extlen_t mod; /* product factor for allocators */
2103 int error;
2104
2105 if (*noalign) {
2106 align = mp->m_sb.sb_rextsize;
2107 } else {
2108 if (ap->flags & XFS_BMAPI_COWFORK)
2109 align = xfs_get_cowextsz_hint(ap->ip);
2110 else
2111 align = xfs_get_extsz_hint(ap->ip);
2112 if (!align)
2113 align = 1;
2114 if (align == mp->m_sb.sb_rextsize)
2115 *noalign = true;
2116 }
2117
2118 error = xfs_bmap_extsize_align(mp, &ap->got, &ap->prev, align, 1,
2119 ap->eof, 0, ap->conv, &ap->offset, &ap->length);
2120 if (error)
2121 return error;
2122 ASSERT(ap->length);
2123 ASSERT(xfs_extlen_to_rtxmod(mp, ap->length) == 0);
2124
2125 /*
2126 * If we shifted the file offset downward to satisfy an extent size
2127 * hint, increase minlen by that amount so that the allocator won't
2128 * give us an allocation that's too short to cover at least one of the
2129 * blocks that the caller asked for.
2130 */
2131 if (ap->offset != orig_offset)
2132 minlen += orig_offset - ap->offset;
2133
2134 /*
2135 * Set ralen to be the actual requested length in rtextents.
2136 *
2137 * If the old value was close enough to XFS_BMBT_MAX_EXTLEN that
2138 * we rounded up to it, cut it back so it's valid again.
2139 * Note that if it's a really large request (bigger than
2140 * XFS_BMBT_MAX_EXTLEN), we don't hear about that number, and can't
2141 * adjust the starting point to match it.
2142 */
2143 *ralen = xfs_extlen_to_rtxlen(mp, min(ap->length, XFS_MAX_BMBT_EXTLEN));
2144 *raminlen = max_t(xfs_rtxlen_t, 1, xfs_extlen_to_rtxlen(mp, minlen));
2145 ASSERT(*raminlen > 0);
2146 ASSERT(*raminlen <= *ralen);
2147
2148 /*
2149 * Only bother calculating a real prod factor if offset & length are
2150 * perfectly aligned, otherwise it will just get us in trouble.
2151 */
2152 div_u64_rem(ap->offset, align, &mod);
2153 if (mod || ap->length % align)
2154 *prod = 1;
2155 else
2156 *prod = xfs_extlen_to_rtxlen(mp, align);
2157
2158 if (*prod > 1)
2159 xfs_rtalloc_align_minmax(raminlen, ralen, prod);
2160 return 0;
2161}
2162
2163int
2164xfs_bmap_rtalloc(
2165 struct xfs_bmalloca *ap)
2166{
2167 xfs_fileoff_t orig_offset = ap->offset;
2168 xfs_rtxlen_t prod = 0; /* product factor for allocators */
2169 xfs_rtxlen_t ralen = 0; /* realtime allocation length */
2170 xfs_rtblock_t bno_hint = NULLRTBLOCK;
2171 xfs_extlen_t orig_length = ap->length;
2172 xfs_rtxlen_t raminlen;
2173 bool rtlocked = false;
2174 bool noalign = false;
2175 bool initial_user_data =
2176 ap->datatype & XFS_ALLOC_INITIAL_USER_DATA;
2177 int error;
2178
2179 ASSERT(!xfs_has_zoned(ap->tp->t_mountp));
2180
2181retry:
2182 error = xfs_rtallocate_align(ap, &ralen, &raminlen, &prod, &noalign);
2183 if (error)
2184 return error;
2185
2186 if (xfs_bmap_adjacent(ap))
2187 bno_hint = ap->blkno;
2188
2189 if (xfs_has_rtgroups(ap->ip->i_mount)) {
2190 error = xfs_rtallocate_rtgs(ap->tp, bno_hint, raminlen, ralen,
2191 prod, ap->wasdel, initial_user_data,
2192 &ap->blkno, &ap->length);
2193 } else {
2194 error = xfs_rtallocate_rtg(ap->tp, 0, bno_hint, raminlen, ralen,
2195 prod, ap->wasdel, initial_user_data,
2196 &rtlocked, &ap->blkno, &ap->length);
2197 }
2198
2199 if (error == -ENOSPC) {
2200 if (!noalign) {
2201 /*
2202 * We previously enlarged the request length to try to
2203 * satisfy an extent size hint. The allocator didn't
2204 * return anything, so reset the parameters to the
2205 * original values and try again without alignment
2206 * criteria.
2207 */
2208 ap->offset = orig_offset;
2209 ap->length = orig_length;
2210 noalign = true;
2211 goto retry;
2212 }
2213
2214 ap->blkno = NULLFSBLOCK;
2215 ap->length = 0;
2216 return 0;
2217 }
2218 if (error)
2219 return error;
2220
2221 xfs_bmap_alloc_account(ap);
2222 return 0;
2223}