| 1 | |
| 2 |
|
| 3 | |
| 4 | * fs/ext4/fast_commit.c
|
| 5 | *
|
| 6 | * Written by Harshad Shirwadkar <[email protected]>
|
| 7 | *
|
| 8 | * Ext4 fast commits routines.
|
| 9 |
|
| 10 | #include "ext4.h"
|
| 11 | #include "ext4_jbd2.h"
|
| 12 | #include "ext4_extents.h"
|
| 13 | #include "mballoc.h"
|
| 14 |
|
| 15 | #include <linux/lockdep.h>
|
| 16 | #include <linux/wait_bit.h>
|
| 17 | |
| 18 | * Ext4 Fast Commits
|
| 19 | * -----------------
|
| 20 | *
|
| 21 | * Ext4 fast commits implement fine grained journalling for Ext4.
|
| 22 | *
|
| 23 | * Fast commits are organized as a log of tag-length-value (TLV) structs. (See
|
| 24 | * struct ext4_fc_tl). Each TLV contains some delta that is replayed TLV by
|
| 25 | * TLV during the recovery phase. For the scenarios for which we currently
|
| 26 | * don't have replay code, fast commit falls back to full commits.
|
| 27 | * Fast commits record delta in one of the following three categories.
|
| 28 | *
|
| 29 | * (A) Directory entry updates:
|
| 30 | *
|
| 31 | * - EXT4_FC_TAG_UNLINK - records directory entry unlink
|
| 32 | * - EXT4_FC_TAG_LINK - records directory entry link
|
| 33 | * - EXT4_FC_TAG_CREAT - records inode and directory entry creation
|
| 34 | *
|
| 35 | * (B) File specific data range updates:
|
| 36 | *
|
| 37 | * - EXT4_FC_TAG_ADD_RANGE - records addition of new blocks to an inode
|
| 38 | * - EXT4_FC_TAG_DEL_RANGE - records deletion of blocks from an inode
|
| 39 | *
|
| 40 | * (C) Inode metadata (mtime / ctime etc):
|
| 41 | *
|
| 42 | * - EXT4_FC_TAG_INODE - record the inode that should be replayed
|
| 43 | * during recovery. Note that iblocks field is
|
| 44 | * not replayed and instead derived during
|
| 45 | * replay.
|
| 46 | * Commit Operation
|
| 47 | * ----------------
|
| 48 | * With fast commits, we maintain all the directory entry operations in the
|
| 49 | * order in which they are issued in an in-memory queue. This queue is flushed
|
| 50 | * to disk during the commit operation. We also maintain a list of inodes
|
| 51 | * that need to be committed during a fast commit in another in memory queue of
|
| 52 | * inodes. During the commit operation, we commit in the following order:
|
| 53 | *
|
| 54 | * [1] Prepare all the inodes to write out their data by setting
|
| 55 | * "EXT4_STATE_FC_FLUSHING_DATA". This ensures that inode cannot be
|
| 56 | * deleted while it is being flushed.
|
| 57 | * [2] Flush data buffers to disk and clear "EXT4_STATE_FC_FLUSHING_DATA"
|
| 58 | * state.
|
| 59 | * [3] Lock the journal by calling jbd2_journal_lock_updates(). This ensures
|
| 60 | * that all the existing handles finish and no new handles can start.
|
| 61 | * [4] Mark all the fast commit eligible inodes as undergoing fast commit by
|
| 62 | * setting "EXT4_STATE_FC_COMMITTING" state, and snapshot the inode state
|
| 63 | * needed for log writing.
|
| 64 | * [5] Unlock the journal by calling jbd2_journal_unlock_updates(). This allows
|
| 65 | * starting of new handles. Updates to inodes being fast committed are
|
| 66 | * tracked for requeue rather than blocking.
|
| 67 | * [6] Commit all the directory entry updates in the fast commit space.
|
| 68 | * [7] Commit all the changed inodes in the fast commit space.
|
| 69 | * [8] Write tail tag (this tag ensures the atomicity, please read the following
|
| 70 | * section for more details).
|
| 71 | * [9] Clear "EXT4_STATE_FC_COMMITTING" and wake up waiters in
|
| 72 | * ext4_fc_cleanup().
|
| 73 | *
|
| 74 | * All the inode updates must be enclosed within jbd2_journal_start()
|
| 75 | * and jbd2_journal_stop() similar to JBD2 journaling.
|
| 76 | *
|
| 77 | * Fast Commit Ineligibility
|
| 78 | * -------------------------
|
| 79 | *
|
| 80 | * Not all operations are supported by fast commits today (e.g extended
|
| 81 | * attributes). Fast commit ineligibility is marked by calling
|
| 82 | * ext4_fc_mark_ineligible(): This makes next fast commit operation to fall back
|
| 83 | * to full commit.
|
| 84 | *
|
| 85 | * Atomicity of commits
|
| 86 | * --------------------
|
| 87 | * In order to guarantee atomicity during the commit operation, fast commit
|
| 88 | * uses "EXT4_FC_TAG_TAIL" tag that marks a fast commit as complete. Tail
|
| 89 | * tag contains CRC of the contents and TID of the transaction after which
|
| 90 | * this fast commit should be applied. Recovery code replays fast commit
|
| 91 | * logs only if there's at least 1 valid tail present. For every fast commit
|
| 92 | * operation, there is 1 tail. This means, we may end up with multiple tails
|
| 93 | * in the fast commit space. Here's an example:
|
| 94 | *
|
| 95 | * - Create a new file A and remove existing file B
|
| 96 | * - fsync()
|
| 97 | * - Append contents to file A
|
| 98 | * - Truncate file A
|
| 99 | * - fsync()
|
| 100 | *
|
| 101 | * The fast commit space at the end of above operations would look like this:
|
| 102 | * [HEAD] [CREAT A] [UNLINK B] [TAIL] [ADD_RANGE A] [DEL_RANGE A] [TAIL]
|
| 103 | * |<--- Fast Commit 1 --->|<--- Fast Commit 2 ---->|
|
| 104 | *
|
| 105 | * Replay code should thus check for all the valid tails in the FC area.
|
| 106 | *
|
| 107 | * Fast Commit Replay Idempotence
|
| 108 | * ------------------------------
|
| 109 | *
|
| 110 | * Fast commits tags are idempotent in nature provided the recovery code follows
|
| 111 | * certain rules. The guiding principle that the commit path follows while
|
| 112 | * committing is that it stores the result of a particular operation instead of
|
| 113 | * storing the procedure.
|
| 114 | *
|
| 115 | * Let's consider this rename operation: 'mv /a /b'. Let's assume dirent '/a'
|
| 116 | * was associated with inode 10. During fast commit, instead of storing this
|
| 117 | * operation as a procedure "rename a to b", we store the resulting file system
|
| 118 | * state as a "series" of outcomes:
|
| 119 | *
|
| 120 | * - Link dirent b to inode 10
|
| 121 | * - Unlink dirent a
|
| 122 | * - Inode <10> with valid refcount
|
| 123 | *
|
| 124 | * Now when recovery code runs, it needs "enforce" this state on the file
|
| 125 | * system. This is what guarantees idempotence of fast commit replay.
|
| 126 | *
|
| 127 | * Let's take an example of a procedure that is not idempotent and see how fast
|
| 128 | * commits make it idempotent. Consider following sequence of operations:
|
| 129 | *
|
| 130 | * rm A; mv B A; read A
|
| 131 | * (x) (y) (z)
|
| 132 | *
|
| 133 | * (x), (y) and (z) are the points at which we can crash. If we store this
|
| 134 | * sequence of operations as is then the replay is not idempotent. Let's say
|
| 135 | * while in replay, we crash at (z). During the second replay, file A (which was
|
| 136 | * actually created as a result of "mv B A" operation) would get deleted. Thus,
|
| 137 | * file named A would be absent when we try to read A. So, this sequence of
|
| 138 | * operations is not idempotent. However, as mentioned above, instead of storing
|
| 139 | * the procedure fast commits store the outcome of each procedure. Thus the fast
|
| 140 | * commit log for above procedure would be as follows:
|
| 141 | *
|
| 142 | * (Let's assume dirent A was linked to inode 10 and dirent B was linked to
|
| 143 | * inode 11 before the replay)
|
| 144 | *
|
| 145 | * [Unlink A] [Link A to inode 11] [Unlink B] [Inode 11]
|
| 146 | * (w) (x) (y) (z)
|
| 147 | *
|
| 148 | * If we crash at (z), we will have file A linked to inode 11. During the second
|
| 149 | * replay, we will remove file A (inode 11). But we will create it back and make
|
| 150 | * it point to inode 11. We won't find B, so we'll just skip that step. At this
|
| 151 | * point, the refcount for inode 11 is not reliable, but that gets fixed by the
|
| 152 | * replay of last inode 11 tag. Crashes at points (w), (x) and (y) get handled
|
| 153 | * similarly. Thus, by converting a non-idempotent procedure into a series of
|
| 154 | * idempotent outcomes, fast commits ensured idempotence during the replay.
|
| 155 | *
|
| 156 | * Locking
|
| 157 | * -------
|
| 158 | * sbi->s_fc_lock protects the fast commit inodes queue and the fast commit
|
| 159 | * dentry queue. ei->i_fc_lock protects the fast commit related info in a given
|
| 160 | * inode. Most of the code avoids acquiring both the locks, but if one must do
|
| 161 | * that then sbi->s_fc_lock must be acquired before ei->i_fc_lock.
|
| 162 | *
|
| 163 | * TODOs
|
| 164 | * -----
|
| 165 | *
|
| 166 | * 0) Fast commit replay path hardening: Fast commit replay code should use
|
| 167 | * journal handles to make sure all the updates it does during the replay
|
| 168 | * path are atomic. With that if we crash during fast commit replay, after
|
| 169 | * trying to do recovery again, we will find a file system where fast commit
|
| 170 | * area is invalid (because new full commit would be found). In order to deal
|
| 171 | * with that, fast commit replay code should ensure that the "FC_REPLAY"
|
| 172 | * superblock state is persisted before starting the replay, so that after
|
| 173 | * the crash, fast commit recovery code can look at that flag and perform
|
| 174 | * fast commit recovery even if that area is invalidated by later full
|
| 175 | * commits.
|
| 176 | *
|
| 177 | * 1) Handle more ineligible cases.
|
| 178 | *
|
| 179 | * 2) Change ext4_fc_commit() to lookup logical to physical mapping using extent
|
| 180 | * status tree. This would get rid of the need to call ext4_fc_track_inode()
|
| 181 | * before acquiring i_data_sem. To do that we would need to ensure that
|
| 182 | * modified extents from the extent status tree are not evicted from memory.
|
| 183 |
|
| 184 |
|
| 185 | #include <trace/events/ext4.h>
|
| 186 | static struct kmem_cache *ext4_fc_dentry_cachep;
|
| 187 | static struct kmem_cache *ext4_fc_range_cachep;
|
| 188 |
|
| 189 | |
| 190 | * Avoid spending unbounded time/memory snapshotting highly fragmented files
|
| 191 | * under jbd2_journal_lock_updates(). If we exceed this limit, fall back to
|
| 192 | * full commit.
|
| 193 |
|
| 194 | #define EXT4_FC_SNAPSHOT_MAX_INODES 1024
|
| 195 | #define EXT4_FC_SNAPSHOT_MAX_RANGES 2048
|
| 196 |
|
| 197 | static inline void ext4_fc_set_snap_err(int *snap_err, int err)
|
| 198 | {
|
| 199 | if (snap_err && *snap_err == EXT4_FC_SNAP_ERR_NONE)
|
| 200 | *snap_err = err;
|
| 201 | }
|
| 202 |
|
| 203 | static void ext4_fc_free_inode_snap(struct inode *inode);
|
| 204 |
|
| 205 | static inline void ext4_fc_reset_inode(struct inode *inode)
|
| 206 | {
|
| 207 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 208 |
|
| 209 | ei->i_fc_lblk_start = 0;
|
| 210 | ei->i_fc_lblk_len = 0;
|
| 211 | }
|
| 212 |
|
| 213 | void ext4_fc_init_inode(struct inode *inode)
|
| 214 | {
|
| 215 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 216 |
|
| 217 | ext4_fc_reset_inode(inode);
|
| 218 | ext4_clear_inode_state(inode, EXT4_STATE_FC_COMMITTING);
|
| 219 | ext4_clear_inode_state(inode, EXT4_STATE_FC_REQUEUE);
|
| 220 | INIT_LIST_HEAD(&ei->i_fc_list);
|
| 221 | INIT_LIST_HEAD(&ei->i_fc_dilist);
|
| 222 | ei->i_fc_snap = NULL;
|
| 223 | }
|
| 224 |
|
| 225 | static bool ext4_fc_disabled(struct super_block *sb)
|
| 226 | {
|
| 227 | return (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
|
| 228 | (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY));
|
| 229 | }
|
| 230 |
|
| 231 | static bool ext4_fc_eligible(struct super_block *sb)
|
| 232 | {
|
| 233 | return !ext4_fc_disabled(sb) &&
|
| 234 | !(ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE));
|
| 235 | }
|
| 236 |
|
| 237 | |
| 238 | * Wait for an inode fast-commit state bit to clear while dropping the
|
| 239 | * fast-commit lock around schedule().
|
| 240 |
|
| 241 | static void ext4_fc_wait_inode_state(struct inode *inode, int bit,
|
| 242 | int *alloc_ctx)
|
| 243 | {
|
| 244 | wait_queue_head_t *wq;
|
| 245 | unsigned long *wait_word = ext4_inode_state_wait_word(inode);
|
| 246 | int wait_bit = ext4_inode_state_wait_bit(bit);
|
| 247 |
|
| 248 | while (ext4_test_inode_state(inode, bit)) {
|
| 249 | DEFINE_WAIT_BIT(wait, wait_word, wait_bit);
|
| 250 |
|
| 251 | wq = bit_waitqueue(wait_word, wait_bit);
|
| 252 | prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
|
| 253 | if (ext4_test_inode_state(inode, bit)) {
|
| 254 | ext4_fc_unlock(inode->i_sb, *alloc_ctx);
|
| 255 | schedule();
|
| 256 | *alloc_ctx = ext4_fc_lock(inode->i_sb);
|
| 257 | }
|
| 258 | finish_wait(wq, &wait.wq_entry);
|
| 259 | }
|
| 260 | }
|
| 261 |
|
| 262 | static inline void ext4_fc_wake_inode_state(struct inode *inode, int bit)
|
| 263 | {
|
| 264 | wake_up_bit(ext4_inode_state_wait_word(inode),
|
| 265 | ext4_inode_state_wait_bit(bit));
|
| 266 | }
|
| 267 |
|
| 268 | static void ext4_fc_snap_stats_update_max(atomic64_t *stat, u64 value)
|
| 269 | {
|
| 270 | u64 old = atomic64_read(stat);
|
| 271 |
|
| 272 | while (value > old) {
|
| 273 | u64 prev = atomic64_cmpxchg(stat, old, value);
|
| 274 |
|
| 275 | if (prev == old)
|
| 276 | break;
|
| 277 | old = prev;
|
| 278 | }
|
| 279 | }
|
| 280 |
|
| 281 | |
| 282 | * Remove inode from fast commit list. If the inode is being committed
|
| 283 | * we wait until inode commit is done.
|
| 284 |
|
| 285 | void ext4_fc_del(struct inode *inode)
|
| 286 | {
|
| 287 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 288 | struct ext4_fc_dentry_update *fc_dentry;
|
| 289 | int alloc_ctx;
|
| 290 |
|
| 291 | if (ext4_fc_disabled(inode->i_sb))
|
| 292 | return;
|
| 293 |
|
| 294 | alloc_ctx = ext4_fc_lock(inode->i_sb);
|
| 295 | if (list_empty(&ei->i_fc_list) && list_empty(&ei->i_fc_dilist)) {
|
| 296 | ext4_fc_free_inode_snap(inode);
|
| 297 | ext4_fc_unlock(inode->i_sb, alloc_ctx);
|
| 298 | return;
|
| 299 | }
|
| 300 |
|
| 301 | |
| 302 | * Wait for ongoing fast commit to finish. We cannot remove the inode
|
| 303 | * from fast commit lists while it is being committed. If we wake from
|
| 304 | * FC_FLUSHING_DATA, re-check FC_COMMITTING before deleting because the
|
| 305 | * commit thread sets FC_COMMITTING only after clearing FLUSHING_DATA.
|
| 306 |
|
| 307 | for (;;) {
|
| 308 | ext4_fc_wait_inode_state(inode, EXT4_STATE_FC_COMMITTING,
|
| 309 | &alloc_ctx);
|
| 310 |
|
| 311 | if (!ext4_test_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA))
|
| 312 | break;
|
| 313 |
|
| 314 | ext4_fc_wait_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA,
|
| 315 | &alloc_ctx);
|
| 316 | }
|
| 317 |
|
| 318 | ext4_fc_free_inode_snap(inode);
|
| 319 | list_del_init(&ei->i_fc_list);
|
| 320 |
|
| 321 | |
| 322 | * Since this inode is getting removed, let's also remove all FC dentry
|
| 323 | * create references, since it is not needed to log it anyways.
|
| 324 |
|
| 325 | if (list_empty(&ei->i_fc_dilist)) {
|
| 326 | ext4_fc_unlock(inode->i_sb, alloc_ctx);
|
| 327 | return;
|
| 328 | }
|
| 329 |
|
| 330 | fc_dentry = list_first_entry(&ei->i_fc_dilist,
|
| 331 | struct ext4_fc_dentry_update,
|
| 332 | fcd_dilist);
|
| 333 | WARN_ON(fc_dentry->fcd_op != EXT4_FC_TAG_CREAT);
|
| 334 | list_del_init(&fc_dentry->fcd_list);
|
| 335 | list_del_init(&fc_dentry->fcd_dilist);
|
| 336 |
|
| 337 | WARN_ON(!list_empty(&ei->i_fc_dilist));
|
| 338 | ext4_fc_unlock(inode->i_sb, alloc_ctx);
|
| 339 |
|
| 340 | release_dentry_name_snapshot(&fc_dentry->fcd_name);
|
| 341 | kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry);
|
| 342 | }
|
| 343 |
|
| 344 | |
| 345 | * Mark file system as fast commit ineligible, and record latest
|
| 346 | * ineligible transaction tid. This means until the recorded
|
| 347 | * transaction, commit operation would result in a full jbd2 commit.
|
| 348 |
|
| 349 | void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handle)
|
| 350 | {
|
| 351 | struct ext4_sb_info *sbi = EXT4_SB(sb);
|
| 352 | tid_t tid;
|
| 353 | bool has_transaction = true;
|
| 354 | bool is_ineligible;
|
| 355 | int alloc_ctx;
|
| 356 |
|
| 357 | if (ext4_fc_disabled(sb))
|
| 358 | return;
|
| 359 |
|
| 360 | if (!IS_ERR_OR_NULL(handle))
|
| 361 | tid = handle->h_transaction->t_tid;
|
| 362 | else {
|
| 363 | read_lock(&sbi->s_journal->j_state_lock);
|
| 364 | if (sbi->s_journal->j_running_transaction)
|
| 365 | tid = sbi->s_journal->j_running_transaction->t_tid;
|
| 366 | else
|
| 367 | has_transaction = false;
|
| 368 | read_unlock(&sbi->s_journal->j_state_lock);
|
| 369 | }
|
| 370 | alloc_ctx = ext4_fc_lock(sb);
|
| 371 | is_ineligible = ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
|
| 372 | if (has_transaction && (!is_ineligible || tid_gt(tid, sbi->s_fc_ineligible_tid)))
|
| 373 | sbi->s_fc_ineligible_tid = tid;
|
| 374 | ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
|
| 375 | ext4_fc_unlock(sb, alloc_ctx);
|
| 376 | WARN_ON(reason >= EXT4_FC_REASON_MAX);
|
| 377 | sbi->s_fc_stats.fc_ineligible_reason_count[reason]++;
|
| 378 | }
|
| 379 |
|
| 380 | |
| 381 | * Generic fast commit tracking function. If this is the first time this we are
|
| 382 | * called after a full commit, we initialize fast commit fields and then call
|
| 383 | * __fc_track_fn() with update = 0. If we have already been called after a full
|
| 384 | * commit, we pass update = 1. Based on that, the track function can determine
|
| 385 | * if it needs to track a field for the first time or if it needs to just
|
| 386 | * update the previously tracked value.
|
| 387 | *
|
| 388 | * If enqueue is set, this function enqueues the inode in fast commit list.
|
| 389 |
|
| 390 | static int ext4_fc_track_template(
|
| 391 | handle_t *handle, struct inode *inode,
|
| 392 | int (*__fc_track_fn)(handle_t *handle, struct inode *, void *, bool),
|
| 393 | void *args, int enqueue)
|
| 394 | {
|
| 395 | bool update = false;
|
| 396 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 397 | struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
|
| 398 | tid_t tid = 0;
|
| 399 | int alloc_ctx;
|
| 400 | int ret;
|
| 401 |
|
| 402 | tid = handle->h_transaction->t_tid;
|
| 403 | spin_lock(&ei->i_fc_lock);
|
| 404 | if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING))
|
| 405 | ext4_set_inode_state(inode, EXT4_STATE_FC_REQUEUE);
|
| 406 | if (tid == ei->i_sync_tid) {
|
| 407 | update = true;
|
| 408 | } else {
|
| 409 | ext4_fc_reset_inode(inode);
|
| 410 | ei->i_sync_tid = tid;
|
| 411 | }
|
| 412 | ret = __fc_track_fn(handle, inode, args, update);
|
| 413 | spin_unlock(&ei->i_fc_lock);
|
| 414 | if (!enqueue)
|
| 415 | return ret;
|
| 416 |
|
| 417 | alloc_ctx = ext4_fc_lock(inode->i_sb);
|
| 418 | if (list_empty(&EXT4_I(inode)->i_fc_list))
|
| 419 | list_add_tail(&EXT4_I(inode)->i_fc_list,
|
| 420 | (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING ||
|
| 421 | sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) ?
|
| 422 | &sbi->s_fc_q[FC_Q_STAGING] :
|
| 423 | &sbi->s_fc_q[FC_Q_MAIN]);
|
| 424 | ext4_fc_unlock(inode->i_sb, alloc_ctx);
|
| 425 |
|
| 426 | return ret;
|
| 427 | }
|
| 428 |
|
| 429 | struct __track_dentry_update_args {
|
| 430 | struct dentry *dentry;
|
| 431 | int op;
|
| 432 | };
|
| 433 |
|
| 434 |
|
| 435 | static int __track_dentry_update(handle_t *handle, struct inode *inode,
|
| 436 | void *arg, bool update)
|
| 437 | {
|
| 438 | struct ext4_fc_dentry_update *node;
|
| 439 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 440 | struct __track_dentry_update_args *dentry_update =
|
| 441 | (struct __track_dentry_update_args *)arg;
|
| 442 | struct dentry *dentry = dentry_update->dentry;
|
| 443 | struct inode *dir = dentry->d_parent->d_inode;
|
| 444 | struct super_block *sb = inode->i_sb;
|
| 445 | struct ext4_sb_info *sbi = EXT4_SB(sb);
|
| 446 | int alloc_ctx;
|
| 447 |
|
| 448 | spin_unlock(&ei->i_fc_lock);
|
| 449 |
|
| 450 | if (IS_ENCRYPTED(dir)) {
|
| 451 | ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_ENCRYPTED_FILENAME,
|
| 452 | handle);
|
| 453 | spin_lock(&ei->i_fc_lock);
|
| 454 | return -EOPNOTSUPP;
|
| 455 | }
|
| 456 |
|
| 457 | node = kmem_cache_alloc(ext4_fc_dentry_cachep, GFP_NOFS);
|
| 458 | if (!node) {
|
| 459 | ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_NOMEM, handle);
|
| 460 | spin_lock(&ei->i_fc_lock);
|
| 461 | return -ENOMEM;
|
| 462 | }
|
| 463 |
|
| 464 | node->fcd_op = dentry_update->op;
|
| 465 | node->fcd_parent = dir->i_ino;
|
| 466 | node->fcd_ino = inode->i_ino;
|
| 467 | take_dentry_name_snapshot(&node->fcd_name, dentry);
|
| 468 | INIT_LIST_HEAD(&node->fcd_dilist);
|
| 469 | INIT_LIST_HEAD(&node->fcd_list);
|
| 470 | alloc_ctx = ext4_fc_lock(sb);
|
| 471 | if (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING ||
|
| 472 | sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING)
|
| 473 | list_add_tail(&node->fcd_list,
|
| 474 | &sbi->s_fc_dentry_q[FC_Q_STAGING]);
|
| 475 | else
|
| 476 | list_add_tail(&node->fcd_list, &sbi->s_fc_dentry_q[FC_Q_MAIN]);
|
| 477 |
|
| 478 | |
| 479 | * This helps us keep a track of all fc_dentry updates which is part of
|
| 480 | * this ext4 inode. So in case the inode is getting unlinked, before
|
| 481 | * even we get a chance to fsync, we could remove all fc_dentry
|
| 482 | * references while evicting the inode in ext4_fc_del().
|
| 483 | * Also with this, we don't need to loop over all the inodes in
|
| 484 | * sbi->s_fc_q to get the corresponding inode in
|
| 485 | * ext4_fc_commit_dentry_updates().
|
| 486 |
|
| 487 | if (dentry_update->op == EXT4_FC_TAG_CREAT) {
|
| 488 | WARN_ON(!list_empty(&ei->i_fc_dilist));
|
| 489 | list_add_tail(&node->fcd_dilist, &ei->i_fc_dilist);
|
| 490 | }
|
| 491 | ext4_fc_unlock(sb, alloc_ctx);
|
| 492 | spin_lock(&ei->i_fc_lock);
|
| 493 |
|
| 494 | return 0;
|
| 495 | }
|
| 496 |
|
| 497 | void __ext4_fc_track_unlink(handle_t *handle,
|
| 498 | struct inode *inode, struct dentry *dentry)
|
| 499 | {
|
| 500 | struct __track_dentry_update_args args;
|
| 501 | int ret;
|
| 502 |
|
| 503 | args.dentry = dentry;
|
| 504 | args.op = EXT4_FC_TAG_UNLINK;
|
| 505 |
|
| 506 | ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
|
| 507 | (void *)&args, 0);
|
| 508 | trace_ext4_fc_track_unlink(handle, inode, dentry, ret);
|
| 509 | }
|
| 510 |
|
| 511 | void ext4_fc_track_unlink(handle_t *handle, struct dentry *dentry)
|
| 512 | {
|
| 513 | struct inode *inode = d_inode(dentry);
|
| 514 |
|
| 515 | if (ext4_fc_eligible(inode->i_sb))
|
| 516 | __ext4_fc_track_unlink(handle, inode, dentry);
|
| 517 | }
|
| 518 |
|
| 519 | void __ext4_fc_track_link(handle_t *handle,
|
| 520 | struct inode *inode, struct dentry *dentry)
|
| 521 | {
|
| 522 | struct __track_dentry_update_args args;
|
| 523 | int ret;
|
| 524 |
|
| 525 | args.dentry = dentry;
|
| 526 | args.op = EXT4_FC_TAG_LINK;
|
| 527 |
|
| 528 | ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
|
| 529 | (void *)&args, 0);
|
| 530 | trace_ext4_fc_track_link(handle, inode, dentry, ret);
|
| 531 | }
|
| 532 |
|
| 533 | void ext4_fc_track_link(handle_t *handle, struct inode *inode,
|
| 534 | struct dentry *dentry)
|
| 535 | {
|
| 536 | if (ext4_fc_eligible(inode->i_sb))
|
| 537 | __ext4_fc_track_link(handle, inode, dentry);
|
| 538 | }
|
| 539 |
|
| 540 | void __ext4_fc_track_create(handle_t *handle, struct inode *inode,
|
| 541 | struct dentry *dentry)
|
| 542 | {
|
| 543 | struct __track_dentry_update_args args;
|
| 544 | int ret;
|
| 545 |
|
| 546 | args.dentry = dentry;
|
| 547 | args.op = EXT4_FC_TAG_CREAT;
|
| 548 |
|
| 549 | ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
|
| 550 | (void *)&args, 0);
|
| 551 | trace_ext4_fc_track_create(handle, inode, dentry, ret);
|
| 552 | }
|
| 553 |
|
| 554 | void ext4_fc_track_create(handle_t *handle, struct dentry *dentry)
|
| 555 | {
|
| 556 | struct inode *inode = d_inode(dentry);
|
| 557 |
|
| 558 | if (ext4_fc_eligible(inode->i_sb))
|
| 559 | __ext4_fc_track_create(handle, inode, dentry);
|
| 560 | }
|
| 561 |
|
| 562 |
|
| 563 | static int __track_inode(handle_t *handle, struct inode *inode, void *arg,
|
| 564 | bool update)
|
| 565 | {
|
| 566 | if (update)
|
| 567 | return -EEXIST;
|
| 568 |
|
| 569 | EXT4_I(inode)->i_fc_lblk_len = 0;
|
| 570 |
|
| 571 | return 0;
|
| 572 | }
|
| 573 |
|
| 574 | void ext4_fc_track_inode(handle_t *handle, struct inode *inode)
|
| 575 | {
|
| 576 | int ret;
|
| 577 |
|
| 578 | if (S_ISDIR(inode->i_mode))
|
| 579 | return;
|
| 580 |
|
| 581 | if (ext4_should_journal_data(inode)) {
|
| 582 | ext4_fc_mark_ineligible(inode->i_sb,
|
| 583 | EXT4_FC_REASON_INODE_JOURNAL_DATA, handle);
|
| 584 | return;
|
| 585 | }
|
| 586 |
|
| 587 | if (!ext4_fc_eligible(inode->i_sb))
|
| 588 | return;
|
| 589 |
|
| 590 | |
| 591 | * Fast commit snapshots inode state at commit time, so there's no need
|
| 592 | * to wait for EXT4_STATE_FC_COMMITTING here. If the inode is already
|
| 593 | * on the commit queue, ext4_fc_cleanup() will requeue it for the new
|
| 594 | * transaction once the current commit finishes.
|
| 595 |
|
| 596 |
|
| 597 | |
| 598 | * From this point on, this inode will not be committed either
|
| 599 | * by fast or full commit as long as the handle is open.
|
| 600 |
|
| 601 | ret = ext4_fc_track_template(handle, inode, __track_inode, NULL, 1);
|
| 602 | trace_ext4_fc_track_inode(handle, inode, ret);
|
| 603 | }
|
| 604 |
|
| 605 | struct __track_range_args {
|
| 606 | ext4_lblk_t start, end;
|
| 607 | };
|
| 608 |
|
| 609 |
|
| 610 | static int __track_range(handle_t *handle, struct inode *inode, void *arg,
|
| 611 | bool update)
|
| 612 | {
|
| 613 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 614 | ext4_lblk_t oldstart;
|
| 615 | struct __track_range_args *__arg =
|
| 616 | (struct __track_range_args *)arg;
|
| 617 |
|
| 618 | if (inode->i_ino < EXT4_FIRST_INO(inode->i_sb)) {
|
| 619 | ext4_debug("Special inode %llu being modified\n", inode->i_ino);
|
| 620 | return -ECANCELED;
|
| 621 | }
|
| 622 |
|
| 623 | oldstart = ei->i_fc_lblk_start;
|
| 624 |
|
| 625 | if (update && ei->i_fc_lblk_len > 0) {
|
| 626 | ei->i_fc_lblk_start = min(ei->i_fc_lblk_start, __arg->start);
|
| 627 | ei->i_fc_lblk_len =
|
| 628 | max(oldstart + ei->i_fc_lblk_len - 1, __arg->end) -
|
| 629 | ei->i_fc_lblk_start + 1;
|
| 630 | } else {
|
| 631 | ei->i_fc_lblk_start = __arg->start;
|
| 632 | ei->i_fc_lblk_len = __arg->end - __arg->start + 1;
|
| 633 | }
|
| 634 |
|
| 635 | return 0;
|
| 636 | }
|
| 637 |
|
| 638 | void ext4_fc_track_range(handle_t *handle, struct inode *inode, ext4_lblk_t start,
|
| 639 | ext4_lblk_t end)
|
| 640 | {
|
| 641 | struct __track_range_args args;
|
| 642 | int ret;
|
| 643 |
|
| 644 | if (S_ISDIR(inode->i_mode))
|
| 645 | return;
|
| 646 |
|
| 647 | if (!ext4_fc_eligible(inode->i_sb))
|
| 648 | return;
|
| 649 |
|
| 650 | if (ext4_has_inline_data(inode)) {
|
| 651 | ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_XATTR,
|
| 652 | handle);
|
| 653 | return;
|
| 654 | }
|
| 655 |
|
| 656 | args.start = start;
|
| 657 | args.end = end;
|
| 658 |
|
| 659 | ret = ext4_fc_track_template(handle, inode, __track_range, &args, 1);
|
| 660 |
|
| 661 | trace_ext4_fc_track_range(handle, inode, start, end, ret);
|
| 662 | }
|
| 663 |
|
| 664 | static void ext4_fc_submit_bh(struct super_block *sb, bool is_tail)
|
| 665 | {
|
| 666 | blk_opf_t write_flags = JBD2_JOURNAL_REQ_FLAGS;
|
| 667 | struct buffer_head *bh = EXT4_SB(sb)->s_fc_bh;
|
| 668 |
|
| 669 |
|
| 670 | if (test_opt(sb, BARRIER) && is_tail)
|
| 671 | write_flags |= REQ_FUA | REQ_PREFLUSH;
|
| 672 | lock_buffer(bh);
|
| 673 | set_buffer_dirty(bh);
|
| 674 | set_buffer_uptodate(bh);
|
| 675 | bh_submit(bh, REQ_OP_WRITE | write_flags, bh_end_write);
|
| 676 | EXT4_SB(sb)->s_fc_bh = NULL;
|
| 677 | }
|
| 678 |
|
| 679 |
|
| 680 |
|
| 681 | |
| 682 | * Allocate len bytes on a fast commit buffer.
|
| 683 | *
|
| 684 | * During the commit time this function is used to manage fast commit
|
| 685 | * block space. We don't split a fast commit log onto different
|
| 686 | * blocks. So this function makes sure that if there's not enough space
|
| 687 | * on the current block, the remaining space in the current block is
|
| 688 | * marked as unused by adding EXT4_FC_TAG_PAD tag. In that case,
|
| 689 | * new block is from jbd2 and CRC is updated to reflect the padding
|
| 690 | * we added.
|
| 691 |
|
| 692 | static u8 *ext4_fc_reserve_space(struct super_block *sb, int len, u32 *crc)
|
| 693 | {
|
| 694 | struct ext4_fc_tl tl;
|
| 695 | struct ext4_sb_info *sbi = EXT4_SB(sb);
|
| 696 | struct buffer_head *bh;
|
| 697 | int bsize = sbi->s_journal->j_blocksize;
|
| 698 | int ret, off = sbi->s_fc_bytes % bsize;
|
| 699 | int remaining;
|
| 700 | u8 *dst;
|
| 701 |
|
| 702 | |
| 703 | * If 'len' is too long to fit in any block alongside a PAD tlv, then we
|
| 704 | * cannot fulfill the request.
|
| 705 |
|
| 706 | if (len > bsize - EXT4_FC_TAG_BASE_LEN)
|
| 707 | return NULL;
|
| 708 |
|
| 709 | if (!sbi->s_fc_bh) {
|
| 710 | ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh);
|
| 711 | if (ret)
|
| 712 | return NULL;
|
| 713 | sbi->s_fc_bh = bh;
|
| 714 | }
|
| 715 | dst = sbi->s_fc_bh->b_data + off;
|
| 716 |
|
| 717 | |
| 718 | * Allocate the bytes in the current block if we can do so while still
|
| 719 | * leaving enough space for a PAD tlv.
|
| 720 |
|
| 721 | remaining = bsize - EXT4_FC_TAG_BASE_LEN - off;
|
| 722 | if (len <= remaining) {
|
| 723 | sbi->s_fc_bytes += len;
|
| 724 | return dst;
|
| 725 | }
|
| 726 |
|
| 727 | |
| 728 | * Else, terminate the current block with a PAD tlv, then allocate a new
|
| 729 | * block and allocate the bytes at the start of that new block.
|
| 730 |
|
| 731 |
|
| 732 | tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_PAD);
|
| 733 | tl.fc_len = cpu_to_le16(remaining);
|
| 734 | memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN);
|
| 735 | memset(dst + EXT4_FC_TAG_BASE_LEN, 0, remaining);
|
| 736 | *crc = ext4_chksum(*crc, sbi->s_fc_bh->b_data, bsize);
|
| 737 |
|
| 738 | ext4_fc_submit_bh(sb, false);
|
| 739 |
|
| 740 | ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh);
|
| 741 | if (ret)
|
| 742 | return NULL;
|
| 743 | sbi->s_fc_bh = bh;
|
| 744 | sbi->s_fc_bytes += bsize - off + len;
|
| 745 | return sbi->s_fc_bh->b_data;
|
| 746 | }
|
| 747 |
|
| 748 | |
| 749 | * Complete a fast commit by writing tail tag.
|
| 750 | *
|
| 751 | * Writing tail tag marks the end of a fast commit. In order to guarantee
|
| 752 | * atomicity, after writing tail tag, even if there's space remaining
|
| 753 | * in the block, next commit shouldn't use it. That's why tail tag
|
| 754 | * has the length as that of the remaining space on the block.
|
| 755 |
|
| 756 | static int ext4_fc_write_tail(struct super_block *sb, u32 crc)
|
| 757 | {
|
| 758 | struct ext4_sb_info *sbi = EXT4_SB(sb);
|
| 759 | struct ext4_fc_tl tl;
|
| 760 | struct ext4_fc_tail tail;
|
| 761 | int off, bsize = sbi->s_journal->j_blocksize;
|
| 762 | u8 *dst;
|
| 763 |
|
| 764 | |
| 765 | * ext4_fc_reserve_space takes care of allocating an extra block if
|
| 766 | * there's no enough space on this block for accommodating this tail.
|
| 767 |
|
| 768 | dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + sizeof(tail), &crc);
|
| 769 | if (!dst)
|
| 770 | return -ENOSPC;
|
| 771 |
|
| 772 | off = sbi->s_fc_bytes % bsize;
|
| 773 |
|
| 774 | tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_TAIL);
|
| 775 | tl.fc_len = cpu_to_le16(bsize - off + sizeof(struct ext4_fc_tail));
|
| 776 | sbi->s_fc_bytes = round_up(sbi->s_fc_bytes, bsize);
|
| 777 |
|
| 778 | memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN);
|
| 779 | dst += EXT4_FC_TAG_BASE_LEN;
|
| 780 | tail.fc_tid = cpu_to_le32(sbi->s_journal->j_running_transaction->t_tid);
|
| 781 | memcpy(dst, &tail.fc_tid, sizeof(tail.fc_tid));
|
| 782 | dst += sizeof(tail.fc_tid);
|
| 783 | crc = ext4_chksum(crc, sbi->s_fc_bh->b_data,
|
| 784 | dst - (u8 *)sbi->s_fc_bh->b_data);
|
| 785 | tail.fc_crc = cpu_to_le32(crc);
|
| 786 | memcpy(dst, &tail.fc_crc, sizeof(tail.fc_crc));
|
| 787 | dst += sizeof(tail.fc_crc);
|
| 788 | memset(dst, 0, bsize - off);
|
| 789 |
|
| 790 | ext4_fc_submit_bh(sb, true);
|
| 791 |
|
| 792 | return 0;
|
| 793 | }
|
| 794 |
|
| 795 | |
| 796 | * Adds tag, length, value and updates CRC. Returns true if tlv was added.
|
| 797 | * Returns false if there's not enough space.
|
| 798 |
|
| 799 | static bool ext4_fc_add_tlv(struct super_block *sb, u16 tag, u16 len, u8 *val,
|
| 800 | u32 *crc)
|
| 801 | {
|
| 802 | struct ext4_fc_tl tl;
|
| 803 | u8 *dst;
|
| 804 |
|
| 805 | dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + len, crc);
|
| 806 | if (!dst)
|
| 807 | return false;
|
| 808 |
|
| 809 | tl.fc_tag = cpu_to_le16(tag);
|
| 810 | tl.fc_len = cpu_to_le16(len);
|
| 811 |
|
| 812 | memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN);
|
| 813 | memcpy(dst + EXT4_FC_TAG_BASE_LEN, val, len);
|
| 814 |
|
| 815 | return true;
|
| 816 | }
|
| 817 |
|
| 818 |
|
| 819 | static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u32 *crc,
|
| 820 | struct ext4_fc_dentry_update *fc_dentry)
|
| 821 | {
|
| 822 | struct ext4_fc_dentry_info fcd;
|
| 823 | struct ext4_fc_tl tl;
|
| 824 | int dlen = fc_dentry->fcd_name.name.len;
|
| 825 | u8 *dst = ext4_fc_reserve_space(sb,
|
| 826 | EXT4_FC_TAG_BASE_LEN + sizeof(fcd) + dlen, crc);
|
| 827 |
|
| 828 | if (!dst)
|
| 829 | return false;
|
| 830 |
|
| 831 | fcd.fc_parent_ino = cpu_to_le32(fc_dentry->fcd_parent);
|
| 832 | fcd.fc_ino = cpu_to_le32(fc_dentry->fcd_ino);
|
| 833 | tl.fc_tag = cpu_to_le16(fc_dentry->fcd_op);
|
| 834 | tl.fc_len = cpu_to_le16(sizeof(fcd) + dlen);
|
| 835 | memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN);
|
| 836 | dst += EXT4_FC_TAG_BASE_LEN;
|
| 837 | memcpy(dst, &fcd, sizeof(fcd));
|
| 838 | dst += sizeof(fcd);
|
| 839 | memcpy(dst, fc_dentry->fcd_name.name.name, dlen);
|
| 840 |
|
| 841 | return true;
|
| 842 | }
|
| 843 |
|
| 844 | struct ext4_fc_range {
|
| 845 | struct list_head list;
|
| 846 | u16 tag;
|
| 847 | ext4_lblk_t lblk;
|
| 848 | ext4_lblk_t len;
|
| 849 | ext4_fsblk_t pblk;
|
| 850 | bool unwritten;
|
| 851 | };
|
| 852 |
|
| 853 | struct ext4_fc_inode_snap {
|
| 854 | struct list_head data_list;
|
| 855 | unsigned int inode_len;
|
| 856 | u8 inode_buf[];
|
| 857 | };
|
| 858 |
|
| 859 | |
| 860 | * Writes inode in the fast commit space under TLV with tag @tag.
|
| 861 | * Returns 0 on success, error on failure.
|
| 862 |
|
| 863 | static int ext4_fc_write_inode(struct inode *inode, u32 *crc)
|
| 864 | {
|
| 865 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 866 | struct ext4_fc_inode_snap *snap = ei->i_fc_snap;
|
| 867 | struct ext4_fc_snap_stats *stats =
|
| 868 | &EXT4_SB(inode->i_sb)->s_fc_snap_stats;
|
| 869 | struct ext4_fc_inode fc_inode;
|
| 870 | struct ext4_fc_tl tl;
|
| 871 | u8 *dst;
|
| 872 | u8 *src;
|
| 873 | int inode_len;
|
| 874 | int ret;
|
| 875 |
|
| 876 | if (!snap) {
|
| 877 | atomic64_inc(&stats->snap_fail_no_snap);
|
| 878 | return -ECANCELED;
|
| 879 | }
|
| 880 |
|
| 881 | src = snap->inode_buf;
|
| 882 | inode_len = snap->inode_len;
|
| 883 | if (!src || inode_len == 0) {
|
| 884 | atomic64_inc(&stats->snap_fail_no_snap);
|
| 885 | return -ECANCELED;
|
| 886 | }
|
| 887 |
|
| 888 | fc_inode.fc_ino = cpu_to_le32(inode->i_ino);
|
| 889 | tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_INODE);
|
| 890 | tl.fc_len = cpu_to_le16(inode_len + sizeof(fc_inode.fc_ino));
|
| 891 |
|
| 892 | ret = -ECANCELED;
|
| 893 | dst = ext4_fc_reserve_space(inode->i_sb,
|
| 894 | EXT4_FC_TAG_BASE_LEN + inode_len + sizeof(fc_inode.fc_ino), crc);
|
| 895 | if (!dst)
|
| 896 | goto err;
|
| 897 |
|
| 898 | memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN);
|
| 899 | dst += EXT4_FC_TAG_BASE_LEN;
|
| 900 | memcpy(dst, &fc_inode, sizeof(fc_inode));
|
| 901 | dst += sizeof(fc_inode);
|
| 902 | memcpy(dst, src, inode_len);
|
| 903 | ret = 0;
|
| 904 | err:
|
| 905 | return ret;
|
| 906 | }
|
| 907 |
|
| 908 | |
| 909 | * Writes updated data ranges for the inode in question. Updates CRC.
|
| 910 | * Returns 0 on success, error otherwise.
|
| 911 |
|
| 912 | static int ext4_fc_write_inode_data(struct inode *inode, u32 *crc)
|
| 913 | {
|
| 914 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 915 | struct ext4_fc_inode_snap *snap = ei->i_fc_snap;
|
| 916 | struct ext4_fc_snap_stats *stats =
|
| 917 | &EXT4_SB(inode->i_sb)->s_fc_snap_stats;
|
| 918 | struct ext4_fc_add_range fc_ext;
|
| 919 | struct ext4_fc_del_range lrange;
|
| 920 | struct ext4_extent *ex;
|
| 921 | struct ext4_fc_range *range;
|
| 922 |
|
| 923 | if (!snap) {
|
| 924 | atomic64_inc(&stats->snap_fail_no_snap);
|
| 925 | return -ECANCELED;
|
| 926 | }
|
| 927 |
|
| 928 | list_for_each_entry(range, &snap->data_list, list) {
|
| 929 | if (range->tag == EXT4_FC_TAG_DEL_RANGE) {
|
| 930 | lrange.fc_ino = cpu_to_le32(inode->i_ino);
|
| 931 | lrange.fc_lblk = cpu_to_le32(range->lblk);
|
| 932 | lrange.fc_len = cpu_to_le32(range->len);
|
| 933 | if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_DEL_RANGE,
|
| 934 | sizeof(lrange), (u8 *)&lrange, crc))
|
| 935 | return -ENOSPC;
|
| 936 | continue;
|
| 937 | }
|
| 938 |
|
| 939 | fc_ext.fc_ino = cpu_to_le32(inode->i_ino);
|
| 940 | ex = (struct ext4_extent *)&fc_ext.fc_ex;
|
| 941 | ex->ee_block = cpu_to_le32(range->lblk);
|
| 942 | ex->ee_len = cpu_to_le16(range->len);
|
| 943 | ext4_ext_store_pblock(ex, range->pblk);
|
| 944 | if (range->unwritten)
|
| 945 | ext4_ext_mark_unwritten(ex);
|
| 946 | else
|
| 947 | ext4_ext_mark_initialized(ex);
|
| 948 |
|
| 949 | if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_ADD_RANGE,
|
| 950 | sizeof(fc_ext), (u8 *)&fc_ext, crc))
|
| 951 | return -ENOSPC;
|
| 952 | }
|
| 953 |
|
| 954 | return 0;
|
| 955 | }
|
| 956 |
|
| 957 | static void ext4_fc_free_ranges(struct list_head *head)
|
| 958 | {
|
| 959 | struct ext4_fc_range *range, *range_n;
|
| 960 |
|
| 961 | list_for_each_entry_safe(range, range_n, head, list) {
|
| 962 | list_del(&range->list);
|
| 963 | kmem_cache_free(ext4_fc_range_cachep, range);
|
| 964 | }
|
| 965 | }
|
| 966 |
|
| 967 | static void ext4_fc_free_inode_snap(struct inode *inode)
|
| 968 | {
|
| 969 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 970 | struct ext4_fc_inode_snap *snap = ei->i_fc_snap;
|
| 971 |
|
| 972 | if (!snap)
|
| 973 | return;
|
| 974 |
|
| 975 | ext4_fc_free_ranges(&snap->data_list);
|
| 976 | kfree(snap);
|
| 977 | ei->i_fc_snap = NULL;
|
| 978 | }
|
| 979 |
|
| 980 | static int ext4_fc_snapshot_inode_data(struct inode *inode,
|
| 981 | struct list_head *ranges,
|
| 982 | unsigned int nr_ranges_total,
|
| 983 | unsigned int *nr_rangesp,
|
| 984 | int *snap_err)
|
| 985 | {
|
| 986 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 987 | struct ext4_fc_snap_stats *stats =
|
| 988 | &EXT4_SB(inode->i_sb)->s_fc_snap_stats;
|
| 989 | ext4_lblk_t start_lblk, end_lblk, cur_lblk;
|
| 990 | unsigned int nr_ranges = 0;
|
| 991 |
|
| 992 | spin_lock(&ei->i_fc_lock);
|
| 993 | if (ei->i_fc_lblk_len == 0) {
|
| 994 | spin_unlock(&ei->i_fc_lock);
|
| 995 | if (nr_rangesp)
|
| 996 | *nr_rangesp = 0;
|
| 997 | return 0;
|
| 998 | }
|
| 999 | start_lblk = ei->i_fc_lblk_start;
|
| 1000 | end_lblk = ei->i_fc_lblk_start + ei->i_fc_lblk_len - 1;
|
| 1001 | ei->i_fc_lblk_len = 0;
|
| 1002 | spin_unlock(&ei->i_fc_lock);
|
| 1003 |
|
| 1004 | cur_lblk = start_lblk;
|
| 1005 | ext4_debug("snapshot data ranges %u-%u for inode %llu\n",
|
| 1006 | start_lblk, end_lblk,
|
| 1007 | (unsigned long long)inode->i_ino);
|
| 1008 |
|
| 1009 | while (cur_lblk <= end_lblk) {
|
| 1010 | struct extent_status es;
|
| 1011 | struct ext4_fc_range *range;
|
| 1012 | ext4_lblk_t len;
|
| 1013 | u64 remaining = (u64)end_lblk - cur_lblk + 1;
|
| 1014 |
|
| 1015 | if (!ext4_es_lookup_extent(inode, cur_lblk, NULL, &es, NULL)) {
|
| 1016 | atomic64_inc(&stats->snap_fail_es_miss);
|
| 1017 | ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_ES_MISS);
|
| 1018 | return -EAGAIN;
|
| 1019 | }
|
| 1020 |
|
| 1021 | if (ext4_es_is_delayed(&es)) {
|
| 1022 | atomic64_inc(&stats->snap_fail_es_delayed);
|
| 1023 | ext4_fc_set_snap_err(snap_err,
|
| 1024 | EXT4_FC_SNAP_ERR_ES_DELAYED);
|
| 1025 | return -EAGAIN;
|
| 1026 | }
|
| 1027 |
|
| 1028 | len = es.es_len - (cur_lblk - es.es_lblk);
|
| 1029 | if (len > remaining)
|
| 1030 | len = remaining;
|
| 1031 | if (len == 0) {
|
| 1032 | cur_lblk++;
|
| 1033 | continue;
|
| 1034 | }
|
| 1035 |
|
| 1036 | if (nr_ranges_total + nr_ranges >= EXT4_FC_SNAPSHOT_MAX_RANGES) {
|
| 1037 | atomic64_inc(&stats->snap_fail_ranges_cap);
|
| 1038 | ext4_fc_set_snap_err(snap_err,
|
| 1039 | EXT4_FC_SNAP_ERR_RANGES_CAP);
|
| 1040 | return -E2BIG;
|
| 1041 | }
|
| 1042 |
|
| 1043 | range = kmem_cache_alloc(ext4_fc_range_cachep, GFP_NOFS);
|
| 1044 | if (!range) {
|
| 1045 | atomic64_inc(&stats->snap_fail_nomem);
|
| 1046 | ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_NOMEM);
|
| 1047 | return -ENOMEM;
|
| 1048 | }
|
| 1049 | nr_ranges++;
|
| 1050 |
|
| 1051 | range->lblk = cur_lblk;
|
| 1052 | range->len = len;
|
| 1053 | range->pblk = 0;
|
| 1054 | range->unwritten = false;
|
| 1055 |
|
| 1056 | if (ext4_es_is_hole(&es)) {
|
| 1057 | range->tag = EXT4_FC_TAG_DEL_RANGE;
|
| 1058 | } else if (ext4_es_is_written(&es) ||
|
| 1059 | ext4_es_is_unwritten(&es)) {
|
| 1060 | unsigned int max;
|
| 1061 |
|
| 1062 | range->tag = EXT4_FC_TAG_ADD_RANGE;
|
| 1063 | range->pblk = ext4_es_pblock(&es) +
|
| 1064 | (cur_lblk - es.es_lblk);
|
| 1065 | range->unwritten = ext4_es_is_unwritten(&es);
|
| 1066 |
|
| 1067 | max = range->unwritten ? EXT_UNWRITTEN_MAX_LEN :
|
| 1068 | EXT_INIT_MAX_LEN;
|
| 1069 | if (range->len > max)
|
| 1070 | range->len = max;
|
| 1071 | } else {
|
| 1072 | kmem_cache_free(ext4_fc_range_cachep, range);
|
| 1073 | atomic64_inc(&stats->snap_fail_es_other);
|
| 1074 | ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_ES_OTHER);
|
| 1075 | return -EAGAIN;
|
| 1076 | }
|
| 1077 |
|
| 1078 | INIT_LIST_HEAD(&range->list);
|
| 1079 | list_add_tail(&range->list, ranges);
|
| 1080 |
|
| 1081 | if ((u64)range->len > (u64)end_lblk - cur_lblk)
|
| 1082 | break;
|
| 1083 |
|
| 1084 | cur_lblk += range->len;
|
| 1085 | }
|
| 1086 |
|
| 1087 | if (nr_rangesp)
|
| 1088 | *nr_rangesp = nr_ranges;
|
| 1089 | return 0;
|
| 1090 | }
|
| 1091 |
|
| 1092 | static int ext4_fc_snapshot_inode(struct inode *inode,
|
| 1093 | unsigned int nr_ranges_total,
|
| 1094 | unsigned int *nr_rangesp, int *snap_err)
|
| 1095 | {
|
| 1096 | struct ext4_inode_info *ei = EXT4_I(inode);
|
| 1097 | struct ext4_fc_snap_stats *stats =
|
| 1098 | &EXT4_SB(inode->i_sb)->s_fc_snap_stats;
|
| 1099 | struct ext4_fc_inode_snap *snap;
|
| 1100 | int inode_len = EXT4_GOOD_OLD_INODE_SIZE;
|
| 1101 | struct ext4_iloc iloc;
|
| 1102 | LIST_HEAD(ranges);
|
| 1103 | unsigned int nr_ranges = 0;
|
| 1104 | int ret;
|
| 1105 | int alloc_ctx;
|
| 1106 |
|
| 1107 | ret = ext4_get_inode_loc_noio(inode, &iloc);
|
| 1108 | if (ret) {
|
| 1109 | atomic64_inc(&stats->snap_fail_inode_loc);
|
| 1110 | ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_INODE_LOC);
|
| 1111 | return ret;
|
| 1112 | }
|
| 1113 |
|
| 1114 | if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA))
|
| 1115 | inode_len = EXT4_INODE_SIZE(inode->i_sb);
|
| 1116 | else if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE)
|
| 1117 | inode_len += ei->i_extra_isize;
|
| 1118 |
|
| 1119 | snap = kmalloc_flex(*snap, inode_buf, inode_len, GFP_NOFS);
|
| 1120 | if (!snap) {
|
| 1121 | atomic64_inc(&stats->snap_fail_nomem);
|
| 1122 | ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_NOMEM);
|
| 1123 | brelse(iloc.bh);
|
| 1124 | return -ENOMEM;
|
| 1125 | }
|
| 1126 | INIT_LIST_HEAD(&snap->data_list);
|
| 1127 | snap->inode_len = inode_len;
|
| 1128 |
|
| 1129 | memcpy(snap->inode_buf, (u8 *)ext4_raw_inode(&iloc), inode_len);
|
| 1130 | brelse(iloc.bh);
|
| 1131 |
|
| 1132 | ret = ext4_fc_snapshot_inode_data(inode, &ranges, nr_ranges_total,
|
| 1133 | &nr_ranges, snap_err);
|
| 1134 | if (ret) {
|
| 1135 | kfree(snap);
|
| 1136 | ext4_fc_free_ranges(&ranges);
|
| 1137 | return ret;
|
| 1138 | }
|
| 1139 |
|
| 1140 | alloc_ctx = ext4_fc_lock(inode->i_sb);
|
| 1141 | ext4_fc_free_inode_snap(inode);
|
| 1142 | ei->i_fc_snap = snap;
|
| 1143 | list_splice_tail_init(&ranges, &snap->data_list);
|
| 1144 | ext4_fc_unlock(inode->i_sb, alloc_ctx);
|
| 1145 |
|
| 1146 | atomic64_inc(&stats->snap_inodes);
|
| 1147 | atomic64_add(nr_ranges, &stats->snap_ranges);
|
| 1148 | if (nr_rangesp)
|
| 1149 | *nr_rangesp = nr_ranges;
|
| 1150 | return 0;
|
| 1151 | }
|
| 1152 |
|
| 1153 |
|
| 1154 | static int ext4_fc_flush_data(journal_t *journal)
|
| 1155 | {
|
| 1156 | struct super_block *sb = journal->j_private;
|
| 1157 | struct ext4_sb_info *sbi = EXT4_SB(sb);
|
| 1158 | struct ext4_inode_info *ei;
|
| 1159 | int ret = 0;
|
| 1160 |
|
| 1161 | list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
|
| 1162 | ret = jbd2_submit_inode_data(journal, READ_ONCE(ei->jinode));
|
| 1163 | if (ret)
|
| 1164 | return ret;
|
| 1165 | }
|
| 1166 |
|
| 1167 | list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
|
| 1168 | ret = jbd2_wait_inode_data(journal, READ_ONCE(ei->jinode));
|
| 1169 | if (ret)
|
| 1170 | return ret;
|
| 1171 | }
|
| 1172 |
|
| 1173 | return 0;
|
| 1174 | }
|
| 1175 |
|
| 1176 |
|
| 1177 | static int ext4_fc_commit_dentry_updates(journal_t *journal, u32 *crc)
|
| 1178 | {
|
| 1179 | struct super_block *sb = journal->j_private;
|
| 1180 | struct ext4_sb_info *sbi = EXT4_SB(sb);
|
| 1181 | struct ext4_fc_dentry_update *fc_dentry, *fc_dentry_n;
|
| 1182 | struct inode *inode;
|
| 1183 | struct ext4_inode_info *ei;
|
| 1184 | int ret;
|
| 1185 |
|
| 1186 | if (list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN]))
|
| 1187 | return 0;
|
| 1188 | list_for_each_entry_safe(fc_dentry, fc_dentry_n,
|
| 1189 | &sbi->s_fc_dentry_q[FC_Q_MAIN], fcd_list) {
|
| 1190 | if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) {
|
| 1191 | if (!ext4_fc_add_dentry_tlv(sb, crc, fc_dentry))
|
| 1192 | return -ENOSPC;
|
| 1193 | continue;
|
| 1194 | }
|
| 1195 | |
| 1196 | * With fcd_dilist we need not loop in sbi->s_fc_q to get the
|
| 1197 | * corresponding inode. Also, the corresponding inode could have been
|
| 1198 | * deleted, in which case, we don't need to do anything.
|
| 1199 |
|
| 1200 | if (list_empty(&fc_dentry->fcd_dilist))
|
| 1201 | continue;
|
| 1202 | |
| 1203 | * For EXT4_FC_TAG_CREAT, fcd_dilist is linked on the created
|
| 1204 | * inode's i_fc_dilist list (kept singular), so we can recover the
|
| 1205 | * inode through it.
|
| 1206 |
|
| 1207 | ei = list_first_entry(&fc_dentry->fcd_dilist,
|
| 1208 | struct ext4_inode_info, i_fc_dilist);
|
| 1209 | inode = &ei->vfs_inode;
|
| 1210 | WARN_ON(inode->i_ino != fc_dentry->fcd_ino);
|
| 1211 |
|
| 1212 | |
| 1213 | * We first write the inode and then the create dirent. This
|
| 1214 | * allows the recovery code to create an unnamed inode first
|
| 1215 | * and then link it to a directory entry. This allows us
|
| 1216 | * to use namei.c routines almost as is and simplifies
|
| 1217 | * the recovery code.
|
| 1218 |
|
| 1219 | ret = ext4_fc_write_inode(inode, crc);
|
| 1220 | if (ret)
|
| 1221 | return ret;
|
| 1222 | ret = ext4_fc_write_inode_data(inode, crc);
|
| 1223 | if (ret)
|
| 1224 | return ret;
|
| 1225 | if (!ext4_fc_add_dentry_tlv(sb, crc, fc_dentry))
|
| 1226 | return -ENOSPC;
|
| 1227 | }
|
| 1228 | return 0;
|
| 1229 | }
|
| 1230 |
|
| 1231 | static int ext4_fc_alloc_snapshot_inodes(struct super_block *sb,
|
| 1232 | struct inode ***inodesp,
|
| 1233 | unsigned int *nr_inodesp);
|
| 1234 |
|
| 1235 | static int ext4_fc_snapshot_inodes(journal_t *journal, struct inode **inodes,
|
| 1236 | unsigned int inodes_size,
|
| 1237 | unsigned int *nr_inodesp,
|
| 1238 | unsigned int *nr_rangesp,
|
| 1239 | int *snap_err)
|
| 1240 | {
|
| 1241 | struct super_block *sb = journal->j_private;
|
| 1242 | struct ext4_sb_info *sbi = EXT4_SB(sb);
|
| 1243 | struct ext4_inode_info *iter;
|
| 1244 | struct ext4_fc_dentry_update *fc_dentry;
|
| 1245 | unsigned int i = 0;
|
| 1246 | unsigned int idx;
|
| 1247 | unsigned int nr_ranges = 0;
|
| 1248 | int ret = 0;
|
| 1249 | int alloc_ctx;
|
| 1250 |
|
| 1251 | alloc_ctx = ext4_fc_lock(sb);
|
| 1252 | list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
|
| 1253 | if (i >= inodes_size) {
|
| 1254 | atomic64_inc(&sbi->s_fc_snap_stats.snap_fail_inodes_cap);
|
| 1255 | ext4_fc_set_snap_err(snap_err,
|
| 1256 | EXT4_FC_SNAP_ERR_INODES_CAP);
|
| 1257 | ret = -E2BIG;
|
| 1258 | goto unlock;
|
| 1259 | }
|
| 1260 | inodes[i++] = &iter->vfs_inode;
|
| 1261 | }
|
| 1262 |
|
| 1263 | list_for_each_entry(fc_dentry, &sbi->s_fc_dentry_q[FC_Q_MAIN], fcd_list) {
|
| 1264 | struct ext4_inode_info *ei;
|
| 1265 | struct inode *inode;
|
| 1266 |
|
| 1267 | if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT)
|
| 1268 | continue;
|
| 1269 | if (list_empty(&fc_dentry->fcd_dilist))
|
| 1270 | continue;
|
| 1271 |
|
| 1272 |
|
| 1273 | ei = list_first_entry(&fc_dentry->fcd_dilist,
|
| 1274 | struct ext4_inode_info, i_fc_dilist);
|
| 1275 | inode = &ei->vfs_inode;
|
| 1276 | if (!list_empty(&ei->i_fc_list))
|
| 1277 | continue;
|
| 1278 |
|
| 1279 | if (i >= inodes_size) {
|
| 1280 | atomic64_inc(&sbi->s_fc_snap_stats.snap_fail_inodes_cap);
|
| 1281 | ext4_fc_set_snap_err(snap_err,
|
| 1282 | EXT4_FC_SNAP_ERR_INODES_CAP);
|
| 1283 | ret = -E2BIG;
|
| 1284 | goto unlock;
|
| 1285 | }
|
| 1286 | |
| 1287 | * Create-only inodes may only be referenced via fcd_dilist and
|
| 1288 | * not appear on s_fc_q[MAIN]. They may hit the last iput while
|
| 1289 | * we are snapshotting, but inode eviction calls ext4_fc_del(),
|
| 1290 | * which waits for FC_COMMITTING to clear. Mark them FC_COMMITTING
|
| 1291 | * so the inode stays pinned and the snapshot stays valid until
|
| 1292 | * ext4_fc_cleanup().
|
| 1293 |
|
| 1294 | ext4_set_inode_state(inode, EXT4_STATE_FC_COMMITTING);
|
| 1295 | inodes[i++] = inode;
|
| 1296 | }
|
| 1297 | unlock:
|
| 1298 | ext4_fc_unlock(sb, alloc_ctx);
|
| 1299 |
|
| 1300 | if (ret)
|
| 1301 | return ret;
|
| 1302 |
|
| 1303 | for (idx = 0; idx < i; idx++) {
|
| 1304 | unsigned int inode_ranges = 0;
|
| 1305 |
|
| 1306 | ret = ext4_fc_snapshot_inode(inodes[idx], nr_ranges,
|
| 1307 | &inode_ranges, snap_err);
|
| 1308 | if (ret)
|
| 1309 | break;
|
| 1310 | nr_ranges += inode_ranges;
|
| 1311 | }
|
| 1312 |
|
| 1313 | if (nr_inodesp)
|
| 1314 | *nr_inodesp = idx;
|
| 1315 | if (nr_rangesp)
|
| 1316 | *nr_rangesp = nr_ranges;
|
| 1317 | return ret;
|
| 1318 | }
|
| 1319 |
|
| 1320 | static int ext4_fc_perform_commit(journal_t *journal, tid_t commit_tid)
|
| 1321 | {
|
| 1322 | struct super_block *sb = journal->j_private;
|
| 1323 | struct ext4_sb_info *sbi = EXT4_SB(sb);
|
| 1324 | struct ext4_fc_snap_stats *snap_stats = &sbi->s_fc_snap_stats;
|
| 1325 | struct ext4_inode_info *iter;
|
| 1326 | struct ext4_fc_head head;
|
| 1327 | struct inode *inode;
|
| 1328 | struct inode **inodes;
|
| 1329 | unsigned int inodes_size;
|
| 1330 | unsigned int snap_inodes = 0;
|
| 1331 | unsigned int snap_ranges = 0;
|
| 1332 | int snap_err = EXT4_FC_SNAP_ERR_NONE;
|
| 1333 | struct blk_plug plug;
|
| 1334 | int ret = 0;
|
| 1335 | u32 crc = 0;
|
| 1336 | int alloc_ctx;
|
| 1337 | ktime_t lock_start;
|
| 1338 | u64 locked_ns;
|
| 1339 |
|
| 1340 | |
| 1341 | * Step 1: Mark all inodes on s_fc_q[MAIN] with
|
| 1342 | * EXT4_STATE_FC_FLUSHING_DATA. This prevents these inodes from being
|
| 1343 | * freed until the data flush is over.
|
| 1344 |
|
| 1345 | alloc_ctx = ext4_fc_lock(sb);
|
| 1346 | list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
|
| 1347 | ext4_set_inode_state(&iter->vfs_inode,
|
| 1348 | EXT4_STATE_FC_FLUSHING_DATA);
|
| 1349 | }
|
| 1350 | ext4_fc_unlock(sb, alloc_ctx);
|
| 1351 |
|
| 1352 |
|
| 1353 | ret = ext4_fc_flush_data(journal);
|
| 1354 |
|
| 1355 | |
| 1356 | * Step 3: Clear EXT4_STATE_FC_FLUSHING_DATA flag, before returning
|
| 1357 | * any error from step 2. This ensures that waiters waiting on
|
| 1358 | * EXT4_STATE_FC_FLUSHING_DATA can resume.
|
| 1359 |
|
| 1360 | alloc_ctx = ext4_fc_lock(sb);
|
| 1361 | list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
|
| 1362 | ext4_clear_inode_state(&iter->vfs_inode,
|
| 1363 | EXT4_STATE_FC_FLUSHING_DATA);
|
| 1364 | ext4_fc_wake_inode_state(&iter->vfs_inode,
|
| 1365 | EXT4_STATE_FC_FLUSHING_DATA);
|
| 1366 | }
|
| 1367 |
|
| 1368 | |
| 1369 | * Make sure clearing of EXT4_STATE_FC_FLUSHING_DATA is visible before
|
| 1370 | * the waiter checks the bit. Pairs with implicit barrier in
|
| 1371 | * prepare_to_wait() in ext4_fc_del().
|
| 1372 |
|
| 1373 | smp_mb();
|
| 1374 | ext4_fc_unlock(sb, alloc_ctx);
|
| 1375 |
|
| 1376 | |
| 1377 | * If we encountered error in Step 2, return it now after clearing
|
| 1378 | * EXT4_STATE_FC_FLUSHING_DATA bit.
|
| 1379 |
|
| 1380 | if (ret)
|
| 1381 | return ret;
|
| 1382 |
|
| 1383 | ret = ext4_fc_alloc_snapshot_inodes(sb, &inodes, &inodes_size);
|
| 1384 | if (ret) {
|
| 1385 | if (ret == -E2BIG)
|
| 1386 | atomic64_inc(&snap_stats->snap_fail_inodes_cap);
|
| 1387 | else if (ret == -ENOMEM)
|
| 1388 | atomic64_inc(&snap_stats->snap_fail_nomem);
|
| 1389 | return ret;
|
| 1390 | }
|
| 1391 |
|
| 1392 |
|
| 1393 | jbd2_journal_lock_updates(journal);
|
| 1394 | lock_start = ktime_get();
|
| 1395 | |
| 1396 | * The journal is now locked. No more handles can start and all the
|
| 1397 | * previous handles are now drained. Snapshotting happens in this
|
| 1398 | * window so log writing can consume only stable snapshots without
|
| 1399 | * doing logical-to-physical mapping.
|
| 1400 |
|
| 1401 | alloc_ctx = ext4_fc_lock(sb);
|
| 1402 | list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
|
| 1403 | ext4_set_inode_state(&iter->vfs_inode,
|
| 1404 | EXT4_STATE_FC_COMMITTING);
|
| 1405 | }
|
| 1406 | ext4_fc_unlock(sb, alloc_ctx);
|
| 1407 |
|
| 1408 | ret = ext4_fc_snapshot_inodes(journal, inodes, inodes_size,
|
| 1409 | &snap_inodes, &snap_ranges, &snap_err);
|
| 1410 | jbd2_journal_unlock_updates(journal);
|
| 1411 | locked_ns = ktime_to_ns(ktime_sub(ktime_get(), lock_start));
|
| 1412 | atomic64_add(locked_ns, &snap_stats->lock_updates_ns_total);
|
| 1413 | atomic64_inc(&snap_stats->lock_updates_samples);
|
| 1414 | ext4_fc_snap_stats_update_max(&snap_stats->lock_updates_ns_max,
|
| 1415 | locked_ns);
|
| 1416 | if (trace_ext4_fc_lock_updates_enabled())
|
| 1417 | trace_call__ext4_fc_lock_updates(sb, commit_tid, locked_ns,
|
| 1418 | snap_inodes, snap_ranges,
|
| 1419 | ret, snap_err);
|
| 1420 | kvfree(inodes);
|
| 1421 | if (ret)
|
| 1422 | return ret;
|
| 1423 |
|
| 1424 | |
| 1425 | * Step 5: If file system device is different from journal device,
|
| 1426 | * issue a cache flush before we start writing fast commit blocks.
|
| 1427 |
|
| 1428 | if (journal->j_fs_dev != journal->j_dev)
|
| 1429 | blkdev_issue_flush(journal->j_fs_dev);
|
| 1430 |
|
| 1431 | blk_start_plug(&plug);
|
| 1432 | alloc_ctx = ext4_fc_lock(sb);
|
| 1433 |
|
| 1434 | if (sbi->s_fc_bytes == 0) {
|
| 1435 | |
| 1436 | * Step 6.1: Add a head tag only if this is the first fast
|
| 1437 | * commit in this TID.
|
| 1438 |
|
| 1439 | head.fc_features = cpu_to_le32(EXT4_FC_SUPPORTED_FEATURES);
|
| 1440 | head.fc_tid = cpu_to_le32(
|
| 1441 | sbi->s_journal->j_running_transaction->t_tid);
|
| 1442 | if (!ext4_fc_add_tlv(sb, EXT4_FC_TAG_HEAD, sizeof(head),
|
| 1443 | (u8 *)&head, &crc)) {
|
| 1444 | ret = -ENOSPC;
|
| 1445 | goto out;
|
| 1446 | }
|
| 1447 | }
|
| 1448 |
|
| 1449 |
|
| 1450 | ret = ext4_fc_commit_dentry_updates(journal, &crc);
|
| 1451 | if (ret)
|
| 1452 | goto out;
|
| 1453 |
|
| 1454 |
|
| 1455 | list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
|
| 1456 | inode = &iter->vfs_inode;
|
| 1457 | if (!ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING))
|
| 1458 | continue;
|
| 1459 |
|
| 1460 | ret = ext4_fc_write_inode_data(inode, &crc);
|
| 1461 | if (ret)
|
| 1462 | goto out;
|
| 1463 | ret = ext4_fc_write_inode(inode, &crc);
|
| 1464 | if (ret)
|
| 1465 | goto out;
|
| 1466 | }
|
| 1467 |
|
| 1468 | ret = ext4_fc_write_tail(sb, crc);
|
| 1469 |
|
| 1470 | out:
|
| 1471 | ext4_fc_unlock(sb, alloc_ctx);
|
| 1472 | blk_finish_plug(&plug);
|
| 1473 | return ret;
|
| 1474 | }
|
| 1475 |
|
| 1476 | static unsigned int ext4_fc_count_snapshot_inodes(struct super_block *sb)
|
| 1477 | {
|
| 1478 | struct ext4_sb_info *sbi = EXT4_SB(sb);
|
| 1479 | struct ext4_inode_info *iter;
|
| 1480 | struct ext4_fc_dentry_update *fc_dentry;
|
| 1481 | unsigned int nr_inodes = 0;
|
| 1482 | int alloc_ctx;
|
| 1483 |
|
| 1484 | alloc_ctx = ext4_fc_lock(sb);
|
| 1485 | list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list)
|
| 1486 | nr_inodes++;
|
| 1487 |
|
| 1488 | list_for_each_entry(fc_dentry, &sbi->s_fc_dentry_q[FC_Q_MAIN], fcd_list) {
|
| 1489 | struct ext4_inode_info *ei;
|
| 1490 |
|
| 1491 | if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT)
|
| 1492 | continue;
|
| 1493 | if (list_empty(&fc_dentry->fcd_dilist))
|
| 1494 | continue;
|
| 1495 |
|
| 1496 |
|
| 1497 | ei = list_first_entry(&fc_dentry->fcd_dilist,
|
| 1498 | struct ext4_inode_info, i_fc_dilist);
|
| 1499 | if (!list_empty(&ei->i_fc_list))
|
| 1500 | continue;
|
| 1501 |
|
| 1502 | nr_inodes++;
|
| 1503 | }
|
| 1504 | ext4_fc_unlock(sb, alloc_ctx);
|
| 1505 |
|
| 1506 | return nr_inodes;
|
| 1507 | }
|
| 1508 |
|
| 1509 | static int ext4_fc_alloc_snapshot_inodes(struct super_block *sb,
|
| 1510 | struct inode ***inodesp,
|
| 1511 | unsigned int *nr_inodesp)
|
| 1512 | {
|
| 1513 | unsigned int nr_inodes = ext4_fc_count_snapshot_inodes(sb);
|
| 1514 | struct inode **inodes;
|
| 1515 |
|
| 1516 | *inodesp = NULL;
|
| 1517 | *nr_inodesp = 0;
|
| 1518 |
|
| 1519 | if (!nr_inodes)
|
| 1520 | return 0;
|
| 1521 |
|
| 1522 | if (nr_inodes > EXT4_FC_SNAPSHOT_MAX_INODES)
|
| 1523 | return -E2BIG;
|
| 1524 |
|
| 1525 | inodes = kvzalloc_objs(*inodes, nr_inodes, GFP_NOFS);
|
| 1526 | if (!inodes)
|
| 1527 | return -ENOMEM;
|
| 1528 |
|
| 1529 | *inodesp = inodes;
|
| 1530 | *nr_inodesp = nr_inodes;
|
| 1531 | return 0;
|
| 1532 | }
|
| 1533 |
|
| 1534 | static void ext4_fc_update_stats(struct super_block *sb, int status,
|
| 1535 | u64 commit_time, int nblks, tid_t commit_tid)
|
| 1536 | {
|
| 1537 | struct ext4_fc_stats *stats = &EXT4_SB(sb)->s_fc_stats;
|
| 1538 |
|
| 1539 | ext4_debug("Fast commit ended with status = %d for tid %u",
|
| 1540 | status, commit_tid);
|
| 1541 | if (status == EXT4_FC_STATUS_OK) {
|
| 1542 | stats->fc_num_commits++;
|
| 1543 | stats->fc_numblks += nblks;
|
| 1544 | if (likely(stats->s_fc_avg_commit_time))
|
| 1545 | stats->s_fc_avg_commit_time =
|
| 1546 | (commit_time +
|
| 1547 | stats->s_fc_avg_commit_time * 3) / 4;
|
| 1548 | else
|
| 1549 | stats->s_fc_avg_commit_time = commit_time;
|
| 1550 | } else if (status == EXT4_FC_STATUS_FAILED ||
|
| 1551 | status == EXT4_FC_STATUS_INELIGIBLE) {
|
| 1552 | if (status == EXT4_FC_STATUS_FAILED)
|
| 1553 | stats->fc_failed_commits++;
|
| 1554 | stats->fc_ineligible_commits++;
|
| 1555 | } else {
|
| 1556 | stats->fc_skipped_commits++;
|
| 1557 | }
|
| 1558 | trace_ext4_fc_commit_stop(sb, nblks, status, commit_tid);
|
| 1559 | }
|
| 1560 |
|
| 1561 | |
| 1562 | * The main commit entry point. Performs a fast commit for transaction
|
| 1563 | * commit_tid if needed. If it's not possible to perform a fast commit
|
| 1564 | * due to various reasons, we fall back to full commit. Returns 0
|
| 1565 | * on success, error otherwise.
|
| 1566 |
|
| 1567 | int ext4_fc_commit(journal_t *journal, tid_t commit_tid)
|
| 1568 | {
|
| 1569 | struct super_block *sb = journal->j_private;
|
| 1570 | struct ext4_sb_info *sbi = EXT4_SB(sb);
|
| 1571 | int nblks = 0, ret, bsize = journal->j_blocksize;
|
| 1572 | int subtid = atomic_read(&sbi->s_fc_subtid);
|
| 1573 | int status = EXT4_FC_STATUS_OK, fc_bufs_before = 0;
|
| 1574 | ktime_t start_time, commit_time;
|
| 1575 | int old_ioprio, journal_ioprio;
|
| 1576 |
|
| 1577 | if (!test_opt2(sb, JOURNAL_FAST_COMMIT))
|
| 1578 | return jbd2_complete_transaction(journal, commit_tid);
|
| 1579 |
|
| 1580 | trace_ext4_fc_commit_start(sb, commit_tid);
|
| 1581 |
|
| 1582 | start_time = ktime_get();
|
| 1583 | old_ioprio = get_current_ioprio();
|
| 1584 |
|
| 1585 | restart_fc:
|
| 1586 | ret = jbd2_fc_begin_commit(journal, commit_tid);
|
| 1587 | if (ret == -EALREADY) {
|
| 1588 |
|
| 1589 | if (atomic_read(&sbi->s_fc_subtid) <= subtid &&
|
| 1590 | tid_gt(commit_tid, journal->j_commit_sequence))
|
| 1591 | goto restart_fc;
|
| 1592 | ext4_fc_update_stats(sb, EXT4_FC_STATUS_SKIPPED, 0, 0,
|
| 1593 | commit_tid);
|
| 1594 | return 0;
|
| 1595 | } else if (ret) {
|
| 1596 | |
| 1597 | * Commit couldn't start. Just update stats and perform a
|
| 1598 | * full commit.
|
| 1599 |
|
| 1600 | ext4_fc_update_stats(sb, EXT4_FC_STATUS_FAILED, 0, 0,
|
| 1601 | commit_tid);
|
| 1602 | return jbd2_complete_transaction(journal, commit_tid);
|
| 1603 | }
|
| 1604 |
|
| 1605 | |
| 1606 | * After establishing journal barrier via jbd2_fc_begin_commit(), check
|
| 1607 | * if we are fast commit ineligible.
|
| 1608 |
|
| 1609 | if (ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE)) {
|
| 1610 | status = EXT4_FC_STATUS_INELIGIBLE;
|
| 1611 | goto fallback;
|
| 1612 | }
|
| 1613 |
|
| 1614 | |
| 1615 | * Now that we know that this thread is going to do a fast commit,
|
| 1616 | * elevate the priority to match that of the journal thread.
|
| 1617 |
|
| 1618 | if (journal->j_task->io_context)
|
| 1619 | journal_ioprio = sbi->s_journal->j_task->io_context->ioprio;
|
| 1620 | else
|
| 1621 | journal_ioprio = EXT4_DEF_JOURNAL_IOPRIO;
|
| 1622 | set_task_ioprio(current, journal_ioprio);
|
| 1623 | fc_bufs_before = (sbi->s_fc_bytes + bsize - 1) / bsize;
|
| 1624 | ret = ext4_fc_perform_commit(journal, commit_tid);
|
| 1625 | if (ret < 0) {
|
| 1626 | if (ret == -EAGAIN || ret == -E2BIG || ret == -ECANCELED)
|
| 1627 | status = EXT4_FC_STATUS_INELIGIBLE;
|
| 1628 | else
|
| 1629 | status = EXT4_FC_STATUS_FAILED;
|
| 1630 | goto fallback;
|
| 1631 | }
|
| 1632 | nblks = (sbi->s_fc_bytes + bsize - 1) / bsize - fc_bufs_before;
|
| 1633 | ret = jbd2_fc_wait_bufs(journal, nblks);
|
| 1634 | if (ret < 0) {
|
| 1635 | status = EXT4_FC_STATUS_FAILED;
|
| 1636 | goto fallback;
|
| 1637 | }
|
| 1638 | atomic_inc(&sbi->s_fc_subtid);
|
| 1639 | ret = jbd2_fc_end_commit(journal);
|
| 1640 | set_task_ioprio(current, old_ioprio);
|
| 1641 | |
| 1642 | * weight the commit time higher than the average time so we
|
| 1643 | * don't react too strongly to vast changes in the commit time
|
| 1644 |
|
| 1645 | commit_time = ktime_to_ns(ktime_sub(ktime_get(), start_time));
|
| 1646 | ext4_fc_update_stats(sb, status, commit_time, nblks, commit_tid);
|
| 1647 | return ret;
|
| 1648 |
|
| 1649 | fallback:
|
| 1650 | set_task_ioprio(current, old_ioprio);
|
| 1651 | ret = jbd2_fc_end_commit_fallback(journal);
|
| 1652 | ext4_fc_update_stats(sb, status, 0, 0, commit_tid);
|
| 1653 | return ret;
|
| 1654 | }
|
| 1655 |
|
| 1656 | |
| 1657 | * Fast commit cleanup routine. This is called after every fast commit and
|
| 1658 | * full commit. full is true if we are called after a full commit.
|
| 1659 |
|
| 1660 | static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid)
|
| 1661 | {
|
| 1662 | struct super_block *sb = journal->j_private;
|
| 1663 | struct ext4_sb_info *sbi = EXT4_SB(sb);
|
| 1664 | struct ext4_inode_info *ei;
|
| 1665 | struct ext4_fc_dentry_update *fc_dentry;
|
| 1666 | int alloc_ctx;
|
| 1667 |
|
| 1668 | if (full && sbi->s_fc_bh)
|
| 1669 | sbi->s_fc_bh = NULL;
|
| 1670 |
|
| 1671 | trace_ext4_fc_cleanup(journal, full, tid);
|
| 1672 | jbd2_fc_release_bufs(journal);
|
| 1673 |
|
| 1674 | alloc_ctx = ext4_fc_lock(sb);
|
| 1675 | while (!list_empty(&sbi->s_fc_q[FC_Q_MAIN])) {
|
| 1676 | bool requeue;
|
| 1677 |
|
| 1678 | ei = list_first_entry(&sbi->s_fc_q[FC_Q_MAIN],
|
| 1679 | struct ext4_inode_info,
|
| 1680 | i_fc_list);
|
| 1681 | list_del_init(&ei->i_fc_list);
|
| 1682 | ext4_fc_free_inode_snap(&ei->vfs_inode);
|
| 1683 | spin_lock(&ei->i_fc_lock);
|
| 1684 | if (full)
|
| 1685 | requeue = !tid_geq(tid, ei->i_sync_tid);
|
| 1686 | else
|
| 1687 | requeue = ext4_test_inode_state(&ei->vfs_inode,
|
| 1688 | EXT4_STATE_FC_REQUEUE);
|
| 1689 | if (!requeue)
|
| 1690 | ext4_fc_reset_inode(&ei->vfs_inode);
|
| 1691 | ext4_clear_inode_state(&ei->vfs_inode, EXT4_STATE_FC_REQUEUE);
|
| 1692 | ext4_clear_inode_state(&ei->vfs_inode,
|
| 1693 | EXT4_STATE_FC_COMMITTING);
|
| 1694 | spin_unlock(&ei->i_fc_lock);
|
| 1695 | if (requeue)
|
| 1696 | list_add_tail(&ei->i_fc_list,
|
| 1697 | &sbi->s_fc_q[FC_Q_STAGING]);
|
| 1698 | |
| 1699 | * Make sure clearing of EXT4_STATE_FC_COMMITTING is
|
| 1700 | * visible before we send the wakeup. Pairs with implicit
|
| 1701 | * barrier in prepare_to_wait() in ext4_fc_del().
|
| 1702 |
|
| 1703 | smp_mb();
|
| 1704 | ext4_fc_wake_inode_state(&ei->vfs_inode,
|
| 1705 | EXT4_STATE_FC_COMMITTING);
|
| 1706 | }
|
| 1707 |
|
| 1708 | while (!list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) {
|
| 1709 | fc_dentry = list_first_entry(&sbi->s_fc_dentry_q[FC_Q_MAIN],
|
| 1710 | struct ext4_fc_dentry_update,
|
| 1711 | fcd_list);
|
| 1712 | list_del_init(&fc_dentry->fcd_list);
|
| 1713 | if (fc_dentry->fcd_op == EXT4_FC_TAG_CREAT &&
|
| 1714 | !list_empty(&fc_dentry->fcd_dilist)) {
|
| 1715 |
|
| 1716 | ei = list_first_entry(&fc_dentry->fcd_dilist,
|
| 1717 | struct ext4_inode_info,
|
| 1718 | i_fc_dilist);
|
| 1719 | ext4_fc_free_inode_snap(&ei->vfs_inode);
|
| 1720 | spin_lock(&ei->i_fc_lock);
|
| 1721 | ext4_clear_inode_state(&ei->vfs_inode,
|
| 1722 | EXT4_STATE_FC_REQUEUE);
|
| 1723 | ext4_clear_inode_state(&ei->vfs_inode,
|
| 1724 | EXT4_STATE_FC_COMMITTING);
|
| 1725 | spin_unlock(&ei->i_fc_lock);
|
| 1726 | |
| 1727 | * Make sure clearing of EXT4_STATE_FC_COMMITTING is
|
| 1728 | * visible before we send the wakeup. Pairs with
|
| 1729 | * implicit barrier in prepare_to_wait() in
|
| 1730 | * ext4_fc_del().
|
| 1731 |
|
| 1732 | smp_mb();
|
| 1733 | ext4_fc_wake_inode_state(&ei->vfs_inode,
|
| 1734 | EXT4_STATE_FC_COMMITTING);
|
| 1735 | }
|
| 1736 | list_del_init(&fc_dentry->fcd_dilist);
|
| 1737 |
|
| 1738 | release_dentry_name_snapshot(&fc_dentry->fcd_name);
|
| 1739 | kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry);
|
| 1740 | }
|
| 1741 |
|
| 1742 | list_splice_init(&sbi->s_fc_dentry_q[FC_Q_STAGING],
|
| 1743 | &sbi->s_fc_dentry_q[FC_Q_MAIN]);
|
| 1744 | list_splice_init(&sbi->s_fc_q[FC_Q_STAGING],
|
| 1745 | &sbi->s_fc_q[FC_Q_MAIN]);
|
| 1746 |
|
| 1747 | if (tid_geq(tid, sbi->s_fc_ineligible_tid)) {
|
| 1748 | sbi->s_fc_ineligible_tid = 0;
|
| 1749 | ext4_clear_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
|
| 1750 | }
|
| 1751 |
|
| 1752 | if (full)
|
| 1753 | sbi->s_fc_bytes = 0;
|
| 1754 | ext4_fc_unlock(sb, alloc_ctx);
|
| 1755 | trace_ext4_fc_stats(sb);
|
| 1756 | }
|
| 1757 |
|
| 1758 |
|
| 1759 |
|
| 1760 |
|
| 1761 | struct dentry_info_args {
|
| 1762 | int parent_ino, dname_len, ino, inode_len;
|
| 1763 | char *dname;
|
| 1764 | };
|
| 1765 |
|
| 1766 |
|
| 1767 | struct ext4_fc_tl_mem {
|
| 1768 | u16 fc_tag;
|
| 1769 | u16 fc_len;
|
| 1770 | };
|
| 1771 |
|
| 1772 | static inline void tl_to_darg(struct dentry_info_args *darg,
|
| 1773 | struct ext4_fc_tl_mem *tl, u8 *val)
|
| 1774 | {
|
| 1775 | struct ext4_fc_dentry_info fcd;
|
| 1776 |
|
| 1777 | memcpy(&fcd, val, sizeof(fcd));
|
| 1778 |
|
| 1779 | darg->parent_ino = le32_to_cpu(fcd.fc_parent_ino);
|
| 1780 | darg->ino = le32_to_cpu(fcd.fc_ino);
|
| 1781 | darg->dname = val + offsetof(struct ext4_fc_dentry_info, fc_dname);
|
| 1782 | darg->dname_len = tl->fc_len - sizeof(struct ext4_fc_dentry_info);
|
| 1783 | }
|
| 1784 |
|
| 1785 | static inline void ext4_fc_get_tl(struct ext4_fc_tl_mem *tl, u8 *val)
|
| 1786 | {
|
| 1787 | struct ext4_fc_tl tl_disk;
|
| 1788 |
|
| 1789 | memcpy(&tl_disk, val, EXT4_FC_TAG_BASE_LEN);
|
| 1790 | tl->fc_len = le16_to_cpu(tl_disk.fc_len);
|
| 1791 | tl->fc_tag = le16_to_cpu(tl_disk.fc_tag);
|
| 1792 | }
|
| 1793 |
|
| 1794 |
|
| 1795 | static int ext4_fc_replay_unlink(struct super_block *sb,
|
| 1796 | struct ext4_fc_tl_mem *tl, u8 *val)
|
| 1797 | {
|
| 1798 | struct inode *inode, *old_parent;
|
| 1799 | struct qstr entry;
|
| 1800 | struct dentry_info_args darg;
|
| 1801 | int ret = 0;
|
| 1802 |
|
| 1803 | tl_to_darg(&darg, tl, val);
|
| 1804 |
|
| 1805 | trace_ext4_fc_replay(sb, EXT4_FC_TAG_UNLINK, darg.ino,
|
| 1806 | darg.parent_ino, darg.dname_len);
|
| 1807 |
|
| 1808 | entry.name = darg.dname;
|
| 1809 | entry.len = darg.dname_len;
|
| 1810 | inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
|
| 1811 |
|
| 1812 | if (IS_ERR(inode)) {
|
| 1813 | ext4_debug("Inode %d not found", darg.ino);
|
| 1814 | return 0;
|
| 1815 | }
|
| 1816 |
|
| 1817 | old_parent = ext4_iget(sb, darg.parent_ino,
|
| 1818 | EXT4_IGET_NORMAL);
|
| 1819 | if (IS_ERR(old_parent)) {
|
| 1820 | ext4_debug("Dir with inode %d not found", darg.parent_ino);
|
| 1821 | iput(inode);
|
| 1822 | return 0;
|
| 1823 | }
|
| 1824 |
|
| 1825 | ret = __ext4_unlink(old_parent, &entry, inode, NULL);
|
| 1826 |
|
| 1827 | if (ret == -ENOENT)
|
| 1828 | ret = 0;
|
| 1829 | iput(old_parent);
|
| 1830 | iput(inode);
|
| 1831 | return ret;
|
| 1832 | }
|
| 1833 |
|
| 1834 | static int ext4_fc_replay_link_internal(struct super_block *sb,
|
| 1835 | struct dentry_info_args *darg,
|
| 1836 | struct inode *inode)
|
| 1837 | {
|
| 1838 | struct inode *dir = NULL;
|
| 1839 | struct qstr qstr_dname = QSTR_INIT(darg->dname, darg->dname_len);
|
| 1840 | int ret = 0;
|
| 1841 |
|
| 1842 | dir = ext4_iget(sb, darg->parent_ino, EXT4_IGET_NORMAL);
|
| 1843 | if (IS_ERR(dir)) {
|
| 1844 | ext4_debug("Dir with inode %d not found.", darg->parent_ino);
|
| 1845 | dir = NULL;
|
| 1846 | goto out;
|
| 1847 | }
|
| 1848 |
|
| 1849 | ret = __ext4_link(dir, inode, &qstr_dname, NULL);
|
| 1850 | |
| 1851 | * It's possible that link already existed since data blocks
|
| 1852 | * for the dir in question got persisted before we crashed OR
|
| 1853 | * we replayed this tag and crashed before the entire replay
|
| 1854 | * could complete.
|
| 1855 |
|
| 1856 | if (ret && ret != -EEXIST) {
|
| 1857 | ext4_debug("Failed to link\n");
|
| 1858 | goto out;
|
| 1859 | }
|
| 1860 |
|
| 1861 | ret = 0;
|
| 1862 | out:
|
| 1863 | if (dir)
|
| 1864 | iput(dir);
|
| 1865 |
|
| 1866 | return ret;
|
| 1867 | }
|
| 1868 |
|
| 1869 |
|
| 1870 | static int ext4_fc_replay_link(struct super_block *sb,
|
| 1871 | struct ext4_fc_tl_mem *tl, u8 *val)
|
| 1872 | {
|
| 1873 | struct inode *inode;
|
| 1874 | struct dentry_info_args darg;
|
| 1875 | int ret = 0;
|
| 1876 |
|
| 1877 | tl_to_darg(&darg, tl, val);
|
| 1878 | trace_ext4_fc_replay(sb, EXT4_FC_TAG_LINK, darg.ino,
|
| 1879 | darg.parent_ino, darg.dname_len);
|
| 1880 |
|
| 1881 | inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
|
| 1882 | if (IS_ERR(inode)) {
|
| 1883 | ext4_debug("Inode not found.");
|
| 1884 | return 0;
|
| 1885 | }
|
| 1886 |
|
| 1887 | ret = ext4_fc_replay_link_internal(sb, &darg, inode);
|
| 1888 | iput(inode);
|
| 1889 | return ret;
|
| 1890 | }
|
| 1891 |
|
| 1892 | |
| 1893 | * Record all the modified inodes during replay. We use this later to setup
|
| 1894 | * block bitmaps correctly.
|
| 1895 |
|
| 1896 | static int ext4_fc_record_modified_inode(struct super_block *sb, int ino)
|
| 1897 | {
|
| 1898 | struct ext4_fc_replay_state *state;
|
| 1899 | int i;
|
| 1900 |
|
| 1901 | state = &EXT4_SB(sb)->s_fc_replay_state;
|
| 1902 | for (i = 0; i < state->fc_modified_inodes_used; i++)
|
| 1903 | if (state->fc_modified_inodes[i] == ino)
|
| 1904 | return 0;
|
| 1905 | if (state->fc_modified_inodes_used == state->fc_modified_inodes_size) {
|
| 1906 | int *fc_modified_inodes;
|
| 1907 |
|
| 1908 | fc_modified_inodes = krealloc(state->fc_modified_inodes,
|
| 1909 | sizeof(int) * (state->fc_modified_inodes_size +
|
| 1910 | EXT4_FC_REPLAY_REALLOC_INCREMENT),
|
| 1911 | GFP_KERNEL);
|
| 1912 | if (!fc_modified_inodes)
|
| 1913 | return -ENOMEM;
|
| 1914 | state->fc_modified_inodes = fc_modified_inodes;
|
| 1915 | state->fc_modified_inodes_size +=
|
| 1916 | EXT4_FC_REPLAY_REALLOC_INCREMENT;
|
| 1917 | }
|
| 1918 | state->fc_modified_inodes[state->fc_modified_inodes_used++] = ino;
|
| 1919 | return 0;
|
| 1920 | }
|
| 1921 |
|
| 1922 | |
| 1923 | * Inode replay function
|
| 1924 |
|
| 1925 | static int ext4_fc_replay_inode(struct super_block *sb,
|
| 1926 | struct ext4_fc_tl_mem *tl, u8 *val)
|
| 1927 | {
|
| 1928 | struct ext4_fc_inode fc_inode;
|
| 1929 | struct ext4_inode *raw_inode;
|
| 1930 | struct ext4_inode *raw_fc_inode;
|
| 1931 | struct inode *inode = NULL;
|
| 1932 | struct ext4_iloc iloc;
|
| 1933 | int inode_len, ino, ret, tag = tl->fc_tag;
|
| 1934 | struct ext4_extent_header *eh;
|
| 1935 | size_t off_gen = offsetof(struct ext4_inode, i_generation);
|
| 1936 |
|
| 1937 | memcpy(&fc_inode, val, sizeof(fc_inode));
|
| 1938 |
|
| 1939 | ino = le32_to_cpu(fc_inode.fc_ino);
|
| 1940 | trace_ext4_fc_replay(sb, tag, ino, 0, 0);
|
| 1941 |
|
| 1942 | inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL);
|
| 1943 | if (!IS_ERR(inode)) {
|
| 1944 | ext4_ext_clear_bb(inode);
|
| 1945 | iput(inode);
|
| 1946 | }
|
| 1947 | inode = NULL;
|
| 1948 |
|
| 1949 | ret = ext4_fc_record_modified_inode(sb, ino);
|
| 1950 | if (ret)
|
| 1951 | goto out;
|
| 1952 |
|
| 1953 | raw_fc_inode = (struct ext4_inode *)
|
| 1954 | (val + offsetof(struct ext4_fc_inode, fc_raw_inode));
|
| 1955 | ret = ext4_get_fc_inode_loc(sb, ino, &iloc);
|
| 1956 | if (ret)
|
| 1957 | goto out;
|
| 1958 |
|
| 1959 | inode_len = tl->fc_len - sizeof(struct ext4_fc_inode);
|
| 1960 | raw_inode = ext4_raw_inode(&iloc);
|
| 1961 |
|
| 1962 | memcpy(raw_inode, raw_fc_inode, offsetof(struct ext4_inode, i_block));
|
| 1963 | memcpy((u8 *)raw_inode + off_gen, (u8 *)raw_fc_inode + off_gen,
|
| 1964 | inode_len - off_gen);
|
| 1965 | if (le32_to_cpu(raw_inode->i_flags) & EXT4_EXTENTS_FL) {
|
| 1966 | eh = (struct ext4_extent_header *)(&raw_inode->i_block[0]);
|
| 1967 | if (eh->eh_magic != EXT4_EXT_MAGIC) {
|
| 1968 | memset(eh, 0, sizeof(*eh));
|
| 1969 | eh->eh_magic = EXT4_EXT_MAGIC;
|
| 1970 | eh->eh_max = cpu_to_le16(
|
| 1971 | (sizeof(raw_inode->i_block) -
|
| 1972 | sizeof(struct ext4_extent_header))
|
| 1973 | / sizeof(struct ext4_extent));
|
| 1974 | }
|
| 1975 | } else if (le32_to_cpu(raw_inode->i_flags) & EXT4_INLINE_DATA_FL) {
|
| 1976 | memcpy(raw_inode->i_block, raw_fc_inode->i_block,
|
| 1977 | sizeof(raw_inode->i_block));
|
| 1978 | }
|
| 1979 |
|
| 1980 |
|
| 1981 | ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh);
|
| 1982 | if (ret)
|
| 1983 | goto out_brelse;
|
| 1984 | ret = sync_dirty_buffer(iloc.bh);
|
| 1985 | if (ret)
|
| 1986 | goto out_brelse;
|
| 1987 | ret = ext4_mark_inode_used(sb, ino);
|
| 1988 | if (ret)
|
| 1989 | goto out_brelse;
|
| 1990 |
|
| 1991 |
|
| 1992 | inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL);
|
| 1993 | if (IS_ERR(inode)) {
|
| 1994 | ext4_debug("Inode not found.");
|
| 1995 | inode = NULL;
|
| 1996 | ret = -EFSCORRUPTED;
|
| 1997 | goto out_brelse;
|
| 1998 | }
|
| 1999 |
|
| 2000 | |
| 2001 | * Our allocator could have made different decisions than before
|
| 2002 | * crashing. This should be fixed but until then, we calculate
|
| 2003 | * the number of blocks the inode.
|
| 2004 |
|
| 2005 | if (!ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA))
|
| 2006 | ext4_ext_replay_set_iblocks(inode);
|
| 2007 |
|
| 2008 | inode->i_generation = le32_to_cpu(ext4_raw_inode(&iloc)->i_generation);
|
| 2009 | ext4_reset_inode_seed(inode);
|
| 2010 |
|
| 2011 | ext4_inode_csum_set(inode, ext4_raw_inode(&iloc), EXT4_I(inode));
|
| 2012 | ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh);
|
| 2013 | sync_dirty_buffer(iloc.bh);
|
| 2014 | out_brelse:
|
| 2015 | brelse(iloc.bh);
|
| 2016 | out:
|
| 2017 | iput(inode);
|
| 2018 | if (!ret)
|
| 2019 | blkdev_issue_flush(sb->s_bdev);
|
| 2020 |
|
| 2021 | return ret;
|
| 2022 | }
|
| 2023 |
|
| 2024 | |
| 2025 | * Dentry create replay function.
|
| 2026 | *
|
| 2027 | * EXT4_FC_TAG_CREAT is preceded by EXT4_FC_TAG_INODE_FULL. Which means, the
|
| 2028 | * inode for which we are trying to create a dentry here, should already have
|
| 2029 | * been replayed before we start here.
|
| 2030 |
|
| 2031 | static int ext4_fc_replay_create(struct super_block *sb,
|
| 2032 | struct ext4_fc_tl_mem *tl, u8 *val)
|
| 2033 | {
|
| 2034 | int ret = 0;
|
| 2035 | struct inode *inode = NULL;
|
| 2036 | struct inode *dir = NULL;
|
| 2037 | struct dentry_info_args darg;
|
| 2038 |
|
| 2039 | tl_to_darg(&darg, tl, val);
|
| 2040 |
|
| 2041 | trace_ext4_fc_replay(sb, EXT4_FC_TAG_CREAT, darg.ino,
|
| 2042 | darg.parent_ino, darg.dname_len);
|
| 2043 |
|
| 2044 |
|
| 2045 | ret = ext4_mark_inode_used(sb, darg.ino);
|
| 2046 | if (ret)
|
| 2047 | goto out;
|
| 2048 |
|
| 2049 | inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
|
| 2050 | if (IS_ERR(inode)) {
|
| 2051 | ext4_debug("inode %d not found.", darg.ino);
|
| 2052 | inode = NULL;
|
| 2053 | ret = -EINVAL;
|
| 2054 | goto out;
|
| 2055 | }
|
| 2056 |
|
| 2057 | if (S_ISDIR(inode->i_mode)) {
|
| 2058 | |
| 2059 | * If we are creating a directory, we need to make sure that the
|
| 2060 | * dot and dot dot dirents are setup properly.
|
| 2061 |
|
| 2062 | dir = ext4_iget(sb, darg.parent_ino, EXT4_IGET_NORMAL);
|
| 2063 | if (IS_ERR(dir)) {
|
| 2064 | ext4_debug("Dir %d not found.", darg.ino);
|
| 2065 | goto out;
|
| 2066 | }
|
| 2067 | ret = ext4_init_new_dir(NULL, dir, inode);
|
| 2068 | iput(dir);
|
| 2069 | if (ret) {
|
| 2070 | ret = 0;
|
| 2071 | goto out;
|
| 2072 | }
|
| 2073 | }
|
| 2074 | ret = ext4_fc_replay_link_internal(sb, &darg, inode);
|
| 2075 | if (ret)
|
| 2076 | goto out;
|
| 2077 | set_nlink(inode, 1);
|
| 2078 | ext4_mark_inode_dirty(NULL, inode);
|
| 2079 | out:
|
| 2080 | iput(inode);
|
| 2081 | return ret;
|
| 2082 | }
|
| 2083 |
|
| 2084 | |
| 2085 | * Record physical disk regions which are in use as per fast commit area,
|
| 2086 | * and used by inodes during replay phase. Our simple replay phase
|
| 2087 | * allocator excludes these regions from allocation.
|
| 2088 |
|
| 2089 | int ext4_fc_record_regions(struct super_block *sb, int ino,
|
| 2090 | ext4_lblk_t lblk, ext4_fsblk_t pblk, int len, int replay)
|
| 2091 | {
|
| 2092 | struct ext4_fc_replay_state *state;
|
| 2093 | struct ext4_fc_alloc_region *region;
|
| 2094 |
|
| 2095 | state = &EXT4_SB(sb)->s_fc_replay_state;
|
| 2096 | |
| 2097 | * during replay phase, the fc_regions_valid may not same as
|
| 2098 | * fc_regions_used, update it when do new additions.
|
| 2099 |
|
| 2100 | if (replay && state->fc_regions_used != state->fc_regions_valid)
|
| 2101 | state->fc_regions_used = state->fc_regions_valid;
|
| 2102 | if (state->fc_regions_used == state->fc_regions_size) {
|
| 2103 | struct ext4_fc_alloc_region *fc_regions;
|
| 2104 |
|
| 2105 | fc_regions = krealloc(state->fc_regions,
|
| 2106 | sizeof(struct ext4_fc_alloc_region) *
|
| 2107 | (state->fc_regions_size +
|
| 2108 | EXT4_FC_REPLAY_REALLOC_INCREMENT),
|
| 2109 | GFP_KERNEL);
|
| 2110 | if (!fc_regions)
|
| 2111 | return -ENOMEM;
|
| 2112 | state->fc_regions_size +=
|
| 2113 | EXT4_FC_REPLAY_REALLOC_INCREMENT;
|
| 2114 | state->fc_regions = fc_regions;
|
| 2115 | }
|
| 2116 | region = &state->fc_regions[state->fc_regions_used++];
|
| 2117 | region->ino = ino;
|
| 2118 | region->lblk = lblk;
|
| 2119 | region->pblk = pblk;
|
| 2120 | region->len = len;
|
| 2121 |
|
| 2122 | if (replay)
|
| 2123 | state->fc_regions_valid++;
|
| 2124 |
|
| 2125 | return 0;
|
| 2126 | }
|
| 2127 |
|
| 2128 |
|
| 2129 | static int ext4_fc_replay_add_range(struct super_block *sb, u8 *val)
|
| 2130 | {
|
| 2131 | struct ext4_fc_add_range fc_add_ex;
|
| 2132 | struct ext4_extent newex, *ex;
|
| 2133 | struct inode *inode;
|
| 2134 | ext4_lblk_t start, cur;
|
| 2135 | int remaining, len;
|
| 2136 | ext4_fsblk_t start_pblk;
|
| 2137 | struct ext4_map_blocks map;
|
| 2138 | struct ext4_ext_path *path = NULL;
|
| 2139 | int ret;
|
| 2140 |
|
| 2141 | memcpy(&fc_add_ex, val, sizeof(fc_add_ex));
|
| 2142 | ex = (struct ext4_extent *)&fc_add_ex.fc_ex;
|
| 2143 |
|
| 2144 | trace_ext4_fc_replay(sb, EXT4_FC_TAG_ADD_RANGE,
|
| 2145 | le32_to_cpu(fc_add_ex.fc_ino), le32_to_cpu(ex->ee_block),
|
| 2146 | ext4_ext_get_actual_len(ex));
|
| 2147 |
|
| 2148 | inode = ext4_iget(sb, le32_to_cpu(fc_add_ex.fc_ino), EXT4_IGET_NORMAL);
|
| 2149 | if (IS_ERR(inode)) {
|
| 2150 | ext4_debug("Inode not found.");
|
| 2151 | return 0;
|
| 2152 | }
|
| 2153 |
|
| 2154 | ret = ext4_fc_record_modified_inode(sb, inode->i_ino);
|
| 2155 | if (ret)
|
| 2156 | goto out;
|
| 2157 |
|
| 2158 | start = le32_to_cpu(ex->ee_block);
|
| 2159 | start_pblk = ext4_ext_pblock(ex);
|
| 2160 | len = ext4_ext_get_actual_len(ex);
|
| 2161 |
|
| 2162 | cur = start;
|
| 2163 | remaining = len;
|
| 2164 | ext4_debug("ADD_RANGE, lblk %d, pblk %lld, len %d, unwritten %d, inode %llu\n",
|
| 2165 | start, start_pblk, len, ext4_ext_is_unwritten(ex),
|
| 2166 | inode->i_ino);
|
| 2167 |
|
| 2168 | while (remaining > 0) {
|
| 2169 | map.m_lblk = cur;
|
| 2170 | map.m_len = remaining;
|
| 2171 | map.m_pblk = 0;
|
| 2172 | ret = ext4_map_blocks(NULL, inode, &map, 0);
|
| 2173 |
|
| 2174 | if (ret < 0)
|
| 2175 | goto out;
|
| 2176 |
|
| 2177 | if (ret == 0) {
|
| 2178 |
|
| 2179 | path = ext4_find_extent(inode, cur, path, 0);
|
| 2180 | if (IS_ERR(path)) {
|
| 2181 | ret = PTR_ERR(path);
|
| 2182 | path = NULL;
|
| 2183 | goto out;
|
| 2184 | }
|
| 2185 | memset(&newex, 0, sizeof(newex));
|
| 2186 | newex.ee_block = cpu_to_le32(cur);
|
| 2187 | ext4_ext_store_pblock(
|
| 2188 | &newex, start_pblk + cur - start);
|
| 2189 | newex.ee_len = cpu_to_le16(map.m_len);
|
| 2190 | if (ext4_ext_is_unwritten(ex))
|
| 2191 | ext4_ext_mark_unwritten(&newex);
|
| 2192 | down_write(&EXT4_I(inode)->i_data_sem);
|
| 2193 | path = ext4_ext_insert_extent(NULL, inode,
|
| 2194 | path, &newex, 0);
|
| 2195 | up_write((&EXT4_I(inode)->i_data_sem));
|
| 2196 | if (IS_ERR(path)) {
|
| 2197 | ret = PTR_ERR(path);
|
| 2198 | path = NULL;
|
| 2199 | goto out;
|
| 2200 | }
|
| 2201 | goto next;
|
| 2202 | }
|
| 2203 |
|
| 2204 | if (start_pblk + cur - start != map.m_pblk) {
|
| 2205 | |
| 2206 | * Logical to physical mapping changed. This can happen
|
| 2207 | * if this range was removed and then reallocated to
|
| 2208 | * map to new physical blocks during a fast commit.
|
| 2209 |
|
| 2210 | ret = ext4_ext_replay_update_ex(inode, cur, map.m_len,
|
| 2211 | ext4_ext_is_unwritten(ex),
|
| 2212 | start_pblk + cur - start);
|
| 2213 | if (ret)
|
| 2214 | goto out;
|
| 2215 | |
| 2216 | * Mark the old blocks as free since they aren't used
|
| 2217 | * anymore. We maintain an array of all the modified
|
| 2218 | * inodes. In case these blocks are still used at either
|
| 2219 | * a different logical range in the same inode or in
|
| 2220 | * some different inode, we will mark them as allocated
|
| 2221 | * at the end of the FC replay using our array of
|
| 2222 | * modified inodes.
|
| 2223 |
|
| 2224 | ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, false);
|
| 2225 | goto next;
|
| 2226 | }
|
| 2227 |
|
| 2228 |
|
| 2229 | ext4_debug("Converting from %ld to %d %lld",
|
| 2230 | map.m_flags & EXT4_MAP_UNWRITTEN,
|
| 2231 | ext4_ext_is_unwritten(ex), map.m_pblk);
|
| 2232 | ret = ext4_ext_replay_update_ex(inode, cur, map.m_len,
|
| 2233 | ext4_ext_is_unwritten(ex), map.m_pblk);
|
| 2234 | if (ret)
|
| 2235 | goto out;
|
| 2236 | |
| 2237 | * We may have split the extent tree while toggling the state.
|
| 2238 | * Try to shrink the extent tree now.
|
| 2239 |
|
| 2240 | ext4_ext_replay_shrink_inode(inode, start + len);
|
| 2241 | next:
|
| 2242 | cur += map.m_len;
|
| 2243 | remaining -= map.m_len;
|
| 2244 | }
|
| 2245 | ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >>
|
| 2246 | sb->s_blocksize_bits);
|
| 2247 | ret = 0;
|
| 2248 | out:
|
| 2249 | ext4_free_ext_path(path);
|
| 2250 | iput(inode);
|
| 2251 | return ret;
|
| 2252 | }
|
| 2253 |
|
| 2254 |
|
| 2255 | static int
|
| 2256 | ext4_fc_replay_del_range(struct super_block *sb, u8 *val)
|
| 2257 | {
|
| 2258 | struct inode *inode;
|
| 2259 | struct ext4_fc_del_range lrange;
|
| 2260 | struct ext4_map_blocks map;
|
| 2261 | ext4_lblk_t cur, remaining;
|
| 2262 | int ret;
|
| 2263 |
|
| 2264 | memcpy(&lrange, val, sizeof(lrange));
|
| 2265 | cur = le32_to_cpu(lrange.fc_lblk);
|
| 2266 | remaining = le32_to_cpu(lrange.fc_len);
|
| 2267 |
|
| 2268 | trace_ext4_fc_replay(sb, EXT4_FC_TAG_DEL_RANGE,
|
| 2269 | le32_to_cpu(lrange.fc_ino), cur, remaining);
|
| 2270 |
|
| 2271 | inode = ext4_iget(sb, le32_to_cpu(lrange.fc_ino), EXT4_IGET_NORMAL);
|
| 2272 | if (IS_ERR(inode)) {
|
| 2273 | ext4_debug("Inode %d not found", le32_to_cpu(lrange.fc_ino));
|
| 2274 | return 0;
|
| 2275 | }
|
| 2276 |
|
| 2277 | ret = ext4_fc_record_modified_inode(sb, inode->i_ino);
|
| 2278 | if (ret)
|
| 2279 | goto out;
|
| 2280 |
|
| 2281 | ext4_debug("DEL_RANGE, inode %llu, lblk %d, len %d\n",
|
| 2282 | inode->i_ino, le32_to_cpu(lrange.fc_lblk),
|
| 2283 | le32_to_cpu(lrange.fc_len));
|
| 2284 | while (remaining > 0) {
|
| 2285 | map.m_lblk = cur;
|
| 2286 | map.m_len = remaining;
|
| 2287 |
|
| 2288 | ret = ext4_map_blocks(NULL, inode, &map, 0);
|
| 2289 | if (ret < 0)
|
| 2290 | goto out;
|
| 2291 | if (ret > 0) {
|
| 2292 | remaining -= ret;
|
| 2293 | cur += ret;
|
| 2294 | ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, false);
|
| 2295 | } else {
|
| 2296 | remaining -= map.m_len;
|
| 2297 | cur += map.m_len;
|
| 2298 | }
|
| 2299 | }
|
| 2300 |
|
| 2301 | down_write(&EXT4_I(inode)->i_data_sem);
|
| 2302 | ret = ext4_ext_remove_space(inode, le32_to_cpu(lrange.fc_lblk),
|
| 2303 | le32_to_cpu(lrange.fc_lblk) +
|
| 2304 | le32_to_cpu(lrange.fc_len) - 1);
|
| 2305 | up_write(&EXT4_I(inode)->i_data_sem);
|
| 2306 | if (ret)
|
| 2307 | goto out;
|
| 2308 | ext4_ext_replay_shrink_inode(inode,
|
| 2309 | i_size_read(inode) >> sb->s_blocksize_bits);
|
| 2310 | ext4_mark_inode_dirty(NULL, inode);
|
| 2311 | ret = 0;
|
| 2312 | out:
|
| 2313 | iput(inode);
|
| 2314 | return ret;
|
| 2315 | }
|
| 2316 |
|
| 2317 | static void ext4_fc_set_bitmaps_and_counters(struct super_block *sb)
|
| 2318 | {
|
| 2319 | struct ext4_fc_replay_state *state;
|
| 2320 | struct inode *inode;
|
| 2321 | struct ext4_ext_path *path = NULL;
|
| 2322 | struct ext4_map_blocks map;
|
| 2323 | int i, ret, j;
|
| 2324 | ext4_lblk_t cur, end;
|
| 2325 |
|
| 2326 | state = &EXT4_SB(sb)->s_fc_replay_state;
|
| 2327 | for (i = 0; i < state->fc_modified_inodes_used; i++) {
|
| 2328 | inode = ext4_iget(sb, state->fc_modified_inodes[i],
|
| 2329 | EXT4_IGET_NORMAL);
|
| 2330 | if (IS_ERR(inode)) {
|
| 2331 | ext4_debug("Inode %d not found.",
|
| 2332 | state->fc_modified_inodes[i]);
|
| 2333 | continue;
|
| 2334 | }
|
| 2335 | cur = 0;
|
| 2336 | end = EXT_MAX_BLOCKS;
|
| 2337 | if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA)) {
|
| 2338 | iput(inode);
|
| 2339 | continue;
|
| 2340 | }
|
| 2341 | while (cur < end) {
|
| 2342 | map.m_lblk = cur;
|
| 2343 | map.m_len = end - cur;
|
| 2344 |
|
| 2345 | ret = ext4_map_blocks(NULL, inode, &map, 0);
|
| 2346 | if (ret < 0)
|
| 2347 | break;
|
| 2348 |
|
| 2349 | if (ret > 0) {
|
| 2350 | path = ext4_find_extent(inode, map.m_lblk, path, 0);
|
| 2351 | if (!IS_ERR(path)) {
|
| 2352 | for (j = 0; j < path->p_depth; j++)
|
| 2353 | ext4_mb_mark_bb(inode->i_sb,
|
| 2354 | path[j].p_block, 1, true);
|
| 2355 | } else {
|
| 2356 | path = NULL;
|
| 2357 | }
|
| 2358 | cur += ret;
|
| 2359 | ext4_mb_mark_bb(inode->i_sb, map.m_pblk,
|
| 2360 | map.m_len, true);
|
| 2361 | } else {
|
| 2362 | cur = cur + (map.m_len ? map.m_len : 1);
|
| 2363 | }
|
| 2364 | }
|
| 2365 | iput(inode);
|
| 2366 | }
|
| 2367 |
|
| 2368 | ext4_free_ext_path(path);
|
| 2369 | }
|
| 2370 |
|
| 2371 | |
| 2372 | * Check if block is in excluded regions for block allocation. The simple
|
| 2373 | * allocator that runs during replay phase is calls this function to see
|
| 2374 | * if it is okay to use a block.
|
| 2375 |
|
| 2376 | bool ext4_fc_replay_check_excluded(struct super_block *sb, ext4_fsblk_t blk)
|
| 2377 | {
|
| 2378 | int i;
|
| 2379 | struct ext4_fc_replay_state *state;
|
| 2380 |
|
| 2381 | state = &EXT4_SB(sb)->s_fc_replay_state;
|
| 2382 | for (i = 0; i < state->fc_regions_valid; i++) {
|
| 2383 | if (state->fc_regions[i].ino == 0 ||
|
| 2384 | state->fc_regions[i].len == 0)
|
| 2385 | continue;
|
| 2386 | if (in_range(blk, state->fc_regions[i].pblk,
|
| 2387 | state->fc_regions[i].len))
|
| 2388 | return true;
|
| 2389 | }
|
| 2390 | return false;
|
| 2391 | }
|
| 2392 |
|
| 2393 |
|
| 2394 | void ext4_fc_replay_cleanup(struct super_block *sb)
|
| 2395 | {
|
| 2396 | struct ext4_sb_info *sbi = EXT4_SB(sb);
|
| 2397 |
|
| 2398 | sbi->s_mount_state &= ~EXT4_FC_REPLAY;
|
| 2399 | kfree(sbi->s_fc_replay_state.fc_regions);
|
| 2400 | kfree(sbi->s_fc_replay_state.fc_modified_inodes);
|
| 2401 | }
|
| 2402 |
|
| 2403 | static bool ext4_fc_value_len_isvalid(struct ext4_sb_info *sbi,
|
| 2404 | int tag, int len)
|
| 2405 | {
|
| 2406 | switch (tag) {
|
| 2407 | case EXT4_FC_TAG_ADD_RANGE:
|
| 2408 | return len == sizeof(struct ext4_fc_add_range);
|
| 2409 | case EXT4_FC_TAG_DEL_RANGE:
|
| 2410 | return len == sizeof(struct ext4_fc_del_range);
|
| 2411 | case EXT4_FC_TAG_CREAT:
|
| 2412 | case EXT4_FC_TAG_LINK:
|
| 2413 | case EXT4_FC_TAG_UNLINK:
|
| 2414 | len -= sizeof(struct ext4_fc_dentry_info);
|
| 2415 | return len >= 1 && len <= EXT4_NAME_LEN;
|
| 2416 | case EXT4_FC_TAG_INODE:
|
| 2417 | len -= sizeof(struct ext4_fc_inode);
|
| 2418 | return len >= EXT4_GOOD_OLD_INODE_SIZE &&
|
| 2419 | len <= sbi->s_inode_size;
|
| 2420 | case EXT4_FC_TAG_PAD:
|
| 2421 | return true;
|
| 2422 | case EXT4_FC_TAG_TAIL:
|
| 2423 | return len >= sizeof(struct ext4_fc_tail);
|
| 2424 | case EXT4_FC_TAG_HEAD:
|
| 2425 | return len == sizeof(struct ext4_fc_head);
|
| 2426 | }
|
| 2427 | return false;
|
| 2428 | }
|
| 2429 |
|
| 2430 | |
| 2431 | * Recovery Scan phase handler
|
| 2432 | *
|
| 2433 | * This function is called during the scan phase and is responsible
|
| 2434 | * for doing following things:
|
| 2435 | * - Make sure the fast commit area has valid tags for replay
|
| 2436 | * - Count number of tags that need to be replayed by the replay handler
|
| 2437 | * - Verify CRC
|
| 2438 | * - Create a list of excluded blocks for allocation during replay phase
|
| 2439 | *
|
| 2440 | * This function returns JBD2_FC_REPLAY_CONTINUE to indicate that SCAN is
|
| 2441 | * incomplete and JBD2 should send more blocks. It returns JBD2_FC_REPLAY_STOP
|
| 2442 | * to indicate that scan has finished and JBD2 can now start replay phase.
|
| 2443 | * It returns a negative error to indicate that there was an error. At the end
|
| 2444 | * of a successful scan phase, sbi->s_fc_replay_state.fc_replay_num_tags is set
|
| 2445 | * to indicate the number of tags that need to replayed during the replay phase.
|
| 2446 |
|
| 2447 | static int ext4_fc_replay_scan(journal_t *journal,
|
| 2448 | struct buffer_head *bh, int off,
|
| 2449 | tid_t expected_tid)
|
| 2450 | {
|
| 2451 | struct super_block *sb = journal->j_private;
|
| 2452 | struct ext4_sb_info *sbi = EXT4_SB(sb);
|
| 2453 | struct ext4_fc_replay_state *state;
|
| 2454 | int ret = JBD2_FC_REPLAY_CONTINUE;
|
| 2455 | struct ext4_fc_add_range ext;
|
| 2456 | struct ext4_fc_tl_mem tl;
|
| 2457 | struct ext4_fc_tail tail;
|
| 2458 | __u8 *start, *end, *cur, *val;
|
| 2459 | struct ext4_fc_head head;
|
| 2460 | struct ext4_extent *ex;
|
| 2461 |
|
| 2462 | state = &sbi->s_fc_replay_state;
|
| 2463 |
|
| 2464 | start = (u8 *)bh->b_data;
|
| 2465 | end = start + journal->j_blocksize;
|
| 2466 |
|
| 2467 | if (state->fc_replay_expected_off == 0) {
|
| 2468 | state->fc_cur_tag = 0;
|
| 2469 | state->fc_replay_num_tags = 0;
|
| 2470 | state->fc_crc = 0;
|
| 2471 | state->fc_regions = NULL;
|
| 2472 | state->fc_regions_valid = state->fc_regions_used =
|
| 2473 | state->fc_regions_size = 0;
|
| 2474 |
|
| 2475 | if (le16_to_cpu(((struct ext4_fc_tl *)start)->fc_tag)
|
| 2476 | != EXT4_FC_TAG_HEAD)
|
| 2477 | return 0;
|
| 2478 | }
|
| 2479 |
|
| 2480 | if (off != state->fc_replay_expected_off) {
|
| 2481 | ret = -EFSCORRUPTED;
|
| 2482 | goto out_err;
|
| 2483 | }
|
| 2484 |
|
| 2485 | state->fc_replay_expected_off++;
|
| 2486 | for (cur = start; cur <= end - EXT4_FC_TAG_BASE_LEN;
|
| 2487 | cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) {
|
| 2488 | ext4_fc_get_tl(&tl, cur);
|
| 2489 | val = cur + EXT4_FC_TAG_BASE_LEN;
|
| 2490 | if (tl.fc_len > end - val ||
|
| 2491 | !ext4_fc_value_len_isvalid(sbi, tl.fc_tag, tl.fc_len)) {
|
| 2492 | ret = state->fc_replay_num_tags ?
|
| 2493 | JBD2_FC_REPLAY_STOP : -ECANCELED;
|
| 2494 | goto out_err;
|
| 2495 | }
|
| 2496 | ext4_debug("Scan phase, tag:%s, blk %lld\n",
|
| 2497 | tag2str(tl.fc_tag), bh->b_blocknr);
|
| 2498 | switch (tl.fc_tag) {
|
| 2499 | case EXT4_FC_TAG_ADD_RANGE:
|
| 2500 | memcpy(&ext, val, sizeof(ext));
|
| 2501 | ex = (struct ext4_extent *)&ext.fc_ex;
|
| 2502 | ret = ext4_fc_record_regions(sb,
|
| 2503 | le32_to_cpu(ext.fc_ino),
|
| 2504 | le32_to_cpu(ex->ee_block), ext4_ext_pblock(ex),
|
| 2505 | ext4_ext_get_actual_len(ex), 0);
|
| 2506 | if (ret < 0)
|
| 2507 | break;
|
| 2508 | ret = JBD2_FC_REPLAY_CONTINUE;
|
| 2509 | fallthrough;
|
| 2510 | case EXT4_FC_TAG_DEL_RANGE:
|
| 2511 | case EXT4_FC_TAG_LINK:
|
| 2512 | case EXT4_FC_TAG_UNLINK:
|
| 2513 | case EXT4_FC_TAG_CREAT:
|
| 2514 | case EXT4_FC_TAG_INODE:
|
| 2515 | case EXT4_FC_TAG_PAD:
|
| 2516 | state->fc_cur_tag++;
|
| 2517 | state->fc_crc = ext4_chksum(state->fc_crc, cur,
|
| 2518 | EXT4_FC_TAG_BASE_LEN + tl.fc_len);
|
| 2519 | break;
|
| 2520 | case EXT4_FC_TAG_TAIL:
|
| 2521 | state->fc_cur_tag++;
|
| 2522 | memcpy(&tail, val, sizeof(tail));
|
| 2523 | state->fc_crc = ext4_chksum(state->fc_crc, cur,
|
| 2524 | EXT4_FC_TAG_BASE_LEN +
|
| 2525 | offsetof(struct ext4_fc_tail,
|
| 2526 | fc_crc));
|
| 2527 | if (le32_to_cpu(tail.fc_tid) == expected_tid &&
|
| 2528 | le32_to_cpu(tail.fc_crc) == state->fc_crc) {
|
| 2529 | state->fc_replay_num_tags = state->fc_cur_tag;
|
| 2530 | state->fc_regions_valid =
|
| 2531 | state->fc_regions_used;
|
| 2532 | } else {
|
| 2533 | ret = state->fc_replay_num_tags ?
|
| 2534 | JBD2_FC_REPLAY_STOP : -EFSBADCRC;
|
| 2535 | }
|
| 2536 | state->fc_crc = 0;
|
| 2537 | break;
|
| 2538 | case EXT4_FC_TAG_HEAD:
|
| 2539 | memcpy(&head, val, sizeof(head));
|
| 2540 | if (le32_to_cpu(head.fc_features) &
|
| 2541 | ~EXT4_FC_SUPPORTED_FEATURES) {
|
| 2542 | ret = -EOPNOTSUPP;
|
| 2543 | break;
|
| 2544 | }
|
| 2545 | if (le32_to_cpu(head.fc_tid) != expected_tid) {
|
| 2546 | ret = JBD2_FC_REPLAY_STOP;
|
| 2547 | break;
|
| 2548 | }
|
| 2549 | state->fc_cur_tag++;
|
| 2550 | state->fc_crc = ext4_chksum(state->fc_crc, cur,
|
| 2551 | EXT4_FC_TAG_BASE_LEN + tl.fc_len);
|
| 2552 | break;
|
| 2553 | default:
|
| 2554 | ret = state->fc_replay_num_tags ?
|
| 2555 | JBD2_FC_REPLAY_STOP : -ECANCELED;
|
| 2556 | }
|
| 2557 | if (ret < 0 || ret == JBD2_FC_REPLAY_STOP)
|
| 2558 | break;
|
| 2559 | }
|
| 2560 |
|
| 2561 | out_err:
|
| 2562 | trace_ext4_fc_replay_scan(sb, ret, off);
|
| 2563 | return ret;
|
| 2564 | }
|
| 2565 |
|
| 2566 | |
| 2567 | * Main recovery path entry point.
|
| 2568 | * The meaning of return codes is similar as above.
|
| 2569 |
|
| 2570 | static int ext4_fc_replay(journal_t *journal, struct buffer_head *bh,
|
| 2571 | enum passtype pass, int off, tid_t expected_tid)
|
| 2572 | {
|
| 2573 | struct super_block *sb = journal->j_private;
|
| 2574 | struct ext4_sb_info *sbi = EXT4_SB(sb);
|
| 2575 | struct ext4_fc_tl_mem tl;
|
| 2576 | __u8 *start, *end, *cur, *val;
|
| 2577 | int ret = JBD2_FC_REPLAY_CONTINUE;
|
| 2578 | struct ext4_fc_replay_state *state = &sbi->s_fc_replay_state;
|
| 2579 | struct ext4_fc_tail tail;
|
| 2580 |
|
| 2581 | if (pass == PASS_SCAN) {
|
| 2582 | state->fc_current_pass = PASS_SCAN;
|
| 2583 | return ext4_fc_replay_scan(journal, bh, off, expected_tid);
|
| 2584 | }
|
| 2585 |
|
| 2586 | if (state->fc_current_pass != pass) {
|
| 2587 | state->fc_current_pass = pass;
|
| 2588 | sbi->s_mount_state |= EXT4_FC_REPLAY;
|
| 2589 | }
|
| 2590 | if (!sbi->s_fc_replay_state.fc_replay_num_tags) {
|
| 2591 | ext4_debug("Replay stops\n");
|
| 2592 | ext4_fc_set_bitmaps_and_counters(sb);
|
| 2593 | return 0;
|
| 2594 | }
|
| 2595 |
|
| 2596 | #ifdef CONFIG_EXT4_DEBUG
|
| 2597 | if (sbi->s_fc_debug_max_replay && off >= sbi->s_fc_debug_max_replay) {
|
| 2598 | pr_warn("Dropping fc block %d because max_replay set\n", off);
|
| 2599 | return JBD2_FC_REPLAY_STOP;
|
| 2600 | }
|
| 2601 | #endif
|
| 2602 |
|
| 2603 | start = (u8 *)bh->b_data;
|
| 2604 | end = start + journal->j_blocksize;
|
| 2605 |
|
| 2606 | for (cur = start; cur <= end - EXT4_FC_TAG_BASE_LEN;
|
| 2607 | cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) {
|
| 2608 | ext4_fc_get_tl(&tl, cur);
|
| 2609 | val = cur + EXT4_FC_TAG_BASE_LEN;
|
| 2610 |
|
| 2611 | if (state->fc_replay_num_tags == 0) {
|
| 2612 | ret = JBD2_FC_REPLAY_STOP;
|
| 2613 | ext4_fc_set_bitmaps_and_counters(sb);
|
| 2614 | break;
|
| 2615 | }
|
| 2616 |
|
| 2617 | ext4_debug("Replay phase, tag:%s\n", tag2str(tl.fc_tag));
|
| 2618 | state->fc_replay_num_tags--;
|
| 2619 | switch (tl.fc_tag) {
|
| 2620 | case EXT4_FC_TAG_LINK:
|
| 2621 | ret = ext4_fc_replay_link(sb, &tl, val);
|
| 2622 | break;
|
| 2623 | case EXT4_FC_TAG_UNLINK:
|
| 2624 | ret = ext4_fc_replay_unlink(sb, &tl, val);
|
| 2625 | break;
|
| 2626 | case EXT4_FC_TAG_ADD_RANGE:
|
| 2627 | ret = ext4_fc_replay_add_range(sb, val);
|
| 2628 | break;
|
| 2629 | case EXT4_FC_TAG_CREAT:
|
| 2630 | ret = ext4_fc_replay_create(sb, &tl, val);
|
| 2631 | break;
|
| 2632 | case EXT4_FC_TAG_DEL_RANGE:
|
| 2633 | ret = ext4_fc_replay_del_range(sb, val);
|
| 2634 | break;
|
| 2635 | case EXT4_FC_TAG_INODE:
|
| 2636 | ret = ext4_fc_replay_inode(sb, &tl, val);
|
| 2637 | break;
|
| 2638 | case EXT4_FC_TAG_PAD:
|
| 2639 | trace_ext4_fc_replay(sb, EXT4_FC_TAG_PAD, 0,
|
| 2640 | tl.fc_len, 0);
|
| 2641 | break;
|
| 2642 | case EXT4_FC_TAG_TAIL:
|
| 2643 | trace_ext4_fc_replay(sb, EXT4_FC_TAG_TAIL,
|
| 2644 | 0, tl.fc_len, 0);
|
| 2645 | memcpy(&tail, val, sizeof(tail));
|
| 2646 | WARN_ON(le32_to_cpu(tail.fc_tid) != expected_tid);
|
| 2647 | break;
|
| 2648 | case EXT4_FC_TAG_HEAD:
|
| 2649 | break;
|
| 2650 | default:
|
| 2651 | trace_ext4_fc_replay(sb, tl.fc_tag, 0, tl.fc_len, 0);
|
| 2652 | ret = -ECANCELED;
|
| 2653 | break;
|
| 2654 | }
|
| 2655 | if (ret < 0)
|
| 2656 | break;
|
| 2657 | ret = JBD2_FC_REPLAY_CONTINUE;
|
| 2658 | }
|
| 2659 | return ret;
|
| 2660 | }
|
| 2661 |
|
| 2662 | void ext4_fc_init(struct super_block *sb, journal_t *journal)
|
| 2663 | {
|
| 2664 | |
| 2665 | * We set replay callback even if fast commit disabled because we may
|
| 2666 | * could still have fast commit blocks that need to be replayed even if
|
| 2667 | * fast commit has now been turned off.
|
| 2668 |
|
| 2669 | journal->j_fc_replay_callback = ext4_fc_replay;
|
| 2670 | if (!test_opt2(sb, JOURNAL_FAST_COMMIT))
|
| 2671 | return;
|
| 2672 | journal->j_fc_cleanup_callback = ext4_fc_cleanup;
|
| 2673 | }
|
| 2674 |
|
| 2675 | static const char * const fc_ineligible_reasons[] = {
|
| 2676 | [EXT4_FC_REASON_XATTR] = "Extended attributes changed",
|
| 2677 | [EXT4_FC_REASON_CROSS_RENAME] = "Cross rename",
|
| 2678 | [EXT4_FC_REASON_JOURNAL_FLAG_CHANGE] = "Journal flag changed",
|
| 2679 | [EXT4_FC_REASON_NOMEM] = "Insufficient memory",
|
| 2680 | [EXT4_FC_REASON_SWAP_BOOT] = "Swap boot",
|
| 2681 | [EXT4_FC_REASON_RESIZE] = "Resize",
|
| 2682 | [EXT4_FC_REASON_RENAME_DIR] = "Dir renamed",
|
| 2683 | [EXT4_FC_REASON_FALLOC_RANGE] = "Falloc range op",
|
| 2684 | [EXT4_FC_REASON_INODE_JOURNAL_DATA] = "Data journalling",
|
| 2685 | [EXT4_FC_REASON_ENCRYPTED_FILENAME] = "Encrypted filename",
|
| 2686 | [EXT4_FC_REASON_MIGRATE] = "Inode format migration",
|
| 2687 | [EXT4_FC_REASON_VERITY] = "fs-verity enable",
|
| 2688 | [EXT4_FC_REASON_MOVE_EXT] = "Move extents",
|
| 2689 | };
|
| 2690 |
|
| 2691 | int ext4_fc_info_show(struct seq_file *seq, void *v)
|
| 2692 | {
|
| 2693 | struct ext4_sb_info *sbi = EXT4_SB((struct super_block *)seq->private);
|
| 2694 | struct ext4_fc_stats *stats = &sbi->s_fc_stats;
|
| 2695 | struct ext4_fc_snap_stats *snap_stats = &sbi->s_fc_snap_stats;
|
| 2696 | u64 lock_avg_ns = 0;
|
| 2697 | u64 lock_updates_samples;
|
| 2698 | u64 lock_updates_ns_total;
|
| 2699 | u64 lock_updates_ns_max;
|
| 2700 | int i;
|
| 2701 |
|
| 2702 | if (v != SEQ_START_TOKEN)
|
| 2703 | return 0;
|
| 2704 |
|
| 2705 | lock_updates_samples =
|
| 2706 | atomic64_read(&snap_stats->lock_updates_samples);
|
| 2707 | lock_updates_ns_total =
|
| 2708 | atomic64_read(&snap_stats->lock_updates_ns_total);
|
| 2709 | lock_updates_ns_max =
|
| 2710 | atomic64_read(&snap_stats->lock_updates_ns_max);
|
| 2711 | if (lock_updates_samples)
|
| 2712 | lock_avg_ns = div64_u64(lock_updates_ns_total,
|
| 2713 | lock_updates_samples);
|
| 2714 |
|
| 2715 | seq_printf(seq,
|
| 2716 | "fc stats:\n%ld commits\n%ld ineligible\n%ld numblks\n%lluus avg_commit_time\n",
|
| 2717 | stats->fc_num_commits, stats->fc_ineligible_commits,
|
| 2718 | stats->fc_numblks,
|
| 2719 | div_u64(stats->s_fc_avg_commit_time, 1000));
|
| 2720 | seq_puts(seq, "Ineligible reasons:\n");
|
| 2721 | for (i = 0; i < EXT4_FC_REASON_MAX; i++)
|
| 2722 | seq_printf(seq, "\"%s\":\t%d\n", fc_ineligible_reasons[i],
|
| 2723 | stats->fc_ineligible_reason_count[i]);
|
| 2724 |
|
| 2725 | seq_printf(seq,
|
| 2726 | "Snapshot stats:\n%llu inodes\n%llu ranges\n%lluus lock_updates_avg\n%lluus lock_updates_max\n",
|
| 2727 | atomic64_read(&snap_stats->snap_inodes),
|
| 2728 | atomic64_read(&snap_stats->snap_ranges),
|
| 2729 | div_u64(lock_avg_ns, 1000),
|
| 2730 | div_u64(lock_updates_ns_max, 1000));
|
| 2731 | seq_printf(seq,
|
| 2732 | "Snapshot failures:\n%llu es_miss\n%llu es_delayed\n%llu es_other\n%llu inodes_cap\n%llu ranges_cap\n%llu nomem\n%llu inode_loc\n%llu no_snap\n",
|
| 2733 | atomic64_read(&snap_stats->snap_fail_es_miss),
|
| 2734 | atomic64_read(&snap_stats->snap_fail_es_delayed),
|
| 2735 | atomic64_read(&snap_stats->snap_fail_es_other),
|
| 2736 | atomic64_read(&snap_stats->snap_fail_inodes_cap),
|
| 2737 | atomic64_read(&snap_stats->snap_fail_ranges_cap),
|
| 2738 | atomic64_read(&snap_stats->snap_fail_nomem),
|
| 2739 | atomic64_read(&snap_stats->snap_fail_inode_loc),
|
| 2740 | atomic64_read(&snap_stats->snap_fail_no_snap));
|
| 2741 |
|
| 2742 | return 0;
|
| 2743 | }
|
| 2744 |
|
| 2745 | int __init ext4_fc_init_dentry_cache(void)
|
| 2746 | {
|
| 2747 | ext4_fc_dentry_cachep = KMEM_CACHE(ext4_fc_dentry_update,
|
| 2748 | SLAB_RECLAIM_ACCOUNT);
|
| 2749 |
|
| 2750 | if (!ext4_fc_dentry_cachep)
|
| 2751 | return -ENOMEM;
|
| 2752 |
|
| 2753 | ext4_fc_range_cachep = KMEM_CACHE(ext4_fc_range, SLAB_RECLAIM_ACCOUNT);
|
| 2754 | if (!ext4_fc_range_cachep) {
|
| 2755 | kmem_cache_destroy(ext4_fc_dentry_cachep);
|
| 2756 | return -ENOMEM;
|
| 2757 | }
|
| 2758 |
|
| 2759 | return 0;
|
| 2760 | }
|
| 2761 |
|
| 2762 | void ext4_fc_destroy_dentry_cache(void)
|
| 2763 | {
|
| 2764 | kmem_cache_destroy(ext4_fc_range_cachep);
|
| 2765 | kmem_cache_destroy(ext4_fc_dentry_cachep);
|
| 2766 | }
|