| 1 | |
| 2 | |
| 3 | * Copyright (c) 2016-present, Facebook, Inc.
|
| 4 | * All rights reserved.
|
| 5 | *
|
| 6 |
|
| 7 |
|
| 8 | #include <linux/bio.h>
|
| 9 | #include <linux/bitmap.h>
|
| 10 | #include <linux/err.h>
|
| 11 | #include <linux/init.h>
|
| 12 | #include <linux/kernel.h>
|
| 13 | #include <linux/mm.h>
|
| 14 | #include <linux/sched/mm.h>
|
| 15 | #include <linux/pagemap.h>
|
| 16 | #include <linux/refcount.h>
|
| 17 | #include <linux/sched.h>
|
| 18 | #include <linux/slab.h>
|
| 19 | #include <linux/zstd.h>
|
| 20 | #include "misc.h"
|
| 21 | #include "fs.h"
|
| 22 | #include "btrfs_inode.h"
|
| 23 | #include "compression.h"
|
| 24 | #include "super.h"
|
| 25 |
|
| 26 | #define ZSTD_BTRFS_MAX_WINDOWLOG 17
|
| 27 | #define ZSTD_BTRFS_MAX_INPUT (1U << ZSTD_BTRFS_MAX_WINDOWLOG)
|
| 28 | #define ZSTD_BTRFS_DEFAULT_LEVEL 3
|
| 29 | #define ZSTD_BTRFS_MIN_LEVEL -15
|
| 30 | #define ZSTD_BTRFS_MAX_LEVEL 15
|
| 31 |
|
| 32 | #define ZSTD_BTRFS_RECLAIM_JIFFIES (307 * HZ)
|
| 33 |
|
| 34 | static zstd_parameters zstd_get_btrfs_parameters(int level,
|
| 35 | size_t src_len)
|
| 36 | {
|
| 37 | zstd_parameters params = zstd_get_params(level, src_len);
|
| 38 |
|
| 39 | if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG)
|
| 40 | params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG;
|
| 41 | WARN_ON(src_len > ZSTD_BTRFS_MAX_INPUT);
|
| 42 | return params;
|
| 43 | }
|
| 44 |
|
| 45 | struct workspace {
|
| 46 | void *mem;
|
| 47 | size_t size;
|
| 48 | char *buf;
|
| 49 | int level;
|
| 50 | int req_level;
|
| 51 | unsigned long last_used;
|
| 52 | struct list_head list;
|
| 53 | struct list_head lru_list;
|
| 54 | zstd_in_buffer in_buf;
|
| 55 | zstd_out_buffer out_buf;
|
| 56 | zstd_parameters params;
|
| 57 | };
|
| 58 |
|
| 59 | |
| 60 | * Zstd Workspace Management
|
| 61 | *
|
| 62 | * Zstd workspaces have different memory requirements depending on the level.
|
| 63 | * The zstd workspaces are managed by having individual lists for each level
|
| 64 | * and a global lru. Forward progress is maintained by protecting a max level
|
| 65 | * workspace.
|
| 66 | *
|
| 67 | * Getting a workspace is done by using the bitmap to identify the levels that
|
| 68 | * have available workspaces and scans up. This lets us recycle higher level
|
| 69 | * workspaces because of the monotonic memory guarantee. A workspace's
|
| 70 | * last_used is only updated if it is being used by the corresponding memory
|
| 71 | * level. Putting a workspace involves adding it back to the appropriate places
|
| 72 | * and adding it back to the lru if necessary.
|
| 73 | *
|
| 74 | * A timer is used to reclaim workspaces if they have not been used for
|
| 75 | * ZSTD_BTRFS_RECLAIM_JIFFIES. This helps keep only active workspaces around.
|
| 76 | * The upper bound is provided by the workqueue limit which is 2 (percpu limit).
|
| 77 |
|
| 78 |
|
| 79 | struct zstd_workspace_manager {
|
| 80 | spinlock_t lock;
|
| 81 | struct list_head lru_list;
|
| 82 | struct list_head idle_ws[ZSTD_BTRFS_MAX_LEVEL];
|
| 83 | unsigned long active_map;
|
| 84 | wait_queue_head_t wait;
|
| 85 | struct timer_list timer;
|
| 86 | };
|
| 87 |
|
| 88 | static size_t zstd_ws_mem_sizes[ZSTD_BTRFS_MAX_LEVEL];
|
| 89 |
|
| 90 | static inline struct workspace *list_to_workspace(struct list_head *list)
|
| 91 | {
|
| 92 | return container_of(list, struct workspace, list);
|
| 93 | }
|
| 94 |
|
| 95 | static inline int clip_level(int level)
|
| 96 | {
|
| 97 | return max(0, level - 1);
|
| 98 | }
|
| 99 |
|
| 100 | |
| 101 | * Timer callback to free unused workspaces.
|
| 102 | *
|
| 103 | * @t: timer
|
| 104 | *
|
| 105 | * This scans the lru_list and attempts to reclaim any workspace that hasn't
|
| 106 | * been used for ZSTD_BTRFS_RECLAIM_JIFFIES.
|
| 107 | *
|
| 108 | * The context is softirq and does not need the _bh locking primitives.
|
| 109 |
|
| 110 | static void zstd_reclaim_timer_fn(struct timer_list *timer)
|
| 111 | {
|
| 112 | struct zstd_workspace_manager *zwsm =
|
| 113 | container_of(timer, struct zstd_workspace_manager, timer);
|
| 114 | unsigned long reclaim_threshold = jiffies - ZSTD_BTRFS_RECLAIM_JIFFIES;
|
| 115 | struct list_head *pos, *next;
|
| 116 |
|
| 117 | spin_lock(&zwsm->lock);
|
| 118 |
|
| 119 | if (list_empty(&zwsm->lru_list)) {
|
| 120 | spin_unlock(&zwsm->lock);
|
| 121 | return;
|
| 122 | }
|
| 123 |
|
| 124 | list_for_each_prev_safe(pos, next, &zwsm->lru_list) {
|
| 125 | struct workspace *victim = container_of(pos, struct workspace,
|
| 126 | lru_list);
|
| 127 | int level;
|
| 128 |
|
| 129 | if (time_after(victim->last_used, reclaim_threshold))
|
| 130 | break;
|
| 131 |
|
| 132 |
|
| 133 | if (victim->req_level)
|
| 134 | continue;
|
| 135 |
|
| 136 | level = victim->level;
|
| 137 | list_del(&victim->lru_list);
|
| 138 | list_del(&victim->list);
|
| 139 | zstd_free_workspace(&victim->list);
|
| 140 |
|
| 141 | if (list_empty(&zwsm->idle_ws[level]))
|
| 142 | clear_bit(level, &zwsm->active_map);
|
| 143 |
|
| 144 | }
|
| 145 |
|
| 146 | if (!list_empty(&zwsm->lru_list))
|
| 147 | mod_timer(&zwsm->timer, jiffies + ZSTD_BTRFS_RECLAIM_JIFFIES);
|
| 148 |
|
| 149 | spin_unlock(&zwsm->lock);
|
| 150 | }
|
| 151 |
|
| 152 | |
| 153 | * Calculate monotonic memory bounds.
|
| 154 | *
|
| 155 | * It is possible based on the level configurations that a higher level
|
| 156 | * workspace uses less memory than a lower level workspace. In order to reuse
|
| 157 | * workspaces, this must be made a monotonic relationship. This precomputes
|
| 158 | * the required memory for each level and enforces the monotonicity between
|
| 159 | * level and memory required.
|
| 160 |
|
| 161 | static void zstd_calc_ws_mem_sizes(void)
|
| 162 | {
|
| 163 | size_t max_size = 0;
|
| 164 | int level;
|
| 165 |
|
| 166 | for (level = ZSTD_BTRFS_MIN_LEVEL; level <= ZSTD_BTRFS_MAX_LEVEL; level++) {
|
| 167 | if (level == 0)
|
| 168 | continue;
|
| 169 | zstd_parameters params =
|
| 170 | zstd_get_btrfs_parameters(level, ZSTD_BTRFS_MAX_INPUT);
|
| 171 | size_t level_size =
|
| 172 | max_t(size_t,
|
| 173 | zstd_cstream_workspace_bound(¶ms.cParams),
|
| 174 | zstd_dstream_workspace_bound(ZSTD_BTRFS_MAX_INPUT));
|
| 175 |
|
| 176 | max_size = max_t(size_t, max_size, level_size);
|
| 177 |
|
| 178 | zstd_ws_mem_sizes[clip_level(level)] = max_size;
|
| 179 | }
|
| 180 | }
|
| 181 |
|
| 182 | int zstd_alloc_workspace_manager(struct btrfs_fs_info *fs_info)
|
| 183 | {
|
| 184 | struct zstd_workspace_manager *zwsm;
|
| 185 | struct list_head *ws;
|
| 186 |
|
| 187 | ASSERT(fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD] == NULL);
|
| 188 | zwsm = kzalloc_obj(*zwsm);
|
| 189 | if (!zwsm)
|
| 190 | return -ENOMEM;
|
| 191 | zstd_calc_ws_mem_sizes();
|
| 192 | spin_lock_init(&zwsm->lock);
|
| 193 | init_waitqueue_head(&zwsm->wait);
|
| 194 | timer_setup(&zwsm->timer, zstd_reclaim_timer_fn, 0);
|
| 195 |
|
| 196 | INIT_LIST_HEAD(&zwsm->lru_list);
|
| 197 | for (int i = 0; i < ZSTD_BTRFS_MAX_LEVEL; i++)
|
| 198 | INIT_LIST_HEAD(&zwsm->idle_ws[i]);
|
| 199 | fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD] = zwsm;
|
| 200 |
|
| 201 | ws = zstd_alloc_workspace(fs_info, ZSTD_BTRFS_MAX_LEVEL);
|
| 202 | if (IS_ERR(ws)) {
|
| 203 | btrfs_warn(NULL, "cannot preallocate zstd compression workspace");
|
| 204 | } else {
|
| 205 | set_bit(ZSTD_BTRFS_MAX_LEVEL - 1, &zwsm->active_map);
|
| 206 | list_add(ws, &zwsm->idle_ws[ZSTD_BTRFS_MAX_LEVEL - 1]);
|
| 207 | }
|
| 208 | return 0;
|
| 209 | }
|
| 210 |
|
| 211 | void zstd_free_workspace_manager(struct btrfs_fs_info *fs_info)
|
| 212 | {
|
| 213 | struct zstd_workspace_manager *zwsm = fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD];
|
| 214 | struct workspace *workspace;
|
| 215 |
|
| 216 | if (!zwsm)
|
| 217 | return;
|
| 218 | fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD] = NULL;
|
| 219 | spin_lock_bh(&zwsm->lock);
|
| 220 | for (int i = 0; i < ZSTD_BTRFS_MAX_LEVEL; i++) {
|
| 221 | while (!list_empty(&zwsm->idle_ws[i])) {
|
| 222 | workspace = container_of(zwsm->idle_ws[i].next,
|
| 223 | struct workspace, list);
|
| 224 | list_del(&workspace->list);
|
| 225 | list_del(&workspace->lru_list);
|
| 226 | zstd_free_workspace(&workspace->list);
|
| 227 | }
|
| 228 | }
|
| 229 | spin_unlock_bh(&zwsm->lock);
|
| 230 | timer_delete_sync(&zwsm->timer);
|
| 231 | kfree(zwsm);
|
| 232 | }
|
| 233 |
|
| 234 | |
| 235 | * Find workspace for given level.
|
| 236 | *
|
| 237 | * @level: compression level
|
| 238 | *
|
| 239 | * This iterates over the set bits in the active_map beginning at the requested
|
| 240 | * compression level. This lets us utilize already allocated workspaces before
|
| 241 | * allocating a new one. If the workspace is of a larger size, it is used, but
|
| 242 | * the place in the lru_list and last_used times are not updated. This is to
|
| 243 | * offer the opportunity to reclaim the workspace in favor of allocating an
|
| 244 | * appropriately sized one in the future.
|
| 245 |
|
| 246 | static struct list_head *zstd_find_workspace(struct btrfs_fs_info *fs_info, int level)
|
| 247 | {
|
| 248 | struct zstd_workspace_manager *zwsm = fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD];
|
| 249 | struct list_head *ws;
|
| 250 | struct workspace *workspace;
|
| 251 | int i = clip_level(level);
|
| 252 |
|
| 253 | ASSERT(zwsm);
|
| 254 | spin_lock_bh(&zwsm->lock);
|
| 255 | for_each_set_bit_from(i, &zwsm->active_map, ZSTD_BTRFS_MAX_LEVEL) {
|
| 256 | if (!list_empty(&zwsm->idle_ws[i])) {
|
| 257 | ws = zwsm->idle_ws[i].next;
|
| 258 | workspace = list_to_workspace(ws);
|
| 259 | list_del_init(ws);
|
| 260 |
|
| 261 | workspace->req_level = level;
|
| 262 | if (clip_level(level) == workspace->level)
|
| 263 | list_del(&workspace->lru_list);
|
| 264 | if (list_empty(&zwsm->idle_ws[i]))
|
| 265 | clear_bit(i, &zwsm->active_map);
|
| 266 | spin_unlock_bh(&zwsm->lock);
|
| 267 | return ws;
|
| 268 | }
|
| 269 | }
|
| 270 | spin_unlock_bh(&zwsm->lock);
|
| 271 |
|
| 272 | return NULL;
|
| 273 | }
|
| 274 |
|
| 275 | |
| 276 | * Zstd get_workspace for level.
|
| 277 | *
|
| 278 | * @level: compression level
|
| 279 | *
|
| 280 | * If @level is 0, then any compression level can be used. Therefore, we begin
|
| 281 | * scanning from 1. We first scan through possible workspaces and then after
|
| 282 | * attempt to allocate a new workspace. If we fail to allocate one due to
|
| 283 | * memory pressure, go to sleep waiting for the max level workspace to free up.
|
| 284 |
|
| 285 | struct list_head *zstd_get_workspace(struct btrfs_fs_info *fs_info, int level)
|
| 286 | {
|
| 287 | struct zstd_workspace_manager *zwsm = fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD];
|
| 288 | struct list_head *ws;
|
| 289 | unsigned int nofs_flag;
|
| 290 |
|
| 291 | ASSERT(zwsm);
|
| 292 |
|
| 293 |
|
| 294 | if (!level)
|
| 295 | level = 1;
|
| 296 |
|
| 297 | again:
|
| 298 | ws = zstd_find_workspace(fs_info, level);
|
| 299 | if (ws)
|
| 300 | return ws;
|
| 301 |
|
| 302 | nofs_flag = memalloc_nofs_save();
|
| 303 | ws = zstd_alloc_workspace(fs_info, level);
|
| 304 | memalloc_nofs_restore(nofs_flag);
|
| 305 |
|
| 306 | if (IS_ERR(ws)) {
|
| 307 | DEFINE_WAIT(wait);
|
| 308 |
|
| 309 | prepare_to_wait(&zwsm->wait, &wait, TASK_UNINTERRUPTIBLE);
|
| 310 | |
| 311 | * Re-check after being queued: zstd_put_workspace() only wakes
|
| 312 | * a queue that already has a sleeper, so a workspace returned
|
| 313 | * since the failed allocation woke nobody.
|
| 314 |
|
| 315 | ws = zstd_find_workspace(fs_info, level);
|
| 316 | if (!ws)
|
| 317 | schedule();
|
| 318 | finish_wait(&zwsm->wait, &wait);
|
| 319 | if (ws)
|
| 320 | return ws;
|
| 321 |
|
| 322 | goto again;
|
| 323 | }
|
| 324 |
|
| 325 | return ws;
|
| 326 | }
|
| 327 |
|
| 328 | |
| 329 | * Zstd put_workspace.
|
| 330 | *
|
| 331 | * @ws: list_head for the workspace
|
| 332 | *
|
| 333 | * When putting back a workspace, we only need to update the LRU if we are of
|
| 334 | * the requested compression level. Here is where we continue to protect the
|
| 335 | * max level workspace or update last_used accordingly. If the reclaim timer
|
| 336 | * isn't set, it is also set here. Only the max level workspace tries and wakes
|
| 337 | * up waiting workspaces.
|
| 338 |
|
| 339 | void zstd_put_workspace(struct btrfs_fs_info *fs_info, struct list_head *ws)
|
| 340 | {
|
| 341 | struct zstd_workspace_manager *zwsm = fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD];
|
| 342 | struct workspace *workspace = list_to_workspace(ws);
|
| 343 |
|
| 344 | ASSERT(zwsm);
|
| 345 | spin_lock_bh(&zwsm->lock);
|
| 346 |
|
| 347 |
|
| 348 | if (clip_level(workspace->req_level) == workspace->level) {
|
| 349 |
|
| 350 | if (list_empty(&zwsm->idle_ws[ZSTD_BTRFS_MAX_LEVEL - 1])) {
|
| 351 | INIT_LIST_HEAD(&workspace->lru_list);
|
| 352 | } else {
|
| 353 | workspace->last_used = jiffies;
|
| 354 | list_add(&workspace->lru_list, &zwsm->lru_list);
|
| 355 | if (!timer_pending(&zwsm->timer))
|
| 356 | mod_timer(&zwsm->timer,
|
| 357 | jiffies + ZSTD_BTRFS_RECLAIM_JIFFIES);
|
| 358 | }
|
| 359 | }
|
| 360 |
|
| 361 | set_bit(workspace->level, &zwsm->active_map);
|
| 362 | list_add(&workspace->list, &zwsm->idle_ws[workspace->level]);
|
| 363 | workspace->req_level = 0;
|
| 364 |
|
| 365 | spin_unlock_bh(&zwsm->lock);
|
| 366 |
|
| 367 | if (workspace->level == clip_level(ZSTD_BTRFS_MAX_LEVEL))
|
| 368 | cond_wake_up(&zwsm->wait);
|
| 369 | }
|
| 370 |
|
| 371 | void zstd_free_workspace(struct list_head *ws)
|
| 372 | {
|
| 373 | struct workspace *workspace = list_entry(ws, struct workspace, list);
|
| 374 |
|
| 375 | kvfree(workspace->mem);
|
| 376 | kvfree(workspace->buf);
|
| 377 | kfree(workspace);
|
| 378 | }
|
| 379 |
|
| 380 | struct list_head *zstd_alloc_workspace(struct btrfs_fs_info *fs_info, int level)
|
| 381 | {
|
| 382 | struct workspace *workspace;
|
| 383 |
|
| 384 | workspace = kzalloc_obj(*workspace);
|
| 385 | if (!workspace)
|
| 386 | return ERR_PTR(-ENOMEM);
|
| 387 |
|
| 388 |
|
| 389 | workspace->size = zstd_ws_mem_sizes[clip_level(level)];
|
| 390 | workspace->level = clip_level(level);
|
| 391 | workspace->req_level = level;
|
| 392 | workspace->last_used = jiffies;
|
| 393 | workspace->mem = kvmalloc(workspace->size, GFP_KERNEL | __GFP_NOWARN);
|
| 394 | workspace->buf = kvmalloc(fs_info->sectorsize, GFP_KERNEL);
|
| 395 | if (!workspace->mem || !workspace->buf)
|
| 396 | goto fail;
|
| 397 |
|
| 398 | INIT_LIST_HEAD(&workspace->list);
|
| 399 | INIT_LIST_HEAD(&workspace->lru_list);
|
| 400 |
|
| 401 | return &workspace->list;
|
| 402 | fail:
|
| 403 | zstd_free_workspace(&workspace->list);
|
| 404 | return ERR_PTR(-ENOMEM);
|
| 405 | }
|
| 406 |
|
| 407 | int zstd_compress_bio(struct list_head *ws, struct compressed_bio *cb)
|
| 408 | {
|
| 409 | struct btrfs_inode *inode = cb->bbio.inode;
|
| 410 | struct btrfs_fs_info *fs_info = inode->root->fs_info;
|
| 411 | struct workspace *workspace = list_entry(ws, struct workspace, list);
|
| 412 | struct address_space *mapping = inode->vfs_inode.i_mapping;
|
| 413 | struct bio *bio = &cb->bbio.bio;
|
| 414 | zstd_cstream *stream;
|
| 415 | int ret = 0;
|
| 416 |
|
| 417 | struct folio *in_folio = NULL;
|
| 418 |
|
| 419 | struct folio *out_folio = NULL;
|
| 420 | unsigned long tot_in = 0;
|
| 421 | unsigned long tot_out = 0;
|
| 422 | const u64 start = cb->start;
|
| 423 | const u32 len = cb->len;
|
| 424 | const u64 end = start + len;
|
| 425 | const u32 min_folio_size = btrfs_min_folio_size(fs_info);
|
| 426 |
|
| 427 | workspace->params = zstd_get_btrfs_parameters(workspace->req_level, len);
|
| 428 |
|
| 429 |
|
| 430 | stream = zstd_init_cstream(&workspace->params, len, workspace->mem, workspace->size);
|
| 431 | if (unlikely(!stream)) {
|
| 432 | btrfs_err(fs_info,
|
| 433 | "zstd compression init level %d failed, root %llu inode %llu offset %llu",
|
| 434 | workspace->req_level, btrfs_root_id(inode->root),
|
| 435 | btrfs_ino(inode), start);
|
| 436 | ret = -EIO;
|
| 437 | goto out;
|
| 438 | }
|
| 439 |
|
| 440 |
|
| 441 | ret = btrfs_compress_filemap_get_folio(mapping, start, &in_folio);
|
| 442 | if (ret < 0)
|
| 443 | goto out;
|
| 444 | workspace->in_buf.src = kmap_local_folio(in_folio, offset_in_folio(in_folio, start));
|
| 445 | workspace->in_buf.pos = 0;
|
| 446 | workspace->in_buf.size = btrfs_calc_input_length(in_folio, end, start);
|
| 447 |
|
| 448 |
|
| 449 | out_folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS);
|
| 450 | if (out_folio == NULL) {
|
| 451 | ret = -ENOMEM;
|
| 452 | goto out;
|
| 453 | }
|
| 454 | workspace->out_buf.dst = folio_address(out_folio);
|
| 455 | workspace->out_buf.pos = 0;
|
| 456 | workspace->out_buf.size = min_folio_size;
|
| 457 |
|
| 458 | while (1) {
|
| 459 | size_t ret2;
|
| 460 |
|
| 461 | ret2 = zstd_compress_stream(stream, &workspace->out_buf, &workspace->in_buf);
|
| 462 | if (unlikely(zstd_is_error(ret2))) {
|
| 463 | btrfs_warn(fs_info,
|
| 464 | "zstd compression level %d failed, error %d root %llu inode %llu offset %llu",
|
| 465 | workspace->req_level, zstd_get_error_code(ret2),
|
| 466 | btrfs_root_id(inode->root), btrfs_ino(inode),
|
| 467 | start + tot_in);
|
| 468 | ret = -EIO;
|
| 469 | goto out;
|
| 470 | }
|
| 471 |
|
| 472 |
|
| 473 | if (tot_in + workspace->in_buf.pos > fs_info->sectorsize * 2 &&
|
| 474 | tot_in + workspace->in_buf.pos < tot_out + workspace->out_buf.pos) {
|
| 475 | ret = -E2BIG;
|
| 476 | goto out;
|
| 477 | }
|
| 478 |
|
| 479 |
|
| 480 | if (workspace->out_buf.pos >= workspace->out_buf.size) {
|
| 481 | tot_out += min_folio_size;
|
| 482 | if (tot_out >= len) {
|
| 483 | ret = -E2BIG;
|
| 484 | goto out;
|
| 485 | }
|
| 486 |
|
| 487 | if (!bio_add_folio(bio, out_folio, folio_size(out_folio), 0)) {
|
| 488 | ret = -E2BIG;
|
| 489 | goto out;
|
| 490 | }
|
| 491 |
|
| 492 | out_folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS);
|
| 493 | if (out_folio == NULL) {
|
| 494 | ret = -ENOMEM;
|
| 495 | goto out;
|
| 496 | }
|
| 497 | workspace->out_buf.dst = folio_address(out_folio);
|
| 498 | workspace->out_buf.pos = 0;
|
| 499 | workspace->out_buf.size = min_folio_size;
|
| 500 | }
|
| 501 |
|
| 502 |
|
| 503 | if (tot_in + workspace->in_buf.pos >= len) {
|
| 504 | tot_in += workspace->in_buf.pos;
|
| 505 | break;
|
| 506 | }
|
| 507 |
|
| 508 |
|
| 509 | if (workspace->in_buf.pos >= workspace->in_buf.size) {
|
| 510 | u64 cur;
|
| 511 |
|
| 512 | tot_in += workspace->in_buf.size;
|
| 513 | cur = start + tot_in;
|
| 514 |
|
| 515 | kunmap_local(workspace->in_buf.src);
|
| 516 | workspace->in_buf.src = NULL;
|
| 517 | folio_put(in_folio);
|
| 518 |
|
| 519 | ret = btrfs_compress_filemap_get_folio(mapping, cur, &in_folio);
|
| 520 | if (ret < 0)
|
| 521 | goto out;
|
| 522 | workspace->in_buf.src = kmap_local_folio(in_folio,
|
| 523 | offset_in_folio(in_folio, cur));
|
| 524 | workspace->in_buf.pos = 0;
|
| 525 | workspace->in_buf.size = btrfs_calc_input_length(in_folio, end, cur);
|
| 526 | }
|
| 527 | }
|
| 528 |
|
| 529 | while (1) {
|
| 530 | size_t ret2;
|
| 531 |
|
| 532 | ret2 = zstd_end_stream(stream, &workspace->out_buf);
|
| 533 | if (unlikely(zstd_is_error(ret2))) {
|
| 534 | btrfs_err(fs_info,
|
| 535 | "zstd compression end level %d failed, error %d root %llu inode %llu offset %llu",
|
| 536 | workspace->req_level, zstd_get_error_code(ret2),
|
| 537 | btrfs_root_id(inode->root), btrfs_ino(inode),
|
| 538 | start + tot_in);
|
| 539 | ret = -EIO;
|
| 540 | goto out;
|
| 541 | }
|
| 542 |
|
| 543 | if (ret2 == 0) {
|
| 544 | tot_out += workspace->out_buf.pos;
|
| 545 | if (tot_out >= len) {
|
| 546 | ret = -E2BIG;
|
| 547 | goto out;
|
| 548 | }
|
| 549 | if (!bio_add_folio(bio, out_folio, workspace->out_buf.pos, 0)) {
|
| 550 | ret = -E2BIG;
|
| 551 | goto out;
|
| 552 | }
|
| 553 | out_folio = NULL;
|
| 554 | break;
|
| 555 | }
|
| 556 | tot_out += min_folio_size;
|
| 557 | if (tot_out >= len) {
|
| 558 | ret = -E2BIG;
|
| 559 | goto out;
|
| 560 | }
|
| 561 | if (!bio_add_folio(bio, out_folio, folio_size(out_folio), 0)) {
|
| 562 | ret = -E2BIG;
|
| 563 | goto out;
|
| 564 | }
|
| 565 | out_folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS);
|
| 566 | if (out_folio == NULL) {
|
| 567 | ret = -ENOMEM;
|
| 568 | goto out;
|
| 569 | }
|
| 570 | workspace->out_buf.dst = folio_address(out_folio);
|
| 571 | workspace->out_buf.pos = 0;
|
| 572 | workspace->out_buf.size = min_folio_size;
|
| 573 | }
|
| 574 |
|
| 575 | if (tot_out >= tot_in) {
|
| 576 | ret = -E2BIG;
|
| 577 | goto out;
|
| 578 | }
|
| 579 |
|
| 580 | ret = 0;
|
| 581 | ASSERT(tot_out == bio->bi_iter.bi_size);
|
| 582 | out:
|
| 583 | if (out_folio)
|
| 584 | btrfs_free_compr_folio(out_folio);
|
| 585 | if (workspace->in_buf.src) {
|
| 586 | kunmap_local(workspace->in_buf.src);
|
| 587 | folio_put(in_folio);
|
| 588 | }
|
| 589 | return ret;
|
| 590 | }
|
| 591 |
|
| 592 | |
| 593 | * Map the destination for the next chunk of output.
|
| 594 | *
|
| 595 | * @decompressed is the offset of the next output byte inside the fully
|
| 596 | * decompressed extent. If that offset has reached the current destination
|
| 597 | * segment, its page-bounded bio_vec is kmapped so that zstd can write into the
|
| 598 | * page cache directly, and the number of bytes writable there is returned.
|
| 599 | * Otherwise @kaddr_ret is set to NULL and the number of bytes to skip before
|
| 600 | * that segment is returned. This covers both the initial prefix and gaps in
|
| 601 | * the destination bio.
|
| 602 |
|
| 603 | static u32 zstd_map_dest(struct compressed_bio *cb, u32 decompressed,
|
| 604 | void **kaddr_ret)
|
| 605 | {
|
| 606 | struct bio *orig_bio = &cb->orig_bbio->bio;
|
| 607 | struct bio_vec bvec;
|
| 608 | u32 bvec_offset;
|
| 609 | u32 off;
|
| 610 |
|
| 611 | bvec = bio_iter_iovec(orig_bio, orig_bio->bi_iter);
|
| 612 | |
| 613 | * cb->start may underflow, but subtracting that value can still give us
|
| 614 | * the correct offset inside the full decompressed extent.
|
| 615 |
|
| 616 | bvec_offset = page_offset(bvec.bv_page) + bvec.bv_offset - cb->start;
|
| 617 |
|
| 618 | if (decompressed < bvec_offset) {
|
| 619 | *kaddr_ret = NULL;
|
| 620 | return bvec_offset - decompressed;
|
| 621 | }
|
| 622 |
|
| 623 | off = decompressed - bvec_offset;
|
| 624 | ASSERT(off < bvec.bv_len);
|
| 625 | *kaddr_ret = bvec_kmap_local(&bvec) + off;
|
| 626 | return bvec.bv_len - off;
|
| 627 | }
|
| 628 |
|
| 629 | int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
|
| 630 | {
|
| 631 | struct btrfs_fs_info *fs_info = cb_to_fs_info(cb);
|
| 632 | struct workspace *workspace = list_entry(ws, struct workspace, list);
|
| 633 | struct bio *orig_bio = &cb->orig_bbio->bio;
|
| 634 | struct folio_iter fi;
|
| 635 | size_t srclen = bio_get_size(&cb->bbio.bio);
|
| 636 | zstd_dstream *stream;
|
| 637 | int ret = 0;
|
| 638 | const unsigned int min_folio_size = btrfs_min_folio_size(fs_info);
|
| 639 | unsigned long folio_in_index = 0;
|
| 640 | unsigned long total_folios_in = DIV_ROUND_UP(srclen, min_folio_size);
|
| 641 | unsigned long total_out = 0;
|
| 642 |
|
| 643 | bio_first_folio(&fi, &cb->bbio.bio, 0);
|
| 644 | if (unlikely(!fi.folio))
|
| 645 | return -EINVAL;
|
| 646 | ASSERT(folio_size(fi.folio) == min_folio_size);
|
| 647 |
|
| 648 | stream = zstd_init_dstream(
|
| 649 | ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
|
| 650 | if (unlikely(!stream)) {
|
| 651 | struct btrfs_inode *inode = cb->bbio.inode;
|
| 652 |
|
| 653 | btrfs_err(inode->root->fs_info,
|
| 654 | "zstd decompression init failed, root %llu inode %llu offset %llu",
|
| 655 | btrfs_root_id(inode->root), btrfs_ino(inode), cb->start);
|
| 656 | ret = -EIO;
|
| 657 | goto done;
|
| 658 | }
|
| 659 |
|
| 660 | workspace->in_buf.src = kmap_local_folio(fi.folio, 0);
|
| 661 | workspace->in_buf.pos = 0;
|
| 662 | workspace->in_buf.size = min_t(size_t, srclen, min_folio_size);
|
| 663 |
|
| 664 | while (orig_bio->bi_iter.bi_size) {
|
| 665 | size_t ret2;
|
| 666 | void *kaddr;
|
| 667 | u32 dstlen;
|
| 668 |
|
| 669 | dstlen = zstd_map_dest(cb, total_out, &kaddr);
|
| 670 | if (kaddr) {
|
| 671 | workspace->out_buf.dst = kaddr;
|
| 672 | workspace->out_buf.size = dstlen;
|
| 673 | } else {
|
| 674 | workspace->out_buf.dst = workspace->buf;
|
| 675 | workspace->out_buf.size = min_t(u32, dstlen,
|
| 676 | fs_info->sectorsize);
|
| 677 | }
|
| 678 | workspace->out_buf.pos = 0;
|
| 679 |
|
| 680 | ret2 = zstd_decompress_stream(stream, &workspace->out_buf,
|
| 681 | &workspace->in_buf);
|
| 682 | if (kaddr)
|
| 683 | kunmap_local(kaddr);
|
| 684 | if (unlikely(zstd_is_error(ret2))) {
|
| 685 | struct btrfs_inode *inode = cb->bbio.inode;
|
| 686 |
|
| 687 | btrfs_err(inode->root->fs_info,
|
| 688 | "zstd decompression failed, error %d root %llu inode %llu offset %llu",
|
| 689 | zstd_get_error_code(ret2), btrfs_root_id(inode->root),
|
| 690 | btrfs_ino(inode), cb->start);
|
| 691 | ret = -EIO;
|
| 692 | goto done;
|
| 693 | }
|
| 694 | total_out += workspace->out_buf.pos;
|
| 695 | if (kaddr)
|
| 696 | bio_advance(orig_bio, workspace->out_buf.pos);
|
| 697 |
|
| 698 | if (workspace->in_buf.pos >= srclen)
|
| 699 | break;
|
| 700 |
|
| 701 |
|
| 702 | if (ret2 == 0)
|
| 703 | break;
|
| 704 |
|
| 705 | if (workspace->in_buf.pos == workspace->in_buf.size) {
|
| 706 | kunmap_local(workspace->in_buf.src);
|
| 707 | folio_in_index++;
|
| 708 | if (unlikely(folio_in_index >= total_folios_in)) {
|
| 709 | workspace->in_buf.src = NULL;
|
| 710 | ret = -EIO;
|
| 711 | goto done;
|
| 712 | }
|
| 713 | srclen -= min_folio_size;
|
| 714 | bio_next_folio(&fi, &cb->bbio.bio);
|
| 715 | ASSERT(fi.folio);
|
| 716 | workspace->in_buf.src = kmap_local_folio(fi.folio, 0);
|
| 717 | workspace->in_buf.pos = 0;
|
| 718 | workspace->in_buf.size = min_t(size_t, srclen, min_folio_size);
|
| 719 | }
|
| 720 | }
|
| 721 | ret = 0;
|
| 722 | done:
|
| 723 | if (workspace->in_buf.src)
|
| 724 | kunmap_local(workspace->in_buf.src);
|
| 725 | return ret;
|
| 726 | }
|
| 727 |
|
| 728 | int zstd_decompress(struct list_head *ws, const u8 *data_in,
|
| 729 | struct folio *dest_folio, unsigned long dest_pgoff, size_t srclen,
|
| 730 | size_t destlen)
|
| 731 | {
|
| 732 | struct workspace *workspace = list_entry(ws, struct workspace, list);
|
| 733 | struct btrfs_fs_info *fs_info = btrfs_sb(folio_inode(dest_folio)->i_sb);
|
| 734 | zstd_dstream *stream;
|
| 735 | int ret = 0;
|
| 736 | unsigned long to_copy = 0;
|
| 737 |
|
| 738 | stream = zstd_init_dstream(
|
| 739 | ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
|
| 740 | if (unlikely(!stream)) {
|
| 741 | struct btrfs_inode *inode = folio_to_inode(dest_folio);
|
| 742 |
|
| 743 | btrfs_err(inode->root->fs_info,
|
| 744 | "zstd decompression init failed, root %llu inode %llu offset %llu",
|
| 745 | btrfs_root_id(inode->root), btrfs_ino(inode),
|
| 746 | folio_pos(dest_folio));
|
| 747 | ret = -EIO;
|
| 748 | goto finish;
|
| 749 | }
|
| 750 |
|
| 751 | workspace->in_buf.src = data_in;
|
| 752 | workspace->in_buf.pos = 0;
|
| 753 | workspace->in_buf.size = srclen;
|
| 754 |
|
| 755 | workspace->out_buf.dst = workspace->buf;
|
| 756 | workspace->out_buf.pos = 0;
|
| 757 | workspace->out_buf.size = fs_info->sectorsize;
|
| 758 |
|
| 759 | |
| 760 | * Since both input and output buffers should not exceed one sector,
|
| 761 | * one call should end the decompression.
|
| 762 |
|
| 763 | ret = zstd_decompress_stream(stream, &workspace->out_buf, &workspace->in_buf);
|
| 764 | if (unlikely(zstd_is_error(ret))) {
|
| 765 | struct btrfs_inode *inode = folio_to_inode(dest_folio);
|
| 766 |
|
| 767 | btrfs_err(inode->root->fs_info,
|
| 768 | "zstd decompression failed, error %d root %llu inode %llu offset %llu",
|
| 769 | zstd_get_error_code(ret), btrfs_root_id(inode->root),
|
| 770 | btrfs_ino(inode), folio_pos(dest_folio));
|
| 771 | goto finish;
|
| 772 | }
|
| 773 | to_copy = workspace->out_buf.pos;
|
| 774 | memcpy_to_folio(dest_folio, dest_pgoff, workspace->out_buf.dst, to_copy);
|
| 775 | finish:
|
| 776 |
|
| 777 | if (unlikely(to_copy < destlen)) {
|
| 778 | ret = -EIO;
|
| 779 | folio_zero_range(dest_folio, dest_pgoff + to_copy, destlen - to_copy);
|
| 780 | }
|
| 781 | return ret;
|
| 782 | }
|
| 783 |
|
| 784 | const struct btrfs_compress_levels btrfs_zstd_compress = {
|
| 785 | .min_level = ZSTD_BTRFS_MIN_LEVEL,
|
| 786 | .max_level = ZSTD_BTRFS_MAX_LEVEL,
|
| 787 | .default_level = ZSTD_BTRFS_DEFAULT_LEVEL,
|
| 788 | };
|