| 1 | |
| 2 | |
| 3 | *
|
| 4 | * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
|
| 5 | * Written by David Howells ([email protected])
|
| 6 |
|
| 7 |
|
| 8 | #include <linux/mount.h>
|
| 9 | #include <linux/slab.h>
|
| 10 | #include <linux/file.h>
|
| 11 | #include <linux/uio.h>
|
| 12 | #include <linux/bio.h>
|
| 13 | #include <linux/falloc.h>
|
| 14 | #include <linux/sched/mm.h>
|
| 15 | #include <trace/events/fscache.h>
|
| 16 | #include <trace/events/netfs.h>
|
| 17 | #include "internal.h"
|
| 18 |
|
| 19 | struct cachefiles_kiocb {
|
| 20 | struct kiocb iocb;
|
| 21 | refcount_t ki_refcnt;
|
| 22 | loff_t start;
|
| 23 | union {
|
| 24 | size_t skipped;
|
| 25 | size_t len;
|
| 26 | };
|
| 27 | struct cachefiles_object *object;
|
| 28 | netfs_io_terminated_t term_func;
|
| 29 | void *term_func_priv;
|
| 30 | bool was_async;
|
| 31 | unsigned int inval_counter;
|
| 32 | u64 b_writing;
|
| 33 | };
|
| 34 |
|
| 35 | static inline void cachefiles_put_kiocb(struct cachefiles_kiocb *ki)
|
| 36 | {
|
| 37 | if (refcount_dec_and_test(&ki->ki_refcnt)) {
|
| 38 | cachefiles_put_object(ki->object, cachefiles_obj_put_ioreq);
|
| 39 | fput(ki->iocb.ki_filp);
|
| 40 | kfree(ki);
|
| 41 | }
|
| 42 | }
|
| 43 |
|
| 44 | |
| 45 | * Handle completion of a read from the cache.
|
| 46 |
|
| 47 | static void cachefiles_read_complete(struct kiocb *iocb, long ret)
|
| 48 | {
|
| 49 | struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb);
|
| 50 | struct inode *inode = file_inode(ki->iocb.ki_filp);
|
| 51 |
|
| 52 | _enter("%ld", ret);
|
| 53 |
|
| 54 | if (ret < 0)
|
| 55 | trace_cachefiles_io_error(ki->object, inode, ret,
|
| 56 | cachefiles_trace_read_error);
|
| 57 |
|
| 58 | if (ki->term_func) {
|
| 59 | if (ret >= 0) {
|
| 60 | if (ki->object->cookie->inval_counter == ki->inval_counter)
|
| 61 | ki->skipped += ret;
|
| 62 | else
|
| 63 | ret = -ESTALE;
|
| 64 | }
|
| 65 |
|
| 66 | ki->term_func(ki->term_func_priv, ret);
|
| 67 | }
|
| 68 |
|
| 69 | cachefiles_put_kiocb(ki);
|
| 70 | }
|
| 71 |
|
| 72 | |
| 73 | * Initiate a read from the cache.
|
| 74 |
|
| 75 | static int cachefiles_read(struct netfs_cache_resources *cres,
|
| 76 | loff_t start_pos,
|
| 77 | struct iov_iter *iter,
|
| 78 | enum netfs_read_from_hole read_hole,
|
| 79 | netfs_io_terminated_t term_func,
|
| 80 | void *term_func_priv)
|
| 81 | {
|
| 82 | struct cachefiles_object *object;
|
| 83 | struct cachefiles_kiocb *ki;
|
| 84 | struct file *file;
|
| 85 | unsigned int old_nofs;
|
| 86 | ssize_t ret = -ENOBUFS;
|
| 87 | size_t len = iov_iter_count(iter), skipped = 0;
|
| 88 |
|
| 89 | if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
|
| 90 | goto presubmission_error;
|
| 91 |
|
| 92 | fscache_count_read();
|
| 93 | object = cachefiles_cres_object(cres);
|
| 94 | file = cachefiles_cres_file(cres);
|
| 95 |
|
| 96 | _enter("%pD,%llu,%llx,%zx/%llx",
|
| 97 | file, file_inode(file)->i_ino, start_pos, len,
|
| 98 | i_size_read(file_inode(file)));
|
| 99 |
|
| 100 | |
| 101 | * we should do that now. If we find a gap, we fill it with zeros.
|
| 102 |
|
| 103 | if (read_hole != NETFS_READ_HOLE_IGNORE) {
|
| 104 | loff_t off = start_pos, off2;
|
| 105 |
|
| 106 | off2 = cachefiles_inject_read_error();
|
| 107 | if (off2 == 0)
|
| 108 | off2 = vfs_llseek(file, off, SEEK_DATA);
|
| 109 | if (off2 < 0 && off2 >= (loff_t)-MAX_ERRNO && off2 != -ENXIO) {
|
| 110 | skipped = 0;
|
| 111 | ret = off2;
|
| 112 | goto presubmission_error;
|
| 113 | }
|
| 114 |
|
| 115 | if (off2 == -ENXIO || off2 >= start_pos + len) {
|
| 116 | |
| 117 | * in the region, so clear the rest of the buffer and
|
| 118 | * return success.
|
| 119 |
|
| 120 | ret = -ENODATA;
|
| 121 | if (read_hole == NETFS_READ_HOLE_FAIL)
|
| 122 | goto presubmission_error;
|
| 123 |
|
| 124 | iov_iter_zero(len, iter);
|
| 125 | skipped = len;
|
| 126 | ret = 0;
|
| 127 | goto presubmission_error;
|
| 128 | }
|
| 129 |
|
| 130 | skipped = off2 - off;
|
| 131 | iov_iter_zero(skipped, iter);
|
| 132 | }
|
| 133 |
|
| 134 | ret = -ENOMEM;
|
| 135 | ki = kzalloc_obj(struct cachefiles_kiocb);
|
| 136 | if (!ki)
|
| 137 | goto presubmission_error;
|
| 138 |
|
| 139 | refcount_set(&ki->ki_refcnt, 2);
|
| 140 | ki->iocb.ki_filp = file;
|
| 141 | ki->iocb.ki_pos = start_pos + skipped;
|
| 142 | ki->iocb.ki_flags = IOCB_DIRECT;
|
| 143 | ki->iocb.ki_ioprio = get_current_ioprio();
|
| 144 | ki->skipped = skipped;
|
| 145 | ki->object = object;
|
| 146 | ki->inval_counter = cres->inval_counter;
|
| 147 | ki->term_func = term_func;
|
| 148 | ki->term_func_priv = term_func_priv;
|
| 149 | ki->was_async = true;
|
| 150 |
|
| 151 | if (ki->term_func)
|
| 152 | ki->iocb.ki_complete = cachefiles_read_complete;
|
| 153 |
|
| 154 | get_file(ki->iocb.ki_filp);
|
| 155 | cachefiles_grab_object(object, cachefiles_obj_get_ioreq);
|
| 156 |
|
| 157 | trace_cachefiles_read(object, file_inode(file), ki->iocb.ki_pos, len - skipped);
|
| 158 | old_nofs = memalloc_nofs_save();
|
| 159 | ret = cachefiles_inject_read_error();
|
| 160 | if (ret == 0)
|
| 161 | ret = vfs_iocb_iter_read(file, &ki->iocb, iter);
|
| 162 | memalloc_nofs_restore(old_nofs);
|
| 163 | switch (ret) {
|
| 164 | case -EIOCBQUEUED:
|
| 165 | goto in_progress;
|
| 166 |
|
| 167 | case -ERESTARTSYS:
|
| 168 | case -ERESTARTNOINTR:
|
| 169 | case -ERESTARTNOHAND:
|
| 170 | case -ERESTART_RESTARTBLOCK:
|
| 171 | |
| 172 | * may be already running. Just fail this IO with EINTR.
|
| 173 |
|
| 174 | ret = -EINTR;
|
| 175 | fallthrough;
|
| 176 | default:
|
| 177 | ki->was_async = false;
|
| 178 | cachefiles_read_complete(&ki->iocb, ret);
|
| 179 | if (ret > 0)
|
| 180 | ret = 0;
|
| 181 | break;
|
| 182 | }
|
| 183 |
|
| 184 | in_progress:
|
| 185 | cachefiles_put_kiocb(ki);
|
| 186 | _leave(" = %zd", ret);
|
| 187 | return ret;
|
| 188 |
|
| 189 | presubmission_error:
|
| 190 | if (term_func)
|
| 191 | term_func(term_func_priv, ret < 0 ? ret : skipped);
|
| 192 | return ret;
|
| 193 | }
|
| 194 |
|
| 195 | |
| 196 | * Query the occupancy of the cache in a region, returning where the next chunk
|
| 197 | * of data starts and how long it is.
|
| 198 |
|
| 199 | static int cachefiles_query_occupancy(struct netfs_cache_resources *cres,
|
| 200 | loff_t start, size_t len, size_t granularity,
|
| 201 | loff_t *_data_start, size_t *_data_len)
|
| 202 | {
|
| 203 | struct cachefiles_object *object;
|
| 204 | struct file *file;
|
| 205 | loff_t off, off2;
|
| 206 |
|
| 207 | *_data_start = -1;
|
| 208 | *_data_len = 0;
|
| 209 |
|
| 210 | if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
|
| 211 | return -ENOBUFS;
|
| 212 |
|
| 213 | object = cachefiles_cres_object(cres);
|
| 214 | file = cachefiles_cres_file(cres);
|
| 215 | granularity = max_t(size_t, object->volume->cache->bsize, granularity);
|
| 216 |
|
| 217 | _enter("%pD,%llu,%llx,%zx/%llx",
|
| 218 | file, file_inode(file)->i_ino, start, len,
|
| 219 | i_size_read(file_inode(file)));
|
| 220 |
|
| 221 | off = cachefiles_inject_read_error();
|
| 222 | if (off == 0)
|
| 223 | off = vfs_llseek(file, start, SEEK_DATA);
|
| 224 | if (off == -ENXIO)
|
| 225 | return -ENODATA;
|
| 226 | if (off < 0 && off >= (loff_t)-MAX_ERRNO)
|
| 227 | return -ENOBUFS;
|
| 228 | if (round_up(off, granularity) >= start + len)
|
| 229 | return -ENODATA;
|
| 230 |
|
| 231 | off2 = cachefiles_inject_read_error();
|
| 232 | if (off2 == 0)
|
| 233 | off2 = vfs_llseek(file, off, SEEK_HOLE);
|
| 234 | if (off2 == -ENXIO)
|
| 235 | return -ENODATA;
|
| 236 | if (off2 < 0 && off2 >= (loff_t)-MAX_ERRNO)
|
| 237 | return -ENOBUFS;
|
| 238 |
|
| 239 |
|
| 240 | off = round_up(off, granularity);
|
| 241 | off2 = round_down(off2, granularity);
|
| 242 | if (off2 <= off)
|
| 243 | return -ENODATA;
|
| 244 |
|
| 245 | *_data_start = off;
|
| 246 | if (off2 > start + len)
|
| 247 | *_data_len = len;
|
| 248 | else
|
| 249 | *_data_len = off2 - off;
|
| 250 | return 0;
|
| 251 | }
|
| 252 |
|
| 253 | |
| 254 | * Handle completion of a write to the cache.
|
| 255 |
|
| 256 | static void cachefiles_write_complete(struct kiocb *iocb, long ret)
|
| 257 | {
|
| 258 | struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb);
|
| 259 | struct cachefiles_object *object = ki->object;
|
| 260 | struct inode *inode = file_inode(ki->iocb.ki_filp);
|
| 261 |
|
| 262 | _enter("%ld", ret);
|
| 263 |
|
| 264 | if (ki->was_async)
|
| 265 | kiocb_end_write(iocb);
|
| 266 |
|
| 267 | if (ret < 0)
|
| 268 | trace_cachefiles_io_error(object, inode, ret,
|
| 269 | cachefiles_trace_write_error);
|
| 270 |
|
| 271 | atomic_long_sub(ki->b_writing, &object->volume->cache->b_writing);
|
| 272 | set_bit(FSCACHE_COOKIE_HAVE_DATA, &object->cookie->flags);
|
| 273 | if (ki->term_func)
|
| 274 | ki->term_func(ki->term_func_priv, ret);
|
| 275 | cachefiles_put_kiocb(ki);
|
| 276 | }
|
| 277 |
|
| 278 | |
| 279 | * Initiate a write to the cache.
|
| 280 |
|
| 281 | int __cachefiles_write(struct cachefiles_object *object,
|
| 282 | struct file *file,
|
| 283 | loff_t start_pos,
|
| 284 | struct iov_iter *iter,
|
| 285 | netfs_io_terminated_t term_func,
|
| 286 | void *term_func_priv)
|
| 287 | {
|
| 288 | struct cachefiles_cache *cache;
|
| 289 | struct cachefiles_kiocb *ki;
|
| 290 | unsigned int old_nofs;
|
| 291 | ssize_t ret;
|
| 292 | size_t len = iov_iter_count(iter);
|
| 293 |
|
| 294 | fscache_count_write();
|
| 295 | cache = object->volume->cache;
|
| 296 |
|
| 297 | _enter("%pD,%llu,%llx,%zx/%llx",
|
| 298 | file, file_inode(file)->i_ino, start_pos, len,
|
| 299 | i_size_read(file_inode(file)));
|
| 300 |
|
| 301 | ki = kzalloc_obj(struct cachefiles_kiocb);
|
| 302 | if (!ki) {
|
| 303 | if (term_func)
|
| 304 | term_func(term_func_priv, -ENOMEM);
|
| 305 | return -ENOMEM;
|
| 306 | }
|
| 307 |
|
| 308 | refcount_set(&ki->ki_refcnt, 2);
|
| 309 | ki->iocb.ki_filp = file;
|
| 310 | ki->iocb.ki_pos = start_pos;
|
| 311 | ki->iocb.ki_flags = IOCB_DIRECT | IOCB_WRITE;
|
| 312 | ki->iocb.ki_ioprio = get_current_ioprio();
|
| 313 | ki->object = object;
|
| 314 | ki->start = start_pos;
|
| 315 | ki->len = len;
|
| 316 | ki->term_func = term_func;
|
| 317 | ki->term_func_priv = term_func_priv;
|
| 318 | ki->was_async = true;
|
| 319 | ki->b_writing = (len + (1 << cache->bshift) - 1) >> cache->bshift;
|
| 320 |
|
| 321 | if (ki->term_func)
|
| 322 | ki->iocb.ki_complete = cachefiles_write_complete;
|
| 323 | atomic_long_add(ki->b_writing, &cache->b_writing);
|
| 324 |
|
| 325 | get_file(ki->iocb.ki_filp);
|
| 326 | cachefiles_grab_object(object, cachefiles_obj_get_ioreq);
|
| 327 |
|
| 328 | trace_cachefiles_write(object, file_inode(file), ki->iocb.ki_pos, len);
|
| 329 | old_nofs = memalloc_nofs_save();
|
| 330 | ret = cachefiles_inject_write_error();
|
| 331 | if (ret == 0)
|
| 332 | ret = vfs_iocb_iter_write(file, &ki->iocb, iter);
|
| 333 | memalloc_nofs_restore(old_nofs);
|
| 334 | switch (ret) {
|
| 335 | case -EIOCBQUEUED:
|
| 336 | goto in_progress;
|
| 337 |
|
| 338 | case -ERESTARTSYS:
|
| 339 | case -ERESTARTNOINTR:
|
| 340 | case -ERESTARTNOHAND:
|
| 341 | case -ERESTART_RESTARTBLOCK:
|
| 342 | |
| 343 | * may be already running. Just fail this IO with EINTR.
|
| 344 |
|
| 345 | ret = -EINTR;
|
| 346 | fallthrough;
|
| 347 | default:
|
| 348 | ki->was_async = false;
|
| 349 | cachefiles_write_complete(&ki->iocb, ret);
|
| 350 | break;
|
| 351 | }
|
| 352 |
|
| 353 | in_progress:
|
| 354 | cachefiles_put_kiocb(ki);
|
| 355 | _leave(" = %zd", ret);
|
| 356 | return ret;
|
| 357 | }
|
| 358 |
|
| 359 | static int cachefiles_write(struct netfs_cache_resources *cres,
|
| 360 | loff_t start_pos,
|
| 361 | struct iov_iter *iter,
|
| 362 | netfs_io_terminated_t term_func,
|
| 363 | void *term_func_priv)
|
| 364 | {
|
| 365 | if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE)) {
|
| 366 | if (term_func)
|
| 367 | term_func(term_func_priv, -ENOBUFS);
|
| 368 | trace_netfs_sreq(term_func_priv, netfs_sreq_trace_cache_nowrite);
|
| 369 | return -ENOBUFS;
|
| 370 | }
|
| 371 |
|
| 372 | return __cachefiles_write(cachefiles_cres_object(cres),
|
| 373 | cachefiles_cres_file(cres),
|
| 374 | start_pos, iter,
|
| 375 | term_func, term_func_priv);
|
| 376 | }
|
| 377 |
|
| 378 | static inline enum netfs_io_source
|
| 379 | cachefiles_do_prepare_read(struct netfs_cache_resources *cres,
|
| 380 | loff_t start, size_t *_len, loff_t i_size,
|
| 381 | unsigned long *_flags, ino_t netfs_ino)
|
| 382 | {
|
| 383 | enum cachefiles_prepare_read_trace why;
|
| 384 | struct cachefiles_object *object = NULL;
|
| 385 | struct cachefiles_cache *cache;
|
| 386 | struct fscache_cookie *cookie = fscache_cres_cookie(cres);
|
| 387 | const struct cred *saved_cred;
|
| 388 | struct file *file = cachefiles_cres_file(cres);
|
| 389 | enum netfs_io_source ret = NETFS_DOWNLOAD_FROM_SERVER;
|
| 390 | size_t len = *_len;
|
| 391 | loff_t off, to;
|
| 392 | ino_t ino = file ? file_inode(file)->i_ino : 0;
|
| 393 |
|
| 394 | _enter("%zx @%llx/%llx", len, start, i_size);
|
| 395 |
|
| 396 | if (start >= i_size) {
|
| 397 | ret = NETFS_FILL_WITH_ZEROES;
|
| 398 | why = cachefiles_trace_read_after_eof;
|
| 399 | goto out_no_object;
|
| 400 | }
|
| 401 |
|
| 402 | if (test_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags)) {
|
| 403 | __set_bit(NETFS_SREQ_COPY_TO_CACHE, _flags);
|
| 404 | why = cachefiles_trace_read_no_data;
|
| 405 | goto out_no_object;
|
| 406 | }
|
| 407 |
|
| 408 |
|
| 409 | if (!file) {
|
| 410 | why = cachefiles_trace_read_no_file;
|
| 411 | if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
|
| 412 | goto out_no_object;
|
| 413 | file = cachefiles_cres_file(cres);
|
| 414 | if (!file)
|
| 415 | goto out_no_object;
|
| 416 | ino = file_inode(file)->i_ino;
|
| 417 | }
|
| 418 |
|
| 419 | object = cachefiles_cres_object(cres);
|
| 420 | cache = object->volume->cache;
|
| 421 | cachefiles_begin_secure(cache, &saved_cred);
|
| 422 | off = cachefiles_inject_read_error();
|
| 423 | if (off == 0)
|
| 424 | off = vfs_llseek(file, start, SEEK_DATA);
|
| 425 | if (off < 0 && off >= (loff_t)-MAX_ERRNO) {
|
| 426 | if (off == (loff_t)-ENXIO) {
|
| 427 | why = cachefiles_trace_read_seek_nxio;
|
| 428 | goto download_and_store;
|
| 429 | }
|
| 430 | trace_cachefiles_io_error(object, file_inode(file), off,
|
| 431 | cachefiles_trace_seek_error);
|
| 432 | why = cachefiles_trace_read_seek_error;
|
| 433 | goto out;
|
| 434 | }
|
| 435 |
|
| 436 | if (off >= start + len) {
|
| 437 | why = cachefiles_trace_read_found_hole;
|
| 438 | goto download_and_store;
|
| 439 | }
|
| 440 |
|
| 441 | if (off > start) {
|
| 442 | off = round_up(off, cache->bsize);
|
| 443 | len = off - start;
|
| 444 | *_len = len;
|
| 445 | why = cachefiles_trace_read_found_part;
|
| 446 | goto download_and_store;
|
| 447 | }
|
| 448 |
|
| 449 | to = cachefiles_inject_read_error();
|
| 450 | if (to == 0)
|
| 451 | to = vfs_llseek(file, start, SEEK_HOLE);
|
| 452 | if (to < 0 && to >= (loff_t)-MAX_ERRNO) {
|
| 453 | trace_cachefiles_io_error(object, file_inode(file), to,
|
| 454 | cachefiles_trace_seek_error);
|
| 455 | why = cachefiles_trace_read_seek_error;
|
| 456 | goto out;
|
| 457 | }
|
| 458 |
|
| 459 | if (to < start + len) {
|
| 460 | if (start + len >= i_size)
|
| 461 | to = round_up(to, cache->bsize);
|
| 462 | else
|
| 463 | to = round_down(to, cache->bsize);
|
| 464 | len = to - start;
|
| 465 | *_len = len;
|
| 466 | }
|
| 467 |
|
| 468 | why = cachefiles_trace_read_have_data;
|
| 469 | ret = NETFS_READ_FROM_CACHE;
|
| 470 | goto out;
|
| 471 |
|
| 472 | download_and_store:
|
| 473 | __set_bit(NETFS_SREQ_COPY_TO_CACHE, _flags);
|
| 474 | out:
|
| 475 | cachefiles_end_secure(cache, saved_cred);
|
| 476 | out_no_object:
|
| 477 | trace_cachefiles_prep_read(object, start, len, *_flags, ret, why, ino, netfs_ino);
|
| 478 | return ret;
|
| 479 | }
|
| 480 |
|
| 481 | |
| 482 | * Prepare a read operation, shortening it to a cached/uncached
|
| 483 | * boundary as appropriate.
|
| 484 |
|
| 485 | static enum netfs_io_source cachefiles_prepare_read(struct netfs_io_subrequest *subreq,
|
| 486 | unsigned long long i_size)
|
| 487 | {
|
| 488 | return cachefiles_do_prepare_read(&subreq->rreq->cache_resources,
|
| 489 | subreq->start, &subreq->len, i_size,
|
| 490 | &subreq->flags, subreq->rreq->inode->i_ino);
|
| 491 | }
|
| 492 |
|
| 493 | |
| 494 | * Prepare for a write to occur.
|
| 495 |
|
| 496 | int __cachefiles_prepare_write(struct cachefiles_object *object,
|
| 497 | struct file *file,
|
| 498 | loff_t *_start, size_t *_len, size_t upper_len,
|
| 499 | bool no_space_allocated_yet)
|
| 500 | {
|
| 501 | struct cachefiles_cache *cache = object->volume->cache;
|
| 502 | loff_t start = *_start, pos;
|
| 503 | size_t len = *_len;
|
| 504 | int ret;
|
| 505 |
|
| 506 |
|
| 507 | start = round_down(*_start, PAGE_SIZE);
|
| 508 | if (start != *_start || *_len > upper_len) {
|
| 509 | |
| 510 | * pagecache when the cookie was temporarily out of service to
|
| 511 | * culling.
|
| 512 |
|
| 513 | fscache_count_dio_misfit();
|
| 514 | return -ENOBUFS;
|
| 515 | }
|
| 516 |
|
| 517 | *_len = round_up(len, PAGE_SIZE);
|
| 518 |
|
| 519 | |
| 520 | * the write - but we can skip that check if we have space already
|
| 521 | * allocated.
|
| 522 |
|
| 523 | if (no_space_allocated_yet)
|
| 524 | goto check_space;
|
| 525 |
|
| 526 | pos = cachefiles_inject_read_error();
|
| 527 | if (pos == 0)
|
| 528 | pos = vfs_llseek(file, start, SEEK_DATA);
|
| 529 | if (pos < 0 && pos >= (loff_t)-MAX_ERRNO) {
|
| 530 | if (pos == -ENXIO)
|
| 531 | goto check_space;
|
| 532 | trace_cachefiles_io_error(object, file_inode(file), pos,
|
| 533 | cachefiles_trace_seek_error);
|
| 534 | return pos;
|
| 535 | }
|
| 536 | if ((u64)pos >= (u64)start + *_len)
|
| 537 | goto check_space;
|
| 538 |
|
| 539 | |
| 540 | * space, we need to see if it's fully allocated. If it's not, we may
|
| 541 | * want to cull it.
|
| 542 |
|
| 543 | if (cachefiles_has_space(cache, 0, *_len / PAGE_SIZE,
|
| 544 | cachefiles_has_space_check) == 0)
|
| 545 | return 0;
|
| 546 |
|
| 547 | pos = cachefiles_inject_read_error();
|
| 548 | if (pos == 0)
|
| 549 | pos = vfs_llseek(file, start, SEEK_HOLE);
|
| 550 | if (pos < 0 && pos >= (loff_t)-MAX_ERRNO) {
|
| 551 | trace_cachefiles_io_error(object, file_inode(file), pos,
|
| 552 | cachefiles_trace_seek_error);
|
| 553 | return pos;
|
| 554 | }
|
| 555 | if ((u64)pos >= (u64)start + *_len)
|
| 556 | return 0;
|
| 557 |
|
| 558 |
|
| 559 | fscache_count_no_write_space();
|
| 560 | ret = cachefiles_inject_remove_error();
|
| 561 | if (ret == 0)
|
| 562 | ret = vfs_fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
|
| 563 | start, *_len);
|
| 564 | if (ret < 0) {
|
| 565 | trace_cachefiles_io_error(object, file_inode(file), ret,
|
| 566 | cachefiles_trace_fallocate_error);
|
| 567 | cachefiles_io_error_obj(object,
|
| 568 | "CacheFiles: fallocate failed (%d)\n", ret);
|
| 569 | ret = -EIO;
|
| 570 | }
|
| 571 |
|
| 572 | return ret;
|
| 573 |
|
| 574 | check_space:
|
| 575 | return cachefiles_has_space(cache, 0, *_len / PAGE_SIZE,
|
| 576 | cachefiles_has_space_for_write);
|
| 577 | }
|
| 578 |
|
| 579 | static int cachefiles_prepare_write(struct netfs_cache_resources *cres,
|
| 580 | loff_t *_start, size_t *_len, size_t upper_len,
|
| 581 | loff_t i_size, bool no_space_allocated_yet)
|
| 582 | {
|
| 583 | struct cachefiles_object *object = cachefiles_cres_object(cres);
|
| 584 | struct cachefiles_cache *cache = object->volume->cache;
|
| 585 | const struct cred *saved_cred;
|
| 586 | int ret;
|
| 587 |
|
| 588 | if (!cachefiles_cres_file(cres)) {
|
| 589 | if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE))
|
| 590 | return -ENOBUFS;
|
| 591 | if (!cachefiles_cres_file(cres))
|
| 592 | return -ENOBUFS;
|
| 593 | }
|
| 594 |
|
| 595 | cachefiles_begin_secure(cache, &saved_cred);
|
| 596 | ret = __cachefiles_prepare_write(object, cachefiles_cres_file(cres),
|
| 597 | _start, _len, upper_len,
|
| 598 | no_space_allocated_yet);
|
| 599 | cachefiles_end_secure(cache, saved_cred);
|
| 600 | return ret;
|
| 601 | }
|
| 602 |
|
| 603 | static void cachefiles_prepare_write_subreq(struct netfs_io_subrequest *subreq)
|
| 604 | {
|
| 605 | struct netfs_io_request *wreq = subreq->rreq;
|
| 606 | struct netfs_cache_resources *cres = &wreq->cache_resources;
|
| 607 | struct netfs_io_stream *stream = &wreq->io_streams[subreq->stream_nr];
|
| 608 |
|
| 609 | _enter("W=%x[%x] %llx", wreq->debug_id, subreq->debug_index, subreq->start);
|
| 610 |
|
| 611 | stream->sreq_max_len = MAX_RW_COUNT;
|
| 612 | stream->sreq_max_segs = BIO_MAX_VECS;
|
| 613 |
|
| 614 | if (!cachefiles_cres_file(cres)) {
|
| 615 | if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE))
|
| 616 | return netfs_prepare_write_failed(subreq);
|
| 617 | if (!cachefiles_cres_file(cres))
|
| 618 | return netfs_prepare_write_failed(subreq);
|
| 619 | }
|
| 620 | }
|
| 621 |
|
| 622 | static void cachefiles_issue_write(struct netfs_io_subrequest *subreq)
|
| 623 | {
|
| 624 | struct netfs_io_request *wreq = subreq->rreq;
|
| 625 | struct netfs_cache_resources *cres = &wreq->cache_resources;
|
| 626 | struct cachefiles_object *object = cachefiles_cres_object(cres);
|
| 627 | struct cachefiles_cache *cache = object->volume->cache;
|
| 628 | struct netfs_io_stream *stream = &wreq->io_streams[subreq->stream_nr];
|
| 629 | const struct cred *saved_cred;
|
| 630 | size_t off, pre, post, len = subreq->len;
|
| 631 | loff_t start = subreq->start;
|
| 632 | int ret;
|
| 633 |
|
| 634 | _enter("W=%x[%x] %llx-%llx",
|
| 635 | wreq->debug_id, subreq->debug_index, start, start + len - 1);
|
| 636 |
|
| 637 |
|
| 638 | off = start & (CACHEFILES_DIO_BLOCK_SIZE - 1);
|
| 639 | if (off) {
|
| 640 | pre = CACHEFILES_DIO_BLOCK_SIZE - off;
|
| 641 | if (pre >= len) {
|
| 642 | fscache_count_dio_misfit();
|
| 643 | netfs_write_subrequest_terminated(subreq, len);
|
| 644 | return;
|
| 645 | }
|
| 646 | subreq->transferred += pre;
|
| 647 | start += pre;
|
| 648 | len -= pre;
|
| 649 | iov_iter_advance(&subreq->io_iter, pre);
|
| 650 | }
|
| 651 |
|
| 652 |
|
| 653 | if (start + len == wreq->i_size) {
|
| 654 | size_t part = len % CACHEFILES_DIO_BLOCK_SIZE;
|
| 655 | size_t need = CACHEFILES_DIO_BLOCK_SIZE - part;
|
| 656 |
|
| 657 | if (part && stream->submit_extendable_to >= need) {
|
| 658 | len += need;
|
| 659 | subreq->len += need;
|
| 660 | subreq->io_iter.count += need;
|
| 661 | }
|
| 662 | }
|
| 663 |
|
| 664 | post = len & (CACHEFILES_DIO_BLOCK_SIZE - 1);
|
| 665 | if (post) {
|
| 666 | len -= post;
|
| 667 | if (len == 0) {
|
| 668 | fscache_count_dio_misfit();
|
| 669 | netfs_write_subrequest_terminated(subreq, post);
|
| 670 | return;
|
| 671 | }
|
| 672 | iov_iter_truncate(&subreq->io_iter, len);
|
| 673 | }
|
| 674 |
|
| 675 | trace_netfs_sreq(subreq, netfs_sreq_trace_cache_prepare);
|
| 676 | cachefiles_begin_secure(cache, &saved_cred);
|
| 677 | ret = __cachefiles_prepare_write(object, cachefiles_cres_file(cres),
|
| 678 | &start, &len, len, true);
|
| 679 | cachefiles_end_secure(cache, saved_cred);
|
| 680 | if (ret < 0) {
|
| 681 | netfs_write_subrequest_terminated(subreq, ret);
|
| 682 | return;
|
| 683 | }
|
| 684 |
|
| 685 | trace_netfs_sreq(subreq, netfs_sreq_trace_cache_write);
|
| 686 | cachefiles_write(&subreq->rreq->cache_resources,
|
| 687 | subreq->start, &subreq->io_iter,
|
| 688 | netfs_write_subrequest_terminated, subreq);
|
| 689 | }
|
| 690 |
|
| 691 | |
| 692 | * Clean up an operation.
|
| 693 |
|
| 694 | static void cachefiles_end_operation(struct netfs_cache_resources *cres)
|
| 695 | {
|
| 696 | struct file *file = cachefiles_cres_file(cres);
|
| 697 |
|
| 698 | if (file)
|
| 699 | fput(file);
|
| 700 | fscache_end_cookie_access(fscache_cres_cookie(cres), fscache_access_io_end);
|
| 701 | }
|
| 702 |
|
| 703 | static const struct netfs_cache_ops cachefiles_netfs_cache_ops = {
|
| 704 | .end_operation = cachefiles_end_operation,
|
| 705 | .read = cachefiles_read,
|
| 706 | .write = cachefiles_write,
|
| 707 | .issue_write = cachefiles_issue_write,
|
| 708 | .prepare_read = cachefiles_prepare_read,
|
| 709 | .prepare_write = cachefiles_prepare_write,
|
| 710 | .prepare_write_subreq = cachefiles_prepare_write_subreq,
|
| 711 | .query_occupancy = cachefiles_query_occupancy,
|
| 712 | };
|
| 713 |
|
| 714 | |
| 715 | * Open the cache file when beginning a cache operation.
|
| 716 |
|
| 717 | bool cachefiles_begin_operation(struct netfs_cache_resources *cres,
|
| 718 | enum fscache_want_state want_state)
|
| 719 | {
|
| 720 | struct cachefiles_object *object = cachefiles_cres_object(cres);
|
| 721 |
|
| 722 | if (!cachefiles_cres_file(cres)) {
|
| 723 | cres->ops = &cachefiles_netfs_cache_ops;
|
| 724 | if (object->file) {
|
| 725 | spin_lock(&object->lock);
|
| 726 | if (!cres->cache_priv2 && object->file)
|
| 727 | cres->cache_priv2 = get_file(object->file);
|
| 728 | spin_unlock(&object->lock);
|
| 729 | }
|
| 730 | }
|
| 731 |
|
| 732 | if (!cachefiles_cres_file(cres) && want_state != FSCACHE_WANT_PARAMS) {
|
| 733 | pr_err("failed to get cres->file\n");
|
| 734 | return false;
|
| 735 | }
|
| 736 |
|
| 737 | return true;
|
| 738 | }
|