| 1 | |
| 2 | |
| 3 | FUSE: Filesystem in Userspace
|
| 4 | Copyright (C) 2001-2008 Miklos Szeredi <[email protected]>
|
| 5 |
|
| 6 |
|
| 7 | #include "fuse_i.h"
|
| 8 | #include "dev.h"
|
| 9 |
|
| 10 | #include <linux/pagemap.h>
|
| 11 | #include <linux/slab.h>
|
| 12 | #include <linux/kernel.h>
|
| 13 | #include <linux/sched.h>
|
| 14 | #include <linux/sched/signal.h>
|
| 15 | #include <linux/module.h>
|
| 16 | #include <linux/swap.h>
|
| 17 | #include <linux/falloc.h>
|
| 18 | #include <linux/uio.h>
|
| 19 | #include <linux/fs.h>
|
| 20 | #include <linux/filelock.h>
|
| 21 | #include <linux/splice.h>
|
| 22 | #include <linux/task_io_accounting_ops.h>
|
| 23 | #include <linux/iomap.h>
|
| 24 |
|
| 25 | static int fuse_send_open(struct fuse_mount *fm, u64 nodeid,
|
| 26 | unsigned int open_flags, int opcode,
|
| 27 | struct fuse_open_out *outargp)
|
| 28 | {
|
| 29 | struct fuse_open_in inarg;
|
| 30 | FUSE_ARGS(args);
|
| 31 |
|
| 32 | memset(&inarg, 0, sizeof(inarg));
|
| 33 | inarg.flags = open_flags & ~(O_CREAT | O_EXCL | O_NOCTTY);
|
| 34 | if (!fm->fc->atomic_o_trunc)
|
| 35 | inarg.flags &= ~O_TRUNC;
|
| 36 |
|
| 37 | if (fm->fc->handle_killpriv_v2 &&
|
| 38 | (inarg.flags & O_TRUNC) && !capable(CAP_FSETID)) {
|
| 39 | inarg.open_flags |= FUSE_OPEN_KILL_SUIDGID;
|
| 40 | }
|
| 41 |
|
| 42 | args.opcode = opcode;
|
| 43 | args.nodeid = nodeid;
|
| 44 | args.in_numargs = 1;
|
| 45 | args.in_args[0].size = sizeof(inarg);
|
| 46 | args.in_args[0].value = &inarg;
|
| 47 | args.out_numargs = 1;
|
| 48 | args.out_args[0].size = sizeof(*outargp);
|
| 49 | args.out_args[0].value = outargp;
|
| 50 |
|
| 51 | return fuse_simple_request(fm, &args);
|
| 52 | }
|
| 53 |
|
| 54 | struct fuse_file *fuse_file_alloc(struct fuse_mount *fm, bool release)
|
| 55 | {
|
| 56 | struct fuse_file *ff;
|
| 57 |
|
| 58 | ff = kzalloc_obj(struct fuse_file, GFP_KERNEL_ACCOUNT);
|
| 59 | if (unlikely(!ff))
|
| 60 | return NULL;
|
| 61 |
|
| 62 | ff->fm = fm;
|
| 63 | if (release) {
|
| 64 | ff->args = kzalloc_obj(*ff->args, GFP_KERNEL_ACCOUNT);
|
| 65 | if (!ff->args) {
|
| 66 | kfree(ff);
|
| 67 | return NULL;
|
| 68 | }
|
| 69 | }
|
| 70 |
|
| 71 | INIT_LIST_HEAD(&ff->write_entry);
|
| 72 | refcount_set(&ff->count, 1);
|
| 73 | RB_CLEAR_NODE(&ff->polled_node);
|
| 74 | init_waitqueue_head(&ff->poll_wait);
|
| 75 |
|
| 76 | ff->kh = atomic64_inc_return(&fm->fc->khctr);
|
| 77 |
|
| 78 | return ff;
|
| 79 | }
|
| 80 |
|
| 81 | void fuse_file_free(struct fuse_file *ff)
|
| 82 | {
|
| 83 | kfree(ff->args);
|
| 84 | kfree(ff);
|
| 85 | }
|
| 86 |
|
| 87 | static struct fuse_file *fuse_file_get(struct fuse_file *ff)
|
| 88 | {
|
| 89 | refcount_inc(&ff->count);
|
| 90 | return ff;
|
| 91 | }
|
| 92 |
|
| 93 | static void fuse_release_end(struct fuse_args *args, int error)
|
| 94 | {
|
| 95 | struct fuse_release_args *ra = container_of(args, typeof(*ra), args);
|
| 96 |
|
| 97 | iput(ra->inode);
|
| 98 | kfree(ra);
|
| 99 | }
|
| 100 |
|
| 101 | static void fuse_file_put(struct fuse_file *ff, bool sync)
|
| 102 | {
|
| 103 | if (refcount_dec_and_test(&ff->count)) {
|
| 104 | struct fuse_release_args *ra = &ff->args->release_args;
|
| 105 | struct fuse_args *args = (ra ? &ra->args : NULL);
|
| 106 |
|
| 107 | if (ra && ra->inode)
|
| 108 | fuse_file_io_release(ff, ra->inode);
|
| 109 |
|
| 110 | if (!args) {
|
| 111 |
|
| 112 | } else if (args->opcode == FUSE_RELEASE && ff->fm->fc->no_open) {
|
| 113 | fuse_release_end(args, 0);
|
| 114 | } else if (sync) {
|
| 115 | fuse_simple_request(ff->fm, args);
|
| 116 | fuse_release_end(args, 0);
|
| 117 | } else {
|
| 118 | |
| 119 | * DAX inodes may need to issue a number of synchronous
|
| 120 | * request for clearing the mappings.
|
| 121 |
|
| 122 | if (ra && ra->inode && FUSE_IS_DAX(ra->inode))
|
| 123 | args->may_block = true;
|
| 124 | args->end = fuse_release_end;
|
| 125 | if (fuse_simple_background(ff->fm, args,
|
| 126 | GFP_KERNEL | __GFP_NOFAIL))
|
| 127 | fuse_release_end(args, -ENOTCONN);
|
| 128 | }
|
| 129 | kfree(ff);
|
| 130 | }
|
| 131 | }
|
| 132 |
|
| 133 | struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid,
|
| 134 | unsigned int open_flags, bool isdir)
|
| 135 | {
|
| 136 | struct fuse_conn *fc = fm->fc;
|
| 137 | struct fuse_file *ff;
|
| 138 | int opcode = isdir ? FUSE_OPENDIR : FUSE_OPEN;
|
| 139 | bool open = isdir ? !fc->no_opendir : !fc->no_open;
|
| 140 | bool release = !isdir || open;
|
| 141 |
|
| 142 | |
| 143 | * ff->args->release_args still needs to be allocated (so we can hold an
|
| 144 | * inode reference while there are pending inflight file operations when
|
| 145 | * ->release() is called, see fuse_prepare_release()) even if
|
| 146 | * fc->no_open is set else it becomes possible for reclaim to deadlock
|
| 147 | * if while servicing the readahead request the server triggers reclaim
|
| 148 | * and reclaim evicts the inode of the file being read ahead.
|
| 149 |
|
| 150 | ff = fuse_file_alloc(fm, release);
|
| 151 | if (!ff)
|
| 152 | return ERR_PTR(-ENOMEM);
|
| 153 |
|
| 154 | ff->fh = 0;
|
| 155 |
|
| 156 | ff->open_flags = FOPEN_KEEP_CACHE | (isdir ? FOPEN_CACHE_DIR : 0);
|
| 157 | if (open) {
|
| 158 |
|
| 159 | struct fuse_open_out *outargp = &ff->args->open_outarg;
|
| 160 | int err;
|
| 161 |
|
| 162 | err = fuse_send_open(fm, nodeid, open_flags, opcode, outargp);
|
| 163 | if (!err) {
|
| 164 | ff->fh = outargp->fh;
|
| 165 | ff->open_flags = outargp->open_flags;
|
| 166 | } else if (err != -ENOSYS) {
|
| 167 | fuse_file_free(ff);
|
| 168 | return ERR_PTR(err);
|
| 169 | } else {
|
| 170 | if (isdir) {
|
| 171 |
|
| 172 | kfree(ff->args);
|
| 173 | ff->args = NULL;
|
| 174 | fc->no_opendir = 1;
|
| 175 | } else {
|
| 176 | fc->no_open = 1;
|
| 177 | }
|
| 178 | }
|
| 179 | }
|
| 180 |
|
| 181 | if (isdir)
|
| 182 | ff->open_flags &= ~FOPEN_DIRECT_IO;
|
| 183 |
|
| 184 | ff->nodeid = nodeid;
|
| 185 |
|
| 186 | return ff;
|
| 187 | }
|
| 188 |
|
| 189 | int fuse_do_open(struct fuse_mount *fm, u64 nodeid, struct file *file,
|
| 190 | bool isdir)
|
| 191 | {
|
| 192 | struct fuse_file *ff = fuse_file_open(fm, nodeid, file->f_flags, isdir);
|
| 193 |
|
| 194 | if (!IS_ERR(ff))
|
| 195 | file->private_data = ff;
|
| 196 |
|
| 197 | return PTR_ERR_OR_ZERO(ff);
|
| 198 | }
|
| 199 | EXPORT_SYMBOL_GPL(fuse_do_open);
|
| 200 |
|
| 201 | static void fuse_link_write_file(struct file *file)
|
| 202 | {
|
| 203 | struct inode *inode = file_inode(file);
|
| 204 | struct fuse_inode *fi = get_fuse_inode(inode);
|
| 205 | struct fuse_file *ff = file->private_data;
|
| 206 | |
| 207 | * file may be written through mmap, so chain it onto the
|
| 208 | * inodes's write_file list
|
| 209 |
|
| 210 | spin_lock(&fi->lock);
|
| 211 | if (list_empty(&ff->write_entry))
|
| 212 | list_add(&ff->write_entry, &fi->write_files);
|
| 213 | spin_unlock(&fi->lock);
|
| 214 | }
|
| 215 |
|
| 216 | int fuse_finish_open(struct inode *inode, struct file *file)
|
| 217 | {
|
| 218 | struct fuse_file *ff = file->private_data;
|
| 219 | struct fuse_conn *fc = get_fuse_conn(inode);
|
| 220 | int err;
|
| 221 |
|
| 222 | err = fuse_file_io_open(file, inode);
|
| 223 | if (err)
|
| 224 | return err;
|
| 225 |
|
| 226 | if (ff->open_flags & FOPEN_STREAM)
|
| 227 | stream_open(inode, file);
|
| 228 | else if (ff->open_flags & FOPEN_NONSEEKABLE)
|
| 229 | nonseekable_open(inode, file);
|
| 230 |
|
| 231 | if ((file->f_mode & FMODE_WRITE) && fc->writeback_cache)
|
| 232 | fuse_link_write_file(file);
|
| 233 |
|
| 234 | return 0;
|
| 235 | }
|
| 236 |
|
| 237 | static void fuse_truncate_update_attr(struct inode *inode, struct file *file)
|
| 238 | {
|
| 239 | struct fuse_conn *fc = get_fuse_conn(inode);
|
| 240 | struct fuse_inode *fi = get_fuse_inode(inode);
|
| 241 |
|
| 242 | spin_lock(&fi->lock);
|
| 243 | fi->attr_version = atomic64_inc_return(&fc->attr_version);
|
| 244 | i_size_write(inode, 0);
|
| 245 | spin_unlock(&fi->lock);
|
| 246 | file_update_time(file);
|
| 247 | fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE);
|
| 248 | }
|
| 249 |
|
| 250 | static int fuse_open(struct inode *inode, struct file *file)
|
| 251 | {
|
| 252 | struct fuse_mount *fm = get_fuse_mount(inode);
|
| 253 | struct fuse_inode *fi = get_fuse_inode(inode);
|
| 254 | struct fuse_conn *fc = fm->fc;
|
| 255 | struct fuse_file *ff;
|
| 256 | int err;
|
| 257 | bool is_truncate = (file->f_flags & O_TRUNC) && fc->atomic_o_trunc;
|
| 258 | bool is_wb_truncate = is_truncate && fc->writeback_cache;
|
| 259 | bool dax_truncate = is_truncate && FUSE_IS_DAX(inode);
|
| 260 |
|
| 261 | if (fuse_is_bad(inode))
|
| 262 | return -EIO;
|
| 263 |
|
| 264 | err = generic_file_open(inode, file);
|
| 265 | if (err)
|
| 266 | return err;
|
| 267 |
|
| 268 | if (is_wb_truncate || dax_truncate)
|
| 269 | inode_lock(inode);
|
| 270 |
|
| 271 | if (dax_truncate) {
|
| 272 | filemap_invalidate_lock(inode->i_mapping);
|
| 273 | err = fuse_dax_break_layouts(inode, 0, -1);
|
| 274 | if (err)
|
| 275 | goto out_unlock;
|
| 276 | }
|
| 277 |
|
| 278 | if (is_wb_truncate || dax_truncate)
|
| 279 | fuse_set_nowrite(inode);
|
| 280 |
|
| 281 | err = fuse_do_open(fm, get_node_id(inode), file, false);
|
| 282 | if (!err) {
|
| 283 | ff = file->private_data;
|
| 284 | err = fuse_finish_open(inode, file);
|
| 285 | if (err)
|
| 286 | fuse_sync_release(fi, ff, file->f_flags);
|
| 287 | else if (is_truncate)
|
| 288 | fuse_truncate_update_attr(inode, file);
|
| 289 | }
|
| 290 |
|
| 291 | if (is_wb_truncate || dax_truncate)
|
| 292 | fuse_release_nowrite(inode);
|
| 293 | if (!err) {
|
| 294 | if (is_truncate)
|
| 295 | truncate_pagecache(inode, 0);
|
| 296 | else if (!(ff->open_flags & FOPEN_KEEP_CACHE))
|
| 297 | invalidate_inode_pages2(inode->i_mapping);
|
| 298 | }
|
| 299 | out_unlock:
|
| 300 | if (dax_truncate)
|
| 301 | filemap_invalidate_unlock(inode->i_mapping);
|
| 302 | if (is_wb_truncate || dax_truncate)
|
| 303 | inode_unlock(inode);
|
| 304 |
|
| 305 | return err;
|
| 306 | }
|
| 307 |
|
| 308 | static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff,
|
| 309 | unsigned int flags, int opcode, bool sync)
|
| 310 | {
|
| 311 | struct fuse_conn *fc = ff->fm->fc;
|
| 312 | struct fuse_release_args *ra = &ff->args->release_args;
|
| 313 |
|
| 314 | if (fuse_file_passthrough(ff))
|
| 315 | fuse_passthrough_release(ff, fuse_inode_backing(fi));
|
| 316 |
|
| 317 |
|
| 318 | if (likely(fi)) {
|
| 319 | spin_lock(&fi->lock);
|
| 320 | list_del(&ff->write_entry);
|
| 321 | spin_unlock(&fi->lock);
|
| 322 | }
|
| 323 | spin_lock(&fc->lock);
|
| 324 | if (!RB_EMPTY_NODE(&ff->polled_node))
|
| 325 | rb_erase(&ff->polled_node, &fc->polled_files);
|
| 326 | spin_unlock(&fc->lock);
|
| 327 |
|
| 328 | wake_up_interruptible_all(&ff->poll_wait);
|
| 329 |
|
| 330 | if (!ra)
|
| 331 | return;
|
| 332 |
|
| 333 |
|
| 334 | memset(ff->args, 0, sizeof(*ff->args));
|
| 335 | ra->inarg.fh = ff->fh;
|
| 336 | ra->inarg.flags = flags;
|
| 337 | ra->args.in_numargs = 1;
|
| 338 | ra->args.in_args[0].size = sizeof(struct fuse_release_in);
|
| 339 | ra->args.in_args[0].value = &ra->inarg;
|
| 340 | ra->args.opcode = opcode;
|
| 341 | ra->args.nodeid = ff->nodeid;
|
| 342 | ra->args.force = true;
|
| 343 | ra->args.nocreds = true;
|
| 344 |
|
| 345 | |
| 346 | * Hold inode until release is finished.
|
| 347 | * From fuse_sync_release() the refcount is 1 and everything's
|
| 348 | * synchronous, so we are fine with not doing igrab() here.
|
| 349 |
|
| 350 | ra->inode = sync ? NULL : igrab(&fi->inode);
|
| 351 | }
|
| 352 |
|
| 353 | void fuse_file_release(struct inode *inode, struct fuse_file *ff,
|
| 354 | unsigned int open_flags, fl_owner_t id, bool isdir)
|
| 355 | {
|
| 356 | struct fuse_inode *fi = get_fuse_inode(inode);
|
| 357 | struct fuse_release_args *ra = &ff->args->release_args;
|
| 358 | int opcode = isdir ? FUSE_RELEASEDIR : FUSE_RELEASE;
|
| 359 |
|
| 360 | fuse_prepare_release(fi, ff, open_flags, opcode, false);
|
| 361 |
|
| 362 | if (ra && ff->flock) {
|
| 363 | ra->inarg.release_flags |= FUSE_RELEASE_FLOCK_UNLOCK;
|
| 364 | ra->inarg.lock_owner = fuse_lock_owner_id(ff->fm->fc, id);
|
| 365 | }
|
| 366 |
|
| 367 | |
| 368 | * Normally this will send the RELEASE request, however if
|
| 369 | * some asynchronous READ or WRITE requests are outstanding,
|
| 370 | * the sending will be delayed.
|
| 371 | *
|
| 372 | * Make the release synchronous if this is a fuseblk mount,
|
| 373 | * synchronous RELEASE is allowed (and desirable) in this case
|
| 374 | * because the server can be trusted not to screw up.
|
| 375 | *
|
| 376 | * Always use the asynchronous file put because the current thread
|
| 377 | * might be the fuse server. This can happen if a process starts some
|
| 378 | * aio and closes the fd before the aio completes. Since aio takes its
|
| 379 | * own ref to the file, the IO completion has to drop the ref, which is
|
| 380 | * how the fuse server can end up closing its clients' files.
|
| 381 | *
|
| 382 | * Exception is virtio-fs, which is not affected by the above (server is
|
| 383 | * on host, cannot close open files in guest). Virtio-fs needs sync
|
| 384 | * release, because the num_waiting mechanism to wait for all requests
|
| 385 | * before commencing with fs shutdown doesn't work if submounts are
|
| 386 | * used.
|
| 387 |
|
| 388 | fuse_file_put(ff, ff->fm->fc->auto_submounts);
|
| 389 | }
|
| 390 |
|
| 391 | void fuse_release_common(struct file *file, bool isdir)
|
| 392 | {
|
| 393 | fuse_file_release(file_inode(file), file->private_data, file->f_flags,
|
| 394 | (fl_owner_t) file, isdir);
|
| 395 | }
|
| 396 |
|
| 397 | static int fuse_release(struct inode *inode, struct file *file)
|
| 398 | {
|
| 399 | struct fuse_conn *fc = get_fuse_conn(inode);
|
| 400 |
|
| 401 | |
| 402 | * Dirty pages might remain despite write_inode_now() call from
|
| 403 | * fuse_flush() due to writes racing with the close.
|
| 404 |
|
| 405 | if (fc->writeback_cache)
|
| 406 | write_inode_now(inode, 1);
|
| 407 |
|
| 408 | fuse_release_common(file, false);
|
| 409 |
|
| 410 |
|
| 411 | return 0;
|
| 412 | }
|
| 413 |
|
| 414 | void fuse_sync_release(struct fuse_inode *fi, struct fuse_file *ff,
|
| 415 | unsigned int flags)
|
| 416 | {
|
| 417 | WARN_ON(refcount_read(&ff->count) > 1);
|
| 418 | fuse_prepare_release(fi, ff, flags, FUSE_RELEASE, true);
|
| 419 | fuse_file_put(ff, true);
|
| 420 | }
|
| 421 | EXPORT_SYMBOL_GPL(fuse_sync_release);
|
| 422 |
|
| 423 | |
| 424 | * Scramble the ID space with XTEA, so that the value of the files_struct
|
| 425 | * pointer is not exposed to userspace.
|
| 426 |
|
| 427 | u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id)
|
| 428 | {
|
| 429 | u32 *k = fc->scramble_key;
|
| 430 | u64 v = (unsigned long) id;
|
| 431 | u32 v0 = v;
|
| 432 | u32 v1 = v >> 32;
|
| 433 | u32 sum = 0;
|
| 434 | int i;
|
| 435 |
|
| 436 | for (i = 0; i < 32; i++) {
|
| 437 | v0 += ((v1 << 4 ^ v1 >> 5) + v1) ^ (sum + k[sum & 3]);
|
| 438 | sum += 0x9E3779B9;
|
| 439 | v1 += ((v0 << 4 ^ v0 >> 5) + v0) ^ (sum + k[sum>>11 & 3]);
|
| 440 | }
|
| 441 |
|
| 442 | return (u64) v0 + ((u64) v1 << 32);
|
| 443 | }
|
| 444 |
|
| 445 | struct fuse_writepage_args {
|
| 446 | struct fuse_io_args ia;
|
| 447 | struct list_head queue_entry;
|
| 448 | struct inode *inode;
|
| 449 | struct fuse_sync_bucket *bucket;
|
| 450 | };
|
| 451 |
|
| 452 | |
| 453 | * Wait for all pending writepages on the inode to finish.
|
| 454 | *
|
| 455 | * This is currently done by blocking further writes with FUSE_NOWRITE
|
| 456 | * and waiting for all sent writes to complete.
|
| 457 | *
|
| 458 | * This must be called under i_mutex, otherwise the FUSE_NOWRITE usage
|
| 459 | * could conflict with truncation.
|
| 460 |
|
| 461 | static void fuse_sync_writes(struct inode *inode)
|
| 462 | {
|
| 463 | fuse_set_nowrite(inode);
|
| 464 | fuse_release_nowrite(inode);
|
| 465 | }
|
| 466 |
|
| 467 | static int fuse_flush(struct file *file, fl_owner_t id)
|
| 468 | {
|
| 469 | struct inode *inode = file_inode(file);
|
| 470 | struct fuse_mount *fm = get_fuse_mount(inode);
|
| 471 | struct fuse_file *ff = file->private_data;
|
| 472 | struct fuse_flush_in inarg;
|
| 473 | FUSE_ARGS(args);
|
| 474 | int err;
|
| 475 |
|
| 476 | if (fuse_is_bad(inode))
|
| 477 | return -EIO;
|
| 478 |
|
| 479 | if (ff->open_flags & FOPEN_NOFLUSH && !fm->fc->writeback_cache)
|
| 480 | return 0;
|
| 481 |
|
| 482 | err = write_inode_now(inode, 1);
|
| 483 | if (err)
|
| 484 | return err;
|
| 485 |
|
| 486 | err = filemap_check_errors(file->f_mapping);
|
| 487 | if (err)
|
| 488 | return err;
|
| 489 |
|
| 490 | err = 0;
|
| 491 | if (fm->fc->no_flush)
|
| 492 | goto inval_attr_out;
|
| 493 |
|
| 494 | memset(&inarg, 0, sizeof(inarg));
|
| 495 | inarg.fh = ff->fh;
|
| 496 | inarg.lock_owner = fuse_lock_owner_id(fm->fc, id);
|
| 497 | args.opcode = FUSE_FLUSH;
|
| 498 | args.nodeid = get_node_id(inode);
|
| 499 | args.in_numargs = 1;
|
| 500 | args.in_args[0].size = sizeof(inarg);
|
| 501 | args.in_args[0].value = &inarg;
|
| 502 | args.force = true;
|
| 503 |
|
| 504 | err = fuse_simple_request(fm, &args);
|
| 505 | if (err == -ENOSYS) {
|
| 506 | fm->fc->no_flush = 1;
|
| 507 | err = 0;
|
| 508 | }
|
| 509 |
|
| 510 | inval_attr_out:
|
| 511 | |
| 512 | * In memory i_blocks is not maintained by fuse, if writeback cache is
|
| 513 | * enabled, i_blocks from cached attr may not be accurate.
|
| 514 |
|
| 515 | if (!err && fm->fc->writeback_cache)
|
| 516 | fuse_invalidate_attr_mask(inode, STATX_BLOCKS);
|
| 517 | return err;
|
| 518 | }
|
| 519 |
|
| 520 | int fuse_fsync_common(struct file *file, loff_t start, loff_t end,
|
| 521 | int datasync, int opcode)
|
| 522 | {
|
| 523 | struct inode *inode = file->f_mapping->host;
|
| 524 | struct fuse_mount *fm = get_fuse_mount(inode);
|
| 525 | struct fuse_file *ff = file->private_data;
|
| 526 | FUSE_ARGS(args);
|
| 527 | struct fuse_fsync_in inarg;
|
| 528 |
|
| 529 | memset(&inarg, 0, sizeof(inarg));
|
| 530 | inarg.fh = ff->fh;
|
| 531 | inarg.fsync_flags = datasync ? FUSE_FSYNC_FDATASYNC : 0;
|
| 532 | args.opcode = opcode;
|
| 533 | args.nodeid = get_node_id(inode);
|
| 534 | args.in_numargs = 1;
|
| 535 | args.in_args[0].size = sizeof(inarg);
|
| 536 | args.in_args[0].value = &inarg;
|
| 537 | return fuse_simple_request(fm, &args);
|
| 538 | }
|
| 539 |
|
| 540 | static int fuse_fsync(struct file *file, loff_t start, loff_t end,
|
| 541 | int datasync)
|
| 542 | {
|
| 543 | struct inode *inode = file->f_mapping->host;
|
| 544 | struct fuse_conn *fc = get_fuse_conn(inode);
|
| 545 | int err;
|
| 546 |
|
| 547 | if (fuse_is_bad(inode))
|
| 548 | return -EIO;
|
| 549 |
|
| 550 | inode_lock(inode);
|
| 551 |
|
| 552 | |
| 553 | * Start writeback against all dirty pages of the inode, then
|
| 554 | * wait for all outstanding writes, before sending the FSYNC
|
| 555 | * request.
|
| 556 |
|
| 557 | err = file_write_and_wait_range(file, start, end);
|
| 558 | if (err)
|
| 559 | goto out;
|
| 560 |
|
| 561 | fuse_sync_writes(inode);
|
| 562 |
|
| 563 | |
| 564 | * Due to implementation of fuse writeback
|
| 565 | * file_write_and_wait_range() does not catch errors.
|
| 566 | * We have to do this directly after fuse_sync_writes()
|
| 567 |
|
| 568 | err = file_check_and_advance_wb_err(file);
|
| 569 | if (err)
|
| 570 | goto out;
|
| 571 |
|
| 572 | err = sync_inode_metadata(inode, 1);
|
| 573 | if (err)
|
| 574 | goto out;
|
| 575 |
|
| 576 | if (fc->no_fsync)
|
| 577 | goto out;
|
| 578 |
|
| 579 | err = fuse_fsync_common(file, start, end, datasync, FUSE_FSYNC);
|
| 580 | if (err == -ENOSYS) {
|
| 581 | fc->no_fsync = 1;
|
| 582 | err = 0;
|
| 583 | }
|
| 584 | out:
|
| 585 | inode_unlock(inode);
|
| 586 |
|
| 587 | return err;
|
| 588 | }
|
| 589 |
|
| 590 | void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos,
|
| 591 | size_t count, int opcode)
|
| 592 | {
|
| 593 | struct fuse_file *ff = file->private_data;
|
| 594 | struct fuse_args *args = &ia->ap.args;
|
| 595 |
|
| 596 | ia->read.in.fh = ff->fh;
|
| 597 | ia->read.in.offset = pos;
|
| 598 | ia->read.in.size = count;
|
| 599 | ia->read.in.flags = file->f_flags;
|
| 600 | args->opcode = opcode;
|
| 601 | args->nodeid = ff->nodeid;
|
| 602 | args->in_numargs = 1;
|
| 603 | args->in_args[0].size = sizeof(ia->read.in);
|
| 604 | args->in_args[0].value = &ia->read.in;
|
| 605 | args->out_argvar = true;
|
| 606 | args->out_numargs = 1;
|
| 607 | args->out_args[0].size = count;
|
| 608 | args->zero_copy = ff->open_flags & FOPEN_IO_URING_ZERO_COPY;
|
| 609 | }
|
| 610 |
|
| 611 | static void fuse_release_user_pages(struct fuse_args_pages *ap, ssize_t nres,
|
| 612 | bool should_dirty)
|
| 613 | {
|
| 614 | unsigned int i;
|
| 615 |
|
| 616 | for (i = 0; i < ap->num_folios; i++) {
|
| 617 | if (should_dirty)
|
| 618 | folio_mark_dirty_lock(ap->folios[i]);
|
| 619 | if (ap->args.is_pinned)
|
| 620 | unpin_folio(ap->folios[i]);
|
| 621 | }
|
| 622 |
|
| 623 | if (nres > 0 && ap->args.invalidate_vmap)
|
| 624 | invalidate_kernel_vmap_range(ap->args.vmap_base, nres);
|
| 625 | }
|
| 626 |
|
| 627 | static void fuse_io_release(struct kref *kref)
|
| 628 | {
|
| 629 | kfree(container_of(kref, struct fuse_io_priv, refcnt));
|
| 630 | }
|
| 631 |
|
| 632 | static ssize_t fuse_get_res_by_io(struct fuse_io_priv *io)
|
| 633 | {
|
| 634 | if (io->err)
|
| 635 | return io->err;
|
| 636 |
|
| 637 | if (io->bytes >= 0 && io->write)
|
| 638 | return -EIO;
|
| 639 |
|
| 640 | return io->bytes < 0 ? io->size : io->bytes;
|
| 641 | }
|
| 642 |
|
| 643 | static void fuse_aio_invalidate_worker(struct work_struct *work)
|
| 644 | {
|
| 645 | struct fuse_io_priv *io = container_of(work, struct fuse_io_priv, work);
|
| 646 | struct address_space *mapping = io->iocb->ki_filp->f_mapping;
|
| 647 | ssize_t res = fuse_get_res_by_io(io);
|
| 648 | pgoff_t start = io->offset >> PAGE_SHIFT;
|
| 649 | pgoff_t end = (io->offset + res - 1) >> PAGE_SHIFT;
|
| 650 |
|
| 651 | invalidate_inode_pages2_range(mapping, start, end);
|
| 652 | io->iocb->ki_complete(io->iocb, res);
|
| 653 | kref_put(&io->refcnt, fuse_io_release);
|
| 654 | }
|
| 655 |
|
| 656 | |
| 657 | * In case of short read, the caller sets 'pos' to the position of
|
| 658 | * actual end of fuse request in IO request. Otherwise, if bytes_requested
|
| 659 | * == bytes_transferred or rw == WRITE, the caller sets 'pos' to -1.
|
| 660 | *
|
| 661 | * An example:
|
| 662 | * User requested DIO read of 64K. It was split into two 32K fuse requests,
|
| 663 | * both submitted asynchronously. The first of them was ACKed by userspace as
|
| 664 | * fully completed (req->out.args[0].size == 32K) resulting in pos == -1. The
|
| 665 | * second request was ACKed as short, e.g. only 1K was read, resulting in
|
| 666 | * pos == 33K.
|
| 667 | *
|
| 668 | * Thus, when all fuse requests are completed, the minimal non-negative 'pos'
|
| 669 | * will be equal to the length of the longest contiguous fragment of
|
| 670 | * transferred data starting from the beginning of IO request.
|
| 671 |
|
| 672 | static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos)
|
| 673 | {
|
| 674 | int left;
|
| 675 |
|
| 676 | spin_lock(&io->lock);
|
| 677 | if (err)
|
| 678 | io->err = io->err ? : err;
|
| 679 | else if (pos >= 0 && (io->bytes < 0 || pos < io->bytes))
|
| 680 | io->bytes = pos;
|
| 681 |
|
| 682 | left = --io->reqs;
|
| 683 | if (!left && io->blocking)
|
| 684 | complete(io->done);
|
| 685 | spin_unlock(&io->lock);
|
| 686 |
|
| 687 | if (!left && !io->blocking) {
|
| 688 | struct inode *inode = file_inode(io->iocb->ki_filp);
|
| 689 | struct address_space *mapping = io->iocb->ki_filp->f_mapping;
|
| 690 | ssize_t res = fuse_get_res_by_io(io);
|
| 691 |
|
| 692 | if (res >= 0) {
|
| 693 | struct fuse_conn *fc = get_fuse_conn(inode);
|
| 694 | struct fuse_inode *fi = get_fuse_inode(inode);
|
| 695 |
|
| 696 | spin_lock(&fi->lock);
|
| 697 | fi->attr_version = atomic64_inc_return(&fc->attr_version);
|
| 698 | spin_unlock(&fi->lock);
|
| 699 | }
|
| 700 |
|
| 701 | if (io->write && res > 0 && mapping->nrpages) {
|
| 702 | |
| 703 | * As in generic_file_direct_write(), invalidate after the
|
| 704 | * write, to invalidate read-ahead cache that may have competed
|
| 705 | * with the write.
|
| 706 |
|
| 707 | INIT_WORK(&io->work, fuse_aio_invalidate_worker);
|
| 708 | queue_work(inode->i_sb->s_dio_done_wq, &io->work);
|
| 709 | return;
|
| 710 | }
|
| 711 |
|
| 712 | io->iocb->ki_complete(io->iocb, res);
|
| 713 | }
|
| 714 |
|
| 715 | kref_put(&io->refcnt, fuse_io_release);
|
| 716 | }
|
| 717 |
|
| 718 | static struct fuse_io_args *fuse_io_alloc(struct fuse_io_priv *io,
|
| 719 | unsigned int nfolios)
|
| 720 | {
|
| 721 | struct fuse_io_args *ia;
|
| 722 |
|
| 723 | ia = kzalloc_obj(*ia);
|
| 724 | if (ia) {
|
| 725 | ia->io = io;
|
| 726 | ia->ap.folios = fuse_folios_alloc(nfolios, GFP_KERNEL,
|
| 727 | &ia->ap.descs);
|
| 728 | if (!ia->ap.folios) {
|
| 729 | kfree(ia);
|
| 730 | ia = NULL;
|
| 731 | }
|
| 732 | }
|
| 733 | return ia;
|
| 734 | }
|
| 735 |
|
| 736 | static void fuse_io_free(struct fuse_io_args *ia)
|
| 737 | {
|
| 738 | kfree(ia->ap.folios);
|
| 739 | kfree(ia);
|
| 740 | }
|
| 741 |
|
| 742 | static void fuse_aio_complete_req(struct fuse_args *args, int err)
|
| 743 | {
|
| 744 | struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
|
| 745 | struct fuse_io_priv *io = ia->io;
|
| 746 | ssize_t pos = -1;
|
| 747 | size_t nres;
|
| 748 |
|
| 749 | if (err) {
|
| 750 |
|
| 751 | } else if (io->write) {
|
| 752 | if (ia->write.out.size > ia->write.in.size) {
|
| 753 | err = -EIO;
|
| 754 | } else {
|
| 755 | nres = ia->write.out.size;
|
| 756 | if (ia->write.in.size != ia->write.out.size)
|
| 757 | pos = ia->write.in.offset - io->offset +
|
| 758 | ia->write.out.size;
|
| 759 | }
|
| 760 | } else {
|
| 761 | u32 outsize = args->out_args[0].size;
|
| 762 |
|
| 763 | nres = outsize;
|
| 764 | if (ia->read.in.size != outsize)
|
| 765 | pos = ia->read.in.offset - io->offset + outsize;
|
| 766 | }
|
| 767 |
|
| 768 | fuse_release_user_pages(&ia->ap, err ?: nres, io->should_dirty);
|
| 769 |
|
| 770 | fuse_aio_complete(io, err, pos);
|
| 771 | fuse_io_free(ia);
|
| 772 | }
|
| 773 |
|
| 774 | static ssize_t fuse_async_req_send(struct fuse_mount *fm,
|
| 775 | struct fuse_io_args *ia, size_t num_bytes)
|
| 776 | {
|
| 777 | ssize_t err;
|
| 778 | struct fuse_io_priv *io = ia->io;
|
| 779 |
|
| 780 | spin_lock(&io->lock);
|
| 781 | kref_get(&io->refcnt);
|
| 782 | io->size += num_bytes;
|
| 783 | io->reqs++;
|
| 784 | spin_unlock(&io->lock);
|
| 785 |
|
| 786 | ia->ap.args.end = fuse_aio_complete_req;
|
| 787 | ia->ap.args.may_block = io->should_dirty;
|
| 788 | err = fuse_simple_background(fm, &ia->ap.args, GFP_KERNEL);
|
| 789 | if (err)
|
| 790 | fuse_aio_complete_req(&ia->ap.args, err);
|
| 791 |
|
| 792 | return num_bytes;
|
| 793 | }
|
| 794 |
|
| 795 | static ssize_t fuse_send_read(struct fuse_io_args *ia, loff_t pos, size_t count,
|
| 796 | fl_owner_t owner)
|
| 797 | {
|
| 798 | struct file *file = ia->io->iocb->ki_filp;
|
| 799 | struct fuse_file *ff = file->private_data;
|
| 800 | struct fuse_mount *fm = ff->fm;
|
| 801 |
|
| 802 | fuse_read_args_fill(ia, file, pos, count, FUSE_READ);
|
| 803 | if (owner != NULL) {
|
| 804 | ia->read.in.read_flags |= FUSE_READ_LOCKOWNER;
|
| 805 | ia->read.in.lock_owner = fuse_lock_owner_id(fm->fc, owner);
|
| 806 | }
|
| 807 |
|
| 808 | if (ia->io->async)
|
| 809 | return fuse_async_req_send(fm, ia, count);
|
| 810 |
|
| 811 | return fuse_simple_request(fm, &ia->ap.args);
|
| 812 | }
|
| 813 |
|
| 814 | static void fuse_read_update_size(struct inode *inode, loff_t size,
|
| 815 | u64 attr_ver)
|
| 816 | {
|
| 817 | struct fuse_conn *fc = get_fuse_conn(inode);
|
| 818 | struct fuse_inode *fi = get_fuse_inode(inode);
|
| 819 |
|
| 820 | spin_lock(&fi->lock);
|
| 821 | if (attr_ver >= fi->attr_version && size < inode->i_size &&
|
| 822 | !test_bit(FUSE_I_SIZE_UNSTABLE, &fi->state)) {
|
| 823 | fi->attr_version = atomic64_inc_return(&fc->attr_version);
|
| 824 | i_size_write(inode, size);
|
| 825 | }
|
| 826 | spin_unlock(&fi->lock);
|
| 827 | }
|
| 828 |
|
| 829 | static void fuse_short_read(struct inode *inode, u64 attr_ver, size_t num_read,
|
| 830 | struct fuse_args_pages *ap)
|
| 831 | {
|
| 832 | struct fuse_conn *fc = get_fuse_conn(inode);
|
| 833 |
|
| 834 | |
| 835 | * If writeback_cache is enabled, a short read means there's a hole in
|
| 836 | * the file. Some data after the hole is in page cache, but has not
|
| 837 | * reached the client fs yet. So the hole is not present there.
|
| 838 |
|
| 839 | if (!fc->writeback_cache) {
|
| 840 | loff_t pos = folio_pos(ap->folios[0]) + num_read;
|
| 841 | fuse_read_update_size(inode, pos, attr_ver);
|
| 842 | }
|
| 843 | }
|
| 844 |
|
| 845 | static int fuse_do_readfolio(struct file *file, struct folio *folio,
|
| 846 | size_t off, size_t len)
|
| 847 | {
|
| 848 | struct inode *inode = folio->mapping->host;
|
| 849 | struct fuse_mount *fm = get_fuse_mount(inode);
|
| 850 | loff_t pos = folio_pos(folio) + off;
|
| 851 | struct fuse_folio_desc desc = {
|
| 852 | .offset = off,
|
| 853 | .length = len,
|
| 854 | };
|
| 855 | struct fuse_io_args ia = {
|
| 856 | .ap.args.page_zeroing = true,
|
| 857 | .ap.args.out_pages = true,
|
| 858 | .ap.num_folios = 1,
|
| 859 | .ap.folios = &folio,
|
| 860 | .ap.descs = &desc,
|
| 861 | };
|
| 862 | ssize_t res;
|
| 863 | u64 attr_ver;
|
| 864 |
|
| 865 | attr_ver = fuse_get_attr_version(fm->fc);
|
| 866 |
|
| 867 |
|
| 868 | if (pos + (desc.length - 1) == LLONG_MAX)
|
| 869 | desc.length--;
|
| 870 |
|
| 871 | fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ);
|
| 872 | res = fuse_simple_request(fm, &ia.ap.args);
|
| 873 | if (res < 0)
|
| 874 | return res;
|
| 875 | |
| 876 | * Short read means EOF. If file size is larger, truncate it
|
| 877 |
|
| 878 | if (res < desc.length)
|
| 879 | fuse_short_read(inode, attr_ver, res, &ia.ap);
|
| 880 |
|
| 881 | return 0;
|
| 882 | }
|
| 883 |
|
| 884 | static int fuse_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
|
| 885 | unsigned int flags, struct iomap *iomap,
|
| 886 | struct iomap *srcmap)
|
| 887 | {
|
| 888 | iomap->type = IOMAP_MAPPED;
|
| 889 | iomap->length = length;
|
| 890 | iomap->offset = offset;
|
| 891 | return 0;
|
| 892 | }
|
| 893 |
|
| 894 | static DEFINE_IOMAP_ITER_NEXT(fuse_iomap_next, fuse_iomap_begin);
|
| 895 |
|
| 896 | static const struct iomap_ops fuse_iomap_ops = {
|
| 897 | .iomap_next = fuse_iomap_next,
|
| 898 | };
|
| 899 |
|
| 900 | struct fuse_fill_read_data {
|
| 901 | struct file *file;
|
| 902 |
|
| 903 |
|
| 904 | struct fuse_conn *fc;
|
| 905 | struct fuse_io_args *ia;
|
| 906 | unsigned int nr_bytes;
|
| 907 | };
|
| 908 |
|
| 909 |
|
| 910 | static bool fuse_folios_need_send(struct fuse_conn *fc, loff_t pos,
|
| 911 | unsigned len, struct fuse_args_pages *ap,
|
| 912 | unsigned cur_bytes, bool write);
|
| 913 | static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file,
|
| 914 | unsigned int count, bool async);
|
| 915 |
|
| 916 | static int fuse_handle_readahead(struct folio *folio,
|
| 917 | struct readahead_control *rac,
|
| 918 | struct fuse_fill_read_data *data, loff_t pos,
|
| 919 | size_t len)
|
| 920 | {
|
| 921 | struct fuse_io_args *ia = data->ia;
|
| 922 | size_t off = offset_in_folio(folio, pos);
|
| 923 | struct fuse_conn *fc = data->fc;
|
| 924 | struct fuse_args_pages *ap;
|
| 925 | unsigned int nr_pages;
|
| 926 |
|
| 927 | if (ia && fuse_folios_need_send(fc, pos, len, &ia->ap, data->nr_bytes,
|
| 928 | false)) {
|
| 929 | fuse_send_readpages(ia, data->file, data->nr_bytes,
|
| 930 | fc->async_read);
|
| 931 | data->nr_bytes = 0;
|
| 932 | data->ia = NULL;
|
| 933 | ia = NULL;
|
| 934 | }
|
| 935 | if (!ia) {
|
| 936 | if (fuse_chan_num_background(fc->chan) >= fc->congestion_threshold &&
|
| 937 | rac->ra->async_size >= readahead_count(rac))
|
| 938 | |
| 939 | * Congested and only async pages left, so skip the
|
| 940 | * rest.
|
| 941 |
|
| 942 | return -EAGAIN;
|
| 943 |
|
| 944 | nr_pages = min(fc->max_pages, readahead_count(rac));
|
| 945 | data->ia = fuse_io_alloc(NULL, nr_pages);
|
| 946 | if (!data->ia)
|
| 947 | return -ENOMEM;
|
| 948 | ia = data->ia;
|
| 949 | }
|
| 950 | folio_get(folio);
|
| 951 | ap = &ia->ap;
|
| 952 | ap->folios[ap->num_folios] = folio;
|
| 953 | ap->descs[ap->num_folios].offset = off;
|
| 954 | ap->descs[ap->num_folios].length = len;
|
| 955 | data->nr_bytes += len;
|
| 956 | ap->num_folios++;
|
| 957 |
|
| 958 | return 0;
|
| 959 | }
|
| 960 |
|
| 961 | static int fuse_iomap_read_folio_range_async(const struct iomap_iter *iter,
|
| 962 | struct iomap_read_folio_ctx *ctx,
|
| 963 | size_t len)
|
| 964 | {
|
| 965 | struct fuse_fill_read_data *data = ctx->read_ctx;
|
| 966 | struct folio *folio = ctx->cur_folio;
|
| 967 | loff_t pos = iter->pos;
|
| 968 | size_t off = offset_in_folio(folio, pos);
|
| 969 | struct file *file = data->file;
|
| 970 | int ret;
|
| 971 |
|
| 972 | if (ctx->rac) {
|
| 973 | ret = fuse_handle_readahead(folio, ctx->rac, data, pos, len);
|
| 974 | } else {
|
| 975 | |
| 976 | * for non-readahead read requests, do reads synchronously
|
| 977 | * since it's not guaranteed that the server can handle
|
| 978 | * out-of-order reads
|
| 979 |
|
| 980 | ret = fuse_do_readfolio(file, folio, off, len);
|
| 981 | if (!ret)
|
| 982 | iomap_finish_folio_read(folio, off, len, ret);
|
| 983 | }
|
| 984 | return ret;
|
| 985 | }
|
| 986 |
|
| 987 | static const struct iomap_read_ops fuse_iomap_read_ops = {
|
| 988 | .read_folio_range = fuse_iomap_read_folio_range_async,
|
| 989 | };
|
| 990 |
|
| 991 | static int fuse_read_folio(struct file *file, struct folio *folio)
|
| 992 | {
|
| 993 | struct inode *inode = folio->mapping->host;
|
| 994 | struct fuse_fill_read_data data = {
|
| 995 | .file = file,
|
| 996 | };
|
| 997 | struct iomap_read_folio_ctx ctx = {
|
| 998 | .cur_folio = folio,
|
| 999 | .ops = &fuse_iomap_read_ops,
|
| 1000 | .read_ctx = &data,
|
| 1001 |
|
| 1002 | };
|
| 1003 |
|
| 1004 | if (fuse_is_bad(inode)) {
|
| 1005 | folio_unlock(folio);
|
| 1006 | return -EIO;
|
| 1007 | }
|
| 1008 |
|
| 1009 | iomap_read_folio(&fuse_iomap_ops, &ctx, NULL);
|
| 1010 | fuse_invalidate_atime(inode);
|
| 1011 | return 0;
|
| 1012 | }
|
| 1013 |
|
| 1014 | static int fuse_iomap_read_folio_range(const struct iomap_iter *iter,
|
| 1015 | struct folio *folio, loff_t pos,
|
| 1016 | size_t len)
|
| 1017 | {
|
| 1018 | struct file *file = iter->private;
|
| 1019 | size_t off = offset_in_folio(folio, pos);
|
| 1020 |
|
| 1021 | return fuse_do_readfolio(file, folio, off, len);
|
| 1022 | }
|
| 1023 |
|
| 1024 | static void fuse_readpages_end(struct fuse_args *args, int err)
|
| 1025 | {
|
| 1026 | int i;
|
| 1027 | struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
|
| 1028 | struct fuse_args_pages *ap = &ia->ap;
|
| 1029 | size_t count = ia->read.in.size;
|
| 1030 | size_t num_read = args->out_args[0].size;
|
| 1031 | struct address_space *mapping;
|
| 1032 | struct inode *inode;
|
| 1033 |
|
| 1034 | WARN_ON_ONCE(!ap->num_folios);
|
| 1035 | mapping = ap->folios[0]->mapping;
|
| 1036 | inode = mapping->host;
|
| 1037 |
|
| 1038 | |
| 1039 | * Short read means EOF. If file size is larger, truncate it
|
| 1040 |
|
| 1041 | if (!err && num_read < count)
|
| 1042 | fuse_short_read(inode, ia->read.attr_ver, num_read, ap);
|
| 1043 |
|
| 1044 | fuse_invalidate_atime(inode);
|
| 1045 |
|
| 1046 | for (i = 0; i < ap->num_folios; i++) {
|
| 1047 | iomap_finish_folio_read(ap->folios[i], ap->descs[i].offset,
|
| 1048 | ap->descs[i].length, err);
|
| 1049 | folio_put(ap->folios[i]);
|
| 1050 | }
|
| 1051 | if (ia->ff)
|
| 1052 | fuse_file_put(ia->ff, false);
|
| 1053 |
|
| 1054 | fuse_io_free(ia);
|
| 1055 | }
|
| 1056 |
|
| 1057 | static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file,
|
| 1058 | unsigned int count, bool async)
|
| 1059 | {
|
| 1060 | struct fuse_file *ff = file->private_data;
|
| 1061 | struct fuse_mount *fm = ff->fm;
|
| 1062 | struct fuse_args_pages *ap = &ia->ap;
|
| 1063 | loff_t pos = folio_pos(ap->folios[0]);
|
| 1064 | ssize_t res;
|
| 1065 | int err;
|
| 1066 |
|
| 1067 | ap->args.out_pages = true;
|
| 1068 | ap->args.page_zeroing = true;
|
| 1069 | ap->args.page_replace = true;
|
| 1070 |
|
| 1071 |
|
| 1072 | if (pos + (count - 1) == LLONG_MAX) {
|
| 1073 | count--;
|
| 1074 | ap->descs[ap->num_folios - 1].length--;
|
| 1075 | }
|
| 1076 | WARN_ON((loff_t) (pos + count) < 0);
|
| 1077 |
|
| 1078 | fuse_read_args_fill(ia, file, pos, count, FUSE_READ);
|
| 1079 | ia->read.attr_ver = fuse_get_attr_version(fm->fc);
|
| 1080 | if (async) {
|
| 1081 | ia->ff = fuse_file_get(ff);
|
| 1082 | ap->args.end = fuse_readpages_end;
|
| 1083 | err = fuse_simple_background(fm, &ap->args, GFP_KERNEL);
|
| 1084 | if (!err)
|
| 1085 | return;
|
| 1086 | } else {
|
| 1087 | res = fuse_simple_request(fm, &ap->args);
|
| 1088 | err = res < 0 ? res : 0;
|
| 1089 | }
|
| 1090 | fuse_readpages_end(&ap->args, err);
|
| 1091 | }
|
| 1092 |
|
| 1093 | static void fuse_readahead(struct readahead_control *rac)
|
| 1094 | {
|
| 1095 | struct inode *inode = rac->mapping->host;
|
| 1096 | struct fuse_conn *fc = get_fuse_conn(inode);
|
| 1097 | struct fuse_fill_read_data data = {
|
| 1098 | .file = rac->file,
|
| 1099 | .fc = fc,
|
| 1100 | };
|
| 1101 | struct iomap_read_folio_ctx ctx = {
|
| 1102 | .ops = &fuse_iomap_read_ops,
|
| 1103 | .rac = rac,
|
| 1104 | .read_ctx = &data
|
| 1105 | };
|
| 1106 |
|
| 1107 | if (fuse_is_bad(inode))
|
| 1108 | return;
|
| 1109 |
|
| 1110 | iomap_readahead(&fuse_iomap_ops, &ctx, NULL);
|
| 1111 | if (data.ia)
|
| 1112 | fuse_send_readpages(data.ia, data.file, data.nr_bytes,
|
| 1113 | fc->async_read);
|
| 1114 | }
|
| 1115 |
|
| 1116 | static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to)
|
| 1117 | {
|
| 1118 | struct inode *inode = iocb->ki_filp->f_mapping->host;
|
| 1119 | struct fuse_conn *fc = get_fuse_conn(inode);
|
| 1120 |
|
| 1121 | |
| 1122 | * In auto invalidate mode, always update attributes on read.
|
| 1123 | * Otherwise, only update if we attempt to read past EOF (to ensure
|
| 1124 | * i_size is up to date).
|
| 1125 |
|
| 1126 | if (fc->auto_inval_data ||
|
| 1127 | (iocb->ki_pos + iov_iter_count(to) > i_size_read(inode))) {
|
| 1128 | int err;
|
| 1129 | err = fuse_update_attributes(inode, iocb->ki_filp, STATX_SIZE);
|
| 1130 | if (err)
|
| 1131 | return err;
|
| 1132 | }
|
| 1133 |
|
| 1134 | return generic_file_read_iter(iocb, to);
|
| 1135 | }
|
| 1136 |
|
| 1137 | static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff,
|
| 1138 | loff_t pos, size_t count)
|
| 1139 | {
|
| 1140 | struct fuse_args *args = &ia->ap.args;
|
| 1141 |
|
| 1142 | ia->write.in.fh = ff->fh;
|
| 1143 | ia->write.in.offset = pos;
|
| 1144 | ia->write.in.size = count;
|
| 1145 | args->opcode = FUSE_WRITE;
|
| 1146 | args->nodeid = ff->nodeid;
|
| 1147 | args->in_numargs = 2;
|
| 1148 | if (ff->fm->fc->minor < 9)
|
| 1149 | args->in_args[0].size = FUSE_COMPAT_WRITE_IN_SIZE;
|
| 1150 | else
|
| 1151 | args->in_args[0].size = sizeof(ia->write.in);
|
| 1152 | args->in_args[0].value = &ia->write.in;
|
| 1153 | args->in_args[1].size = count;
|
| 1154 | args->out_numargs = 1;
|
| 1155 | args->out_args[0].size = sizeof(ia->write.out);
|
| 1156 | args->out_args[0].value = &ia->write.out;
|
| 1157 | args->zero_copy = ff->open_flags & FOPEN_IO_URING_ZERO_COPY;
|
| 1158 | }
|
| 1159 |
|
| 1160 | static unsigned int fuse_write_flags(struct kiocb *iocb)
|
| 1161 | {
|
| 1162 | unsigned int flags = iocb->ki_filp->f_flags;
|
| 1163 |
|
| 1164 | if (iocb_is_dsync(iocb))
|
| 1165 | flags |= O_DSYNC;
|
| 1166 | if (iocb->ki_flags & IOCB_SYNC)
|
| 1167 | flags |= O_SYNC;
|
| 1168 |
|
| 1169 | return flags;
|
| 1170 | }
|
| 1171 |
|
| 1172 | static ssize_t fuse_send_write(struct fuse_io_args *ia, loff_t pos,
|
| 1173 | size_t count, fl_owner_t owner)
|
| 1174 | {
|
| 1175 | struct kiocb *iocb = ia->io->iocb;
|
| 1176 | struct file *file = iocb->ki_filp;
|
| 1177 | struct fuse_file *ff = file->private_data;
|
| 1178 | struct fuse_mount *fm = ff->fm;
|
| 1179 | struct fuse_write_in *inarg = &ia->write.in;
|
| 1180 | ssize_t err;
|
| 1181 |
|
| 1182 | fuse_write_args_fill(ia, ff, pos, count);
|
| 1183 | inarg->flags = fuse_write_flags(iocb);
|
| 1184 | if (owner != NULL) {
|
| 1185 | inarg->write_flags |= FUSE_WRITE_LOCKOWNER;
|
| 1186 | inarg->lock_owner = fuse_lock_owner_id(fm->fc, owner);
|
| 1187 | }
|
| 1188 |
|
| 1189 | if (ia->io->async)
|
| 1190 | return fuse_async_req_send(fm, ia, count);
|
| 1191 |
|
| 1192 | err = fuse_simple_request(fm, &ia->ap.args);
|
| 1193 | if (!err && ia->write.out.size > count)
|
| 1194 | err = -EIO;
|
| 1195 |
|
| 1196 | return err ?: ia->write.out.size;
|
| 1197 | }
|
| 1198 |
|
| 1199 | bool fuse_write_update_attr(struct inode *inode, loff_t pos, ssize_t written)
|
| 1200 | {
|
| 1201 | struct fuse_conn *fc = get_fuse_conn(inode);
|
| 1202 | struct fuse_inode *fi = get_fuse_inode(inode);
|
| 1203 | bool ret = false;
|
| 1204 |
|
| 1205 | spin_lock(&fi->lock);
|
| 1206 | fi->attr_version = atomic64_inc_return(&fc->attr_version);
|
| 1207 | if (written > 0 && pos > inode->i_size) {
|
| 1208 | i_size_write(inode, pos);
|
| 1209 | ret = true;
|
| 1210 | }
|
| 1211 | spin_unlock(&fi->lock);
|
| 1212 |
|
| 1213 | fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE);
|
| 1214 |
|
| 1215 | return ret;
|
| 1216 | }
|
| 1217 |
|
| 1218 | static ssize_t fuse_send_write_pages(struct fuse_io_args *ia,
|
| 1219 | struct kiocb *iocb, struct inode *inode,
|
| 1220 | loff_t pos, size_t count)
|
| 1221 | {
|
| 1222 | struct fuse_args_pages *ap = &ia->ap;
|
| 1223 | struct file *file = iocb->ki_filp;
|
| 1224 | struct fuse_file *ff = file->private_data;
|
| 1225 | struct fuse_mount *fm = ff->fm;
|
| 1226 | unsigned int i;
|
| 1227 | int err;
|
| 1228 |
|
| 1229 | for (i = 0; i < ap->num_folios; i++)
|
| 1230 | folio_wait_writeback(ap->folios[i]);
|
| 1231 |
|
| 1232 | fuse_write_args_fill(ia, ff, pos, count);
|
| 1233 | ia->write.in.flags = fuse_write_flags(iocb);
|
| 1234 | if (fm->fc->handle_killpriv_v2 && !capable(CAP_FSETID))
|
| 1235 | ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID;
|
| 1236 |
|
| 1237 | err = fuse_simple_request(fm, &ap->args);
|
| 1238 | if (!err && ia->write.out.size > count)
|
| 1239 | err = -EIO;
|
| 1240 |
|
| 1241 | for (i = 0; i < ap->num_folios; i++) {
|
| 1242 | struct folio *folio = ap->folios[i];
|
| 1243 |
|
| 1244 | if (ia->write.folio_locked && (i == ap->num_folios - 1))
|
| 1245 | folio_unlock(folio);
|
| 1246 | folio_put(folio);
|
| 1247 | }
|
| 1248 |
|
| 1249 | return err;
|
| 1250 | }
|
| 1251 |
|
| 1252 | static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia,
|
| 1253 | struct address_space *mapping,
|
| 1254 | struct iov_iter *ii, loff_t pos,
|
| 1255 | unsigned int max_folios)
|
| 1256 | {
|
| 1257 | struct fuse_args_pages *ap = &ia->ap;
|
| 1258 | struct fuse_conn *fc = get_fuse_conn(mapping->host);
|
| 1259 | size_t count = 0;
|
| 1260 | unsigned int num;
|
| 1261 | int err = 0;
|
| 1262 |
|
| 1263 | num = min(iov_iter_count(ii), fc->max_write);
|
| 1264 |
|
| 1265 | ap->args.in_pages = true;
|
| 1266 |
|
| 1267 | while (num && ap->num_folios < max_folios) {
|
| 1268 | size_t tmp;
|
| 1269 | struct folio *folio;
|
| 1270 | pgoff_t index = pos >> PAGE_SHIFT;
|
| 1271 | unsigned int bytes;
|
| 1272 | unsigned int folio_offset;
|
| 1273 |
|
| 1274 | again:
|
| 1275 | folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN,
|
| 1276 | mapping_gfp_mask(mapping));
|
| 1277 | if (IS_ERR(folio)) {
|
| 1278 | err = PTR_ERR(folio);
|
| 1279 | break;
|
| 1280 | }
|
| 1281 |
|
| 1282 | if (mapping_writably_mapped(mapping))
|
| 1283 | flush_dcache_folio(folio);
|
| 1284 |
|
| 1285 | folio_offset = offset_in_folio(folio, pos);
|
| 1286 | bytes = min(folio_size(folio) - folio_offset, num);
|
| 1287 |
|
| 1288 | tmp = copy_folio_from_iter_atomic(folio, folio_offset, bytes, ii);
|
| 1289 | flush_dcache_folio(folio);
|
| 1290 |
|
| 1291 | if (!tmp) {
|
| 1292 | folio_unlock(folio);
|
| 1293 | folio_put(folio);
|
| 1294 |
|
| 1295 | |
| 1296 | * Ensure forward progress by faulting in
|
| 1297 | * while not holding the folio lock:
|
| 1298 |
|
| 1299 | if (fault_in_iov_iter_readable(ii, bytes)) {
|
| 1300 | err = -EFAULT;
|
| 1301 | break;
|
| 1302 | }
|
| 1303 |
|
| 1304 | goto again;
|
| 1305 | }
|
| 1306 |
|
| 1307 | ap->folios[ap->num_folios] = folio;
|
| 1308 | ap->descs[ap->num_folios].offset = folio_offset;
|
| 1309 | ap->descs[ap->num_folios].length = tmp;
|
| 1310 | ap->num_folios++;
|
| 1311 |
|
| 1312 | count += tmp;
|
| 1313 | pos += tmp;
|
| 1314 | num -= tmp;
|
| 1315 |
|
| 1316 |
|
| 1317 | if (tmp == folio_size(folio))
|
| 1318 | iomap_folio_mark_uptodate(folio);
|
| 1319 |
|
| 1320 | if (folio_test_uptodate(folio)) {
|
| 1321 | folio_unlock(folio);
|
| 1322 | } else {
|
| 1323 | ia->write.folio_locked = true;
|
| 1324 | break;
|
| 1325 | }
|
| 1326 | if (!fc->big_writes)
|
| 1327 | break;
|
| 1328 | if (folio_offset + tmp != folio_size(folio))
|
| 1329 | break;
|
| 1330 | }
|
| 1331 |
|
| 1332 | return count > 0 ? count : err;
|
| 1333 | }
|
| 1334 |
|
| 1335 | static inline unsigned int fuse_wr_pages(loff_t pos, size_t len,
|
| 1336 | unsigned int max_pages)
|
| 1337 | {
|
| 1338 | unsigned int pages = ((pos + len - 1) >> PAGE_SHIFT) -
|
| 1339 | (pos >> PAGE_SHIFT) + 1;
|
| 1340 |
|
| 1341 | return min(pages, max_pages);
|
| 1342 | }
|
| 1343 |
|
| 1344 | static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii)
|
| 1345 | {
|
| 1346 | struct address_space *mapping = iocb->ki_filp->f_mapping;
|
| 1347 | struct inode *inode = mapping->host;
|
| 1348 | struct fuse_conn *fc = get_fuse_conn(inode);
|
| 1349 | struct fuse_inode *fi = get_fuse_inode(inode);
|
| 1350 | loff_t pos = iocb->ki_pos;
|
| 1351 | loff_t old_size = i_size_read(inode);
|
| 1352 | int err = 0;
|
| 1353 | ssize_t res = 0;
|
| 1354 |
|
| 1355 | if (pos > old_size)
|
| 1356 | truncate_pagecache_range(inode, old_size, pos - 1);
|
| 1357 |
|
| 1358 | if (inode->i_size < pos + iov_iter_count(ii))
|
| 1359 | set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
|
| 1360 |
|
| 1361 | do {
|
| 1362 | ssize_t count;
|
| 1363 | struct fuse_io_args ia = {};
|
| 1364 | struct fuse_args_pages *ap = &ia.ap;
|
| 1365 | unsigned int nr_pages = fuse_wr_pages(pos, iov_iter_count(ii),
|
| 1366 | fc->max_pages);
|
| 1367 |
|
| 1368 | ap->folios = fuse_folios_alloc(nr_pages, GFP_KERNEL, &ap->descs);
|
| 1369 | if (!ap->folios) {
|
| 1370 | err = -ENOMEM;
|
| 1371 | break;
|
| 1372 | }
|
| 1373 |
|
| 1374 | count = fuse_fill_write_pages(&ia, mapping, ii, pos, nr_pages);
|
| 1375 | if (count <= 0) {
|
| 1376 | err = count;
|
| 1377 | } else {
|
| 1378 | err = fuse_send_write_pages(&ia, iocb, inode,
|
| 1379 | pos, count);
|
| 1380 | if (!err) {
|
| 1381 | size_t num_written = ia.write.out.size;
|
| 1382 |
|
| 1383 | res += num_written;
|
| 1384 | pos += num_written;
|
| 1385 |
|
| 1386 |
|
| 1387 | if (num_written != count)
|
| 1388 | err = -EIO;
|
| 1389 | }
|
| 1390 | }
|
| 1391 | kfree(ap->folios);
|
| 1392 | } while (!err && iov_iter_count(ii));
|
| 1393 |
|
| 1394 | fuse_write_update_attr(inode, pos, res);
|
| 1395 | clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
|
| 1396 |
|
| 1397 | if (!res)
|
| 1398 | return err;
|
| 1399 | iocb->ki_pos += res;
|
| 1400 | return res;
|
| 1401 | }
|
| 1402 |
|
| 1403 | static bool fuse_io_past_eof(struct kiocb *iocb, struct iov_iter *iter)
|
| 1404 | {
|
| 1405 | struct inode *inode = file_inode(iocb->ki_filp);
|
| 1406 |
|
| 1407 | return iocb->ki_pos + iov_iter_count(iter) > i_size_read(inode);
|
| 1408 | }
|
| 1409 |
|
| 1410 | |
| 1411 | * @return true if an exclusive lock for direct IO writes is needed
|
| 1412 |
|
| 1413 | static bool fuse_dio_wr_exclusive_lock(struct kiocb *iocb, struct iov_iter *from)
|
| 1414 | {
|
| 1415 | struct file *file = iocb->ki_filp;
|
| 1416 | struct fuse_file *ff = file->private_data;
|
| 1417 | struct inode *inode = file_inode(iocb->ki_filp);
|
| 1418 | struct fuse_inode *fi = get_fuse_inode(inode);
|
| 1419 |
|
| 1420 |
|
| 1421 | if (!(ff->open_flags & FOPEN_PARALLEL_DIRECT_WRITES))
|
| 1422 | return true;
|
| 1423 |
|
| 1424 | |
| 1425 | * Append will need to know the eventual EOF - always needs an
|
| 1426 | * exclusive lock.
|
| 1427 |
|
| 1428 | if (iocb->ki_flags & IOCB_APPEND)
|
| 1429 | return true;
|
| 1430 |
|
| 1431 |
|
| 1432 | if (test_bit(FUSE_I_CACHE_IO_MODE, &fi->state))
|
| 1433 | return true;
|
| 1434 |
|
| 1435 |
|
| 1436 | if (fuse_io_past_eof(iocb, from))
|
| 1437 | return true;
|
| 1438 |
|
| 1439 | return false;
|
| 1440 | }
|
| 1441 |
|
| 1442 | static void fuse_dio_lock(struct kiocb *iocb, struct iov_iter *from,
|
| 1443 | bool *exclusive)
|
| 1444 | {
|
| 1445 | struct inode *inode = file_inode(iocb->ki_filp);
|
| 1446 | struct fuse_inode *fi = get_fuse_inode(inode);
|
| 1447 |
|
| 1448 | *exclusive = fuse_dio_wr_exclusive_lock(iocb, from);
|
| 1449 | if (*exclusive) {
|
| 1450 | inode_lock(inode);
|
| 1451 | } else {
|
| 1452 | inode_lock_shared(inode);
|
| 1453 | |
| 1454 | * New parallal dio allowed only if inode is not in caching
|
| 1455 | * mode and denies new opens in caching mode. This check
|
| 1456 | * should be performed only after taking shared inode lock.
|
| 1457 | * Previous past eof check was without inode lock and might
|
| 1458 | * have raced, so check it again.
|
| 1459 |
|
| 1460 | if (fuse_io_past_eof(iocb, from) ||
|
| 1461 | fuse_inode_uncached_io_start(fi, NULL) != 0) {
|
| 1462 | inode_unlock_shared(inode);
|
| 1463 | inode_lock(inode);
|
| 1464 | *exclusive = true;
|
| 1465 | }
|
| 1466 | }
|
| 1467 | }
|
| 1468 |
|
| 1469 | static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive)
|
| 1470 | {
|
| 1471 | struct inode *inode = file_inode(iocb->ki_filp);
|
| 1472 | struct fuse_inode *fi = get_fuse_inode(inode);
|
| 1473 |
|
| 1474 | if (exclusive) {
|
| 1475 | inode_unlock(inode);
|
| 1476 | } else {
|
| 1477 |
|
| 1478 | fuse_inode_uncached_io_end(fi);
|
| 1479 | inode_unlock_shared(inode);
|
| 1480 | }
|
| 1481 | }
|
| 1482 |
|
| 1483 | static const struct iomap_write_ops fuse_iomap_write_ops = {
|
| 1484 | .read_folio_range = fuse_iomap_read_folio_range,
|
| 1485 | };
|
| 1486 |
|
| 1487 | static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
|
| 1488 | {
|
| 1489 | struct file *file = iocb->ki_filp;
|
| 1490 | struct mnt_idmap *idmap = file_mnt_idmap(file);
|
| 1491 | struct address_space *mapping = file->f_mapping;
|
| 1492 | ssize_t written = 0;
|
| 1493 | struct inode *inode = mapping->host;
|
| 1494 | ssize_t err, count;
|
| 1495 | struct fuse_conn *fc = get_fuse_conn(inode);
|
| 1496 | bool writeback = false;
|
| 1497 |
|
| 1498 | if (fc->writeback_cache) {
|
| 1499 |
|
| 1500 | err = fuse_update_attributes(mapping->host, file,
|
| 1501 | STATX_SIZE | STATX_MODE);
|
| 1502 | if (err)
|
| 1503 | return err;
|
| 1504 |
|
| 1505 | if (!fc->handle_killpriv_v2 ||
|
| 1506 | !setattr_should_drop_suidgid(idmap, file_inode(file)))
|
| 1507 | writeback = true;
|
| 1508 | }
|
| 1509 |
|
| 1510 | inode_lock(inode);
|
| 1511 |
|
| 1512 | err = count = generic_write_checks(iocb, from);
|
| 1513 | if (err <= 0)
|
| 1514 | goto out;
|
| 1515 |
|
| 1516 | task_io_account_write(count);
|
| 1517 |
|
| 1518 | err = kiocb_modified(iocb);
|
| 1519 | if (err)
|
| 1520 | goto out;
|
| 1521 |
|
| 1522 | if (iocb->ki_flags & IOCB_DIRECT) {
|
| 1523 | written = generic_file_direct_write(iocb, from);
|
| 1524 | if (written < 0 || !iov_iter_count(from))
|
| 1525 | goto out;
|
| 1526 | written = direct_write_fallback(iocb, from, written,
|
| 1527 | fuse_perform_write(iocb, from));
|
| 1528 | } else if (writeback) {
|
| 1529 | |
| 1530 | * Use iomap so that we can do granular uptodate reads
|
| 1531 | * and granular dirty tracking for large folios.
|
| 1532 |
|
| 1533 | written = iomap_file_buffered_write(iocb, from,
|
| 1534 | &fuse_iomap_ops,
|
| 1535 | &fuse_iomap_write_ops,
|
| 1536 | file);
|
| 1537 | } else {
|
| 1538 | written = fuse_perform_write(iocb, from);
|
| 1539 | }
|
| 1540 | out:
|
| 1541 | inode_unlock(inode);
|
| 1542 | if (written > 0)
|
| 1543 | written = generic_write_sync(iocb, written);
|
| 1544 |
|
| 1545 | return written ? written : err;
|
| 1546 | }
|
| 1547 |
|
| 1548 | static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii)
|
| 1549 | {
|
| 1550 | return (unsigned long)iter_iov(ii)->iov_base + ii->iov_offset;
|
| 1551 | }
|
| 1552 |
|
| 1553 | static inline size_t fuse_get_frag_size(const struct iov_iter *ii,
|
| 1554 | size_t max_size)
|
| 1555 | {
|
| 1556 | return min(iov_iter_single_seg_count(ii), max_size);
|
| 1557 | }
|
| 1558 |
|
| 1559 | static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii,
|
| 1560 | size_t *nbytesp, int write,
|
| 1561 | unsigned int max_pages,
|
| 1562 | bool use_pages_for_kvec_io)
|
| 1563 | {
|
| 1564 | bool flush_or_invalidate = false;
|
| 1565 | unsigned int nr_pages = 0;
|
| 1566 | size_t nbytes = 0;
|
| 1567 | ssize_t ret = 0;
|
| 1568 |
|
| 1569 | |
| 1570 | * However if the implementation of fuse_conn requires pages instead of
|
| 1571 | * pointer (e.g., virtio-fs), use iov_iter_extract_pages() instead.
|
| 1572 |
|
| 1573 | if (iov_iter_is_kvec(ii)) {
|
| 1574 | void *user_addr = (void *)fuse_get_user_addr(ii);
|
| 1575 |
|
| 1576 | if (!use_pages_for_kvec_io) {
|
| 1577 | size_t frag_size = fuse_get_frag_size(ii, *nbytesp);
|
| 1578 |
|
| 1579 | if (write)
|
| 1580 | ap->args.in_args[1].value = user_addr;
|
| 1581 | else
|
| 1582 | ap->args.out_args[0].value = user_addr;
|
| 1583 |
|
| 1584 | iov_iter_advance(ii, frag_size);
|
| 1585 | *nbytesp = frag_size;
|
| 1586 | return 0;
|
| 1587 | }
|
| 1588 |
|
| 1589 | if (is_vmalloc_addr(user_addr)) {
|
| 1590 | ap->args.vmap_base = user_addr;
|
| 1591 | flush_or_invalidate = true;
|
| 1592 | }
|
| 1593 | }
|
| 1594 |
|
| 1595 | |
| 1596 | * Until there is support for iov_iter_extract_folios(), we have to
|
| 1597 | * manually extract pages using iov_iter_extract_pages() and then
|
| 1598 | * copy that to a folios array.
|
| 1599 |
|
| 1600 | struct page **pages = kzalloc_objs(struct page *, max_pages);
|
| 1601 | if (!pages) {
|
| 1602 | ret = -ENOMEM;
|
| 1603 | goto out;
|
| 1604 | }
|
| 1605 |
|
| 1606 | while (nbytes < *nbytesp && nr_pages < max_pages) {
|
| 1607 | unsigned nfolios, i;
|
| 1608 | size_t start;
|
| 1609 |
|
| 1610 | ret = iov_iter_extract_pages(ii, &pages,
|
| 1611 | *nbytesp - nbytes,
|
| 1612 | max_pages - nr_pages,
|
| 1613 | 0, &start);
|
| 1614 | if (ret < 0)
|
| 1615 | break;
|
| 1616 |
|
| 1617 | nbytes += ret;
|
| 1618 |
|
| 1619 | nfolios = DIV_ROUND_UP(ret + start, PAGE_SIZE);
|
| 1620 |
|
| 1621 | for (i = 0; i < nfolios; i++) {
|
| 1622 | struct folio *folio = page_folio(pages[i]);
|
| 1623 | unsigned int offset = start +
|
| 1624 | (folio_page_idx(folio, pages[i]) << PAGE_SHIFT);
|
| 1625 | unsigned int len = umin(ret, PAGE_SIZE - start);
|
| 1626 |
|
| 1627 | ap->descs[ap->num_folios].offset = offset;
|
| 1628 | ap->descs[ap->num_folios].length = len;
|
| 1629 | ap->folios[ap->num_folios] = folio;
|
| 1630 | start = 0;
|
| 1631 | ret -= len;
|
| 1632 | ap->num_folios++;
|
| 1633 | }
|
| 1634 |
|
| 1635 | nr_pages += nfolios;
|
| 1636 | }
|
| 1637 | kfree(pages);
|
| 1638 |
|
| 1639 | if (write && flush_or_invalidate)
|
| 1640 | flush_kernel_vmap_range(ap->args.vmap_base, nbytes);
|
| 1641 |
|
| 1642 | ap->args.invalidate_vmap = !write && flush_or_invalidate;
|
| 1643 | ap->args.is_pinned = iov_iter_extract_will_pin(ii);
|
| 1644 | ap->args.user_pages = true;
|
| 1645 | if (write)
|
| 1646 | ap->args.in_pages = true;
|
| 1647 | else
|
| 1648 | ap->args.out_pages = true;
|
| 1649 |
|
| 1650 | out:
|
| 1651 | *nbytesp = nbytes;
|
| 1652 |
|
| 1653 | return ret < 0 ? ret : 0;
|
| 1654 | }
|
| 1655 |
|
| 1656 | ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,
|
| 1657 | loff_t *ppos, int flags)
|
| 1658 | {
|
| 1659 | int write = flags & FUSE_DIO_WRITE;
|
| 1660 | int cuse = flags & FUSE_DIO_CUSE;
|
| 1661 | struct file *file = io->iocb->ki_filp;
|
| 1662 | struct address_space *mapping = file->f_mapping;
|
| 1663 | struct inode *inode = mapping->host;
|
| 1664 | struct fuse_file *ff = file->private_data;
|
| 1665 | struct fuse_conn *fc = ff->fm->fc;
|
| 1666 | size_t nmax = write ? fc->max_write : fc->max_read;
|
| 1667 | loff_t pos = *ppos;
|
| 1668 | size_t count = iov_iter_count(iter);
|
| 1669 | pgoff_t idx_from = pos >> PAGE_SHIFT;
|
| 1670 | pgoff_t idx_to = (pos + count - 1) >> PAGE_SHIFT;
|
| 1671 | ssize_t res = 0;
|
| 1672 | int err = 0;
|
| 1673 | struct fuse_io_args *ia;
|
| 1674 | unsigned int max_pages;
|
| 1675 | bool fopen_direct_io = ff->open_flags & FOPEN_DIRECT_IO;
|
| 1676 |
|
| 1677 | max_pages = iov_iter_npages(iter, fc->max_pages);
|
| 1678 | ia = fuse_io_alloc(io, max_pages);
|
| 1679 | if (!ia)
|
| 1680 | return -ENOMEM;
|
| 1681 |
|
| 1682 | if (fopen_direct_io) {
|
| 1683 | res = filemap_write_and_wait_range(mapping, pos, pos + count - 1);
|
| 1684 | if (res) {
|
| 1685 | fuse_io_free(ia);
|
| 1686 | return res;
|
| 1687 | }
|
| 1688 | }
|
| 1689 | if (!cuse && filemap_range_has_writeback(mapping, pos, (pos + count - 1))) {
|
| 1690 | if (!write)
|
| 1691 | inode_lock(inode);
|
| 1692 | fuse_sync_writes(inode);
|
| 1693 | if (!write)
|
| 1694 | inode_unlock(inode);
|
| 1695 | }
|
| 1696 |
|
| 1697 | if (fopen_direct_io && write) {
|
| 1698 | res = invalidate_inode_pages2_range(mapping, idx_from, idx_to);
|
| 1699 | if (res) {
|
| 1700 | fuse_io_free(ia);
|
| 1701 | return res;
|
| 1702 | }
|
| 1703 | }
|
| 1704 |
|
| 1705 | io->should_dirty = !write && user_backed_iter(iter);
|
| 1706 | while (count) {
|
| 1707 | ssize_t nres;
|
| 1708 | fl_owner_t owner = current->files;
|
| 1709 | size_t nbytes = min(count, nmax);
|
| 1710 |
|
| 1711 | err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write,
|
| 1712 | max_pages, fc->use_pages_for_kvec_io);
|
| 1713 | if (err && !nbytes)
|
| 1714 | break;
|
| 1715 |
|
| 1716 | if (write) {
|
| 1717 | if (!capable(CAP_FSETID))
|
| 1718 | ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID;
|
| 1719 |
|
| 1720 | nres = fuse_send_write(ia, pos, nbytes, owner);
|
| 1721 | } else {
|
| 1722 | nres = fuse_send_read(ia, pos, nbytes, owner);
|
| 1723 | }
|
| 1724 |
|
| 1725 | if (!io->async || nres < 0) {
|
| 1726 | fuse_release_user_pages(&ia->ap, nres, io->should_dirty);
|
| 1727 | fuse_io_free(ia);
|
| 1728 | }
|
| 1729 | ia = NULL;
|
| 1730 | if (nres < 0) {
|
| 1731 | iov_iter_revert(iter, nbytes);
|
| 1732 | err = nres;
|
| 1733 | break;
|
| 1734 | }
|
| 1735 | WARN_ON(nres > nbytes);
|
| 1736 |
|
| 1737 | count -= nres;
|
| 1738 | res += nres;
|
| 1739 | pos += nres;
|
| 1740 | if (nres != nbytes) {
|
| 1741 | iov_iter_revert(iter, nbytes - nres);
|
| 1742 | break;
|
| 1743 | }
|
| 1744 | if (count) {
|
| 1745 | max_pages = iov_iter_npages(iter, fc->max_pages);
|
| 1746 | ia = fuse_io_alloc(io, max_pages);
|
| 1747 | if (!ia)
|
| 1748 | break;
|
| 1749 | }
|
| 1750 | }
|
| 1751 | if (ia)
|
| 1752 | fuse_io_free(ia);
|
| 1753 | if (res > 0)
|
| 1754 | *ppos = pos;
|
| 1755 |
|
| 1756 | return res > 0 ? res : err;
|
| 1757 | }
|
| 1758 | EXPORT_SYMBOL_GPL(fuse_direct_io);
|
| 1759 |
|
| 1760 | static ssize_t __fuse_direct_read(struct fuse_io_priv *io,
|
| 1761 | struct iov_iter *iter,
|
| 1762 | loff_t *ppos)
|
| 1763 | {
|
| 1764 | ssize_t res;
|
| 1765 | struct inode *inode = file_inode(io->iocb->ki_filp);
|
| 1766 |
|
| 1767 | res = fuse_direct_io(io, iter, ppos, 0);
|
| 1768 |
|
| 1769 | fuse_invalidate_atime(inode);
|
| 1770 |
|
| 1771 | return res;
|
| 1772 | }
|
| 1773 |
|
| 1774 | static ssize_t fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter);
|
| 1775 |
|
| 1776 | static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to)
|
| 1777 | {
|
| 1778 | ssize_t res;
|
| 1779 |
|
| 1780 | if (!is_sync_kiocb(iocb)) {
|
| 1781 | res = fuse_direct_IO(iocb, to);
|
| 1782 | } else {
|
| 1783 | struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
|
| 1784 |
|
| 1785 | res = __fuse_direct_read(&io, to, &iocb->ki_pos);
|
| 1786 | }
|
| 1787 |
|
| 1788 | return res;
|
| 1789 | }
|
| 1790 |
|
| 1791 | static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from)
|
| 1792 | {
|
| 1793 | struct inode *inode = file_inode(iocb->ki_filp);
|
| 1794 | struct address_space *mapping = inode->i_mapping;
|
| 1795 | ssize_t res;
|
| 1796 | bool exclusive;
|
| 1797 |
|
| 1798 | fuse_dio_lock(iocb, from, &exclusive);
|
| 1799 | res = generic_write_checks(iocb, from);
|
| 1800 | if (res > 0) {
|
| 1801 | loff_t pos = iocb->ki_pos;
|
| 1802 |
|
| 1803 | task_io_account_write(res);
|
| 1804 | if (!is_sync_kiocb(iocb)) {
|
| 1805 | res = fuse_direct_IO(iocb, from);
|
| 1806 | } else {
|
| 1807 | struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
|
| 1808 |
|
| 1809 | res = fuse_direct_io(&io, from, &iocb->ki_pos,
|
| 1810 | FUSE_DIO_WRITE);
|
| 1811 | fuse_write_update_attr(inode, iocb->ki_pos, res);
|
| 1812 | }
|
| 1813 | if (res > 0 && mapping->nrpages) {
|
| 1814 | |
| 1815 | * As in generic_file_direct_write(), invalidate after
|
| 1816 | * write, to invalidate read-ahead cache that may have
|
| 1817 | * competed with the write.
|
| 1818 |
|
| 1819 | invalidate_inode_pages2_range(mapping,
|
| 1820 | pos >> PAGE_SHIFT,
|
| 1821 | (pos + res - 1) >> PAGE_SHIFT);
|
| 1822 | }
|
| 1823 | }
|
| 1824 | fuse_dio_unlock(iocb, exclusive);
|
| 1825 |
|
| 1826 | return res;
|
| 1827 | }
|
| 1828 |
|
| 1829 | static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
|
| 1830 | {
|
| 1831 | struct file *file = iocb->ki_filp;
|
| 1832 | struct fuse_file *ff = file->private_data;
|
| 1833 | struct inode *inode = file_inode(file);
|
| 1834 |
|
| 1835 | if (fuse_is_bad(inode))
|
| 1836 | return -EIO;
|
| 1837 |
|
| 1838 | if (FUSE_IS_DAX(inode))
|
| 1839 | return fuse_dax_read_iter(iocb, to);
|
| 1840 |
|
| 1841 |
|
| 1842 | if (ff->open_flags & FOPEN_DIRECT_IO)
|
| 1843 | return fuse_direct_read_iter(iocb, to);
|
| 1844 | else if (fuse_file_passthrough(ff))
|
| 1845 | return fuse_passthrough_read_iter(iocb, to);
|
| 1846 | else
|
| 1847 | return fuse_cache_read_iter(iocb, to);
|
| 1848 | }
|
| 1849 |
|
| 1850 | static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
|
| 1851 | {
|
| 1852 | struct file *file = iocb->ki_filp;
|
| 1853 | struct fuse_file *ff = file->private_data;
|
| 1854 | struct inode *inode = file_inode(file);
|
| 1855 |
|
| 1856 | if (fuse_is_bad(inode))
|
| 1857 | return -EIO;
|
| 1858 |
|
| 1859 | if (FUSE_IS_DAX(inode))
|
| 1860 | return fuse_dax_write_iter(iocb, from);
|
| 1861 |
|
| 1862 |
|
| 1863 | if (ff->open_flags & FOPEN_DIRECT_IO)
|
| 1864 | return fuse_direct_write_iter(iocb, from);
|
| 1865 | else if (fuse_file_passthrough(ff))
|
| 1866 | return fuse_passthrough_write_iter(iocb, from);
|
| 1867 | else
|
| 1868 | return fuse_cache_write_iter(iocb, from);
|
| 1869 | }
|
| 1870 |
|
| 1871 | static ssize_t fuse_splice_read(struct file *in, loff_t *ppos,
|
| 1872 | struct pipe_inode_info *pipe, size_t len,
|
| 1873 | unsigned int flags)
|
| 1874 | {
|
| 1875 | struct fuse_file *ff = in->private_data;
|
| 1876 |
|
| 1877 |
|
| 1878 | if (fuse_file_passthrough(ff) && !(ff->open_flags & FOPEN_DIRECT_IO))
|
| 1879 | return fuse_passthrough_splice_read(in, ppos, pipe, len, flags);
|
| 1880 | else
|
| 1881 | return filemap_splice_read(in, ppos, pipe, len, flags);
|
| 1882 | }
|
| 1883 |
|
| 1884 | static ssize_t fuse_splice_write(struct pipe_inode_info *pipe, struct file *out,
|
| 1885 | loff_t *ppos, size_t len, unsigned int flags)
|
| 1886 | {
|
| 1887 | struct fuse_file *ff = out->private_data;
|
| 1888 |
|
| 1889 |
|
| 1890 | if (fuse_file_passthrough(ff) && !(ff->open_flags & FOPEN_DIRECT_IO))
|
| 1891 | return fuse_passthrough_splice_write(pipe, out, ppos, len, flags);
|
| 1892 | else
|
| 1893 | return iter_file_splice_write(pipe, out, ppos, len, flags);
|
| 1894 | }
|
| 1895 |
|
| 1896 | static void fuse_writepage_free(struct fuse_writepage_args *wpa)
|
| 1897 | {
|
| 1898 | struct fuse_args_pages *ap = &wpa->ia.ap;
|
| 1899 |
|
| 1900 | if (wpa->bucket)
|
| 1901 | fuse_sync_bucket_dec(wpa->bucket);
|
| 1902 |
|
| 1903 | fuse_file_put(wpa->ia.ff, false);
|
| 1904 |
|
| 1905 | kfree(ap->folios);
|
| 1906 | kfree(wpa);
|
| 1907 | }
|
| 1908 |
|
| 1909 | static void fuse_writepage_finish(struct fuse_writepage_args *wpa)
|
| 1910 | {
|
| 1911 | struct fuse_args_pages *ap = &wpa->ia.ap;
|
| 1912 | struct inode *inode = wpa->inode;
|
| 1913 | struct fuse_inode *fi = get_fuse_inode(inode);
|
| 1914 | int i;
|
| 1915 |
|
| 1916 | for (i = 0; i < ap->num_folios; i++)
|
| 1917 | |
| 1918 | * Benchmarks showed that ending writeback within the
|
| 1919 | * scope of the fi->lock alleviates xarray lock
|
| 1920 | * contention and noticeably improves performance.
|
| 1921 |
|
| 1922 | iomap_finish_folio_write(inode, ap->folios[i],
|
| 1923 | ap->descs[i].length);
|
| 1924 |
|
| 1925 | wake_up(&fi->page_waitq);
|
| 1926 | }
|
| 1927 |
|
| 1928 |
|
| 1929 | static void fuse_send_writepage(struct fuse_mount *fm,
|
| 1930 | struct fuse_writepage_args *wpa, loff_t size)
|
| 1931 | __releases(fi->lock)
|
| 1932 | __acquires(fi->lock)
|
| 1933 | {
|
| 1934 | struct fuse_inode *fi = get_fuse_inode(wpa->inode);
|
| 1935 | struct fuse_args_pages *ap = &wpa->ia.ap;
|
| 1936 | struct fuse_write_in *inarg = &wpa->ia.write.in;
|
| 1937 | struct fuse_args *args = &ap->args;
|
| 1938 | __u64 data_size = 0;
|
| 1939 | int err, i;
|
| 1940 |
|
| 1941 | for (i = 0; i < ap->num_folios; i++)
|
| 1942 | data_size += ap->descs[i].length;
|
| 1943 |
|
| 1944 | fi->writectr++;
|
| 1945 | if (inarg->offset + data_size <= size) {
|
| 1946 | inarg->size = data_size;
|
| 1947 | } else if (inarg->offset < size) {
|
| 1948 | inarg->size = size - inarg->offset;
|
| 1949 | } else {
|
| 1950 |
|
| 1951 | goto out_free;
|
| 1952 | }
|
| 1953 |
|
| 1954 | args->in_args[1].size = inarg->size;
|
| 1955 | args->force = true;
|
| 1956 | args->nocreds = true;
|
| 1957 |
|
| 1958 | err = fuse_simple_background(fm, args, GFP_ATOMIC);
|
| 1959 | if (err == -ENOMEM) {
|
| 1960 | spin_unlock(&fi->lock);
|
| 1961 | err = fuse_simple_background(fm, args, GFP_NOFS | __GFP_NOFAIL);
|
| 1962 | spin_lock(&fi->lock);
|
| 1963 | }
|
| 1964 |
|
| 1965 |
|
| 1966 | if (unlikely(err))
|
| 1967 | goto out_free;
|
| 1968 |
|
| 1969 | return;
|
| 1970 |
|
| 1971 | out_free:
|
| 1972 | fi->writectr--;
|
| 1973 | fuse_writepage_finish(wpa);
|
| 1974 | spin_unlock(&fi->lock);
|
| 1975 | fuse_writepage_free(wpa);
|
| 1976 | spin_lock(&fi->lock);
|
| 1977 | }
|
| 1978 |
|
| 1979 | |
| 1980 | * If fi->writectr is positive (no truncate or fsync going on) send
|
| 1981 | * all queued writepage requests.
|
| 1982 | *
|
| 1983 | * Called with fi->lock
|
| 1984 |
|
| 1985 | void fuse_flush_writepages(struct inode *inode)
|
| 1986 | __releases(fi->lock)
|
| 1987 | __acquires(fi->lock)
|
| 1988 | {
|
| 1989 | struct fuse_mount *fm = get_fuse_mount(inode);
|
| 1990 | struct fuse_inode *fi = get_fuse_inode(inode);
|
| 1991 | loff_t crop = i_size_read(inode);
|
| 1992 | struct fuse_writepage_args *wpa;
|
| 1993 |
|
| 1994 | while (fi->writectr >= 0 && !list_empty(&fi->queued_writes)) {
|
| 1995 | wpa = list_entry(fi->queued_writes.next,
|
| 1996 | struct fuse_writepage_args, queue_entry);
|
| 1997 | list_del_init(&wpa->queue_entry);
|
| 1998 | fuse_send_writepage(fm, wpa, crop);
|
| 1999 | }
|
| 2000 | }
|
| 2001 |
|
| 2002 | static void fuse_writepage_end(struct fuse_args *args, int error)
|
| 2003 | {
|
| 2004 | struct fuse_writepage_args *wpa =
|
| 2005 | container_of(args, typeof(*wpa), ia.ap.args);
|
| 2006 | struct inode *inode = wpa->inode;
|
| 2007 | struct fuse_inode *fi = get_fuse_inode(inode);
|
| 2008 | struct fuse_conn *fc = get_fuse_conn(inode);
|
| 2009 |
|
| 2010 | mapping_set_error(inode->i_mapping, error);
|
| 2011 | |
| 2012 | * A writeback finished and this might have updated mtime/ctime on
|
| 2013 | * server making local mtime/ctime stale. Hence invalidate attrs.
|
| 2014 | * Do this only if writeback_cache is not enabled. If writeback_cache
|
| 2015 | * is enabled, we trust local ctime/mtime.
|
| 2016 |
|
| 2017 | if (!fc->writeback_cache)
|
| 2018 | fuse_invalidate_attr_mask(inode, FUSE_STATX_MODIFY);
|
| 2019 | spin_lock(&fi->lock);
|
| 2020 | fi->writectr--;
|
| 2021 | fuse_writepage_finish(wpa);
|
| 2022 | spin_unlock(&fi->lock);
|
| 2023 | fuse_writepage_free(wpa);
|
| 2024 | }
|
| 2025 |
|
| 2026 | static struct fuse_file *__fuse_write_file_get(struct fuse_inode *fi)
|
| 2027 | {
|
| 2028 | struct fuse_file *ff;
|
| 2029 |
|
| 2030 | spin_lock(&fi->lock);
|
| 2031 | ff = list_first_entry_or_null(&fi->write_files, struct fuse_file,
|
| 2032 | write_entry);
|
| 2033 | if (ff)
|
| 2034 | fuse_file_get(ff);
|
| 2035 | spin_unlock(&fi->lock);
|
| 2036 |
|
| 2037 | return ff;
|
| 2038 | }
|
| 2039 |
|
| 2040 | static struct fuse_file *fuse_write_file_get(struct fuse_inode *fi)
|
| 2041 | {
|
| 2042 | struct fuse_file *ff = __fuse_write_file_get(fi);
|
| 2043 | WARN_ON(!ff);
|
| 2044 | return ff;
|
| 2045 | }
|
| 2046 |
|
| 2047 | int fuse_write_inode(struct inode *inode, struct writeback_control *wbc)
|
| 2048 | {
|
| 2049 | struct fuse_inode *fi = get_fuse_inode(inode);
|
| 2050 | struct fuse_file *ff;
|
| 2051 | int err;
|
| 2052 |
|
| 2053 | ff = __fuse_write_file_get(fi);
|
| 2054 | err = fuse_flush_times(inode, ff);
|
| 2055 | if (ff)
|
| 2056 | fuse_file_put(ff, false);
|
| 2057 |
|
| 2058 | return err;
|
| 2059 | }
|
| 2060 |
|
| 2061 | static struct fuse_writepage_args *fuse_writepage_args_alloc(void)
|
| 2062 | {
|
| 2063 | struct fuse_writepage_args *wpa;
|
| 2064 | struct fuse_args_pages *ap;
|
| 2065 |
|
| 2066 | wpa = kzalloc_obj(*wpa, GFP_NOFS);
|
| 2067 | if (wpa) {
|
| 2068 | ap = &wpa->ia.ap;
|
| 2069 | ap->num_folios = 0;
|
| 2070 | ap->folios = fuse_folios_alloc(1, GFP_NOFS, &ap->descs);
|
| 2071 | if (!ap->folios) {
|
| 2072 | kfree(wpa);
|
| 2073 | wpa = NULL;
|
| 2074 | }
|
| 2075 | }
|
| 2076 | return wpa;
|
| 2077 |
|
| 2078 | }
|
| 2079 |
|
| 2080 | static void fuse_writepage_add_to_bucket(struct fuse_conn *fc,
|
| 2081 | struct fuse_writepage_args *wpa)
|
| 2082 | {
|
| 2083 | if (!fc->sync_fs)
|
| 2084 | return;
|
| 2085 |
|
| 2086 | rcu_read_lock();
|
| 2087 |
|
| 2088 | do {
|
| 2089 | wpa->bucket = rcu_dereference(fc->curr_bucket);
|
| 2090 | } while (unlikely(!atomic_inc_not_zero(&wpa->bucket->count)));
|
| 2091 | rcu_read_unlock();
|
| 2092 | }
|
| 2093 |
|
| 2094 | static void fuse_writepage_args_page_fill(struct fuse_writepage_args *wpa, struct folio *folio,
|
| 2095 | uint32_t folio_index, loff_t offset, unsigned len)
|
| 2096 | {
|
| 2097 | struct fuse_args_pages *ap = &wpa->ia.ap;
|
| 2098 |
|
| 2099 | ap->folios[folio_index] = folio;
|
| 2100 | ap->descs[folio_index].offset = offset;
|
| 2101 | ap->descs[folio_index].length = len;
|
| 2102 | }
|
| 2103 |
|
| 2104 | static struct fuse_writepage_args *fuse_writepage_args_setup(struct folio *folio,
|
| 2105 | size_t offset,
|
| 2106 | struct fuse_file *ff)
|
| 2107 | {
|
| 2108 | struct inode *inode = folio->mapping->host;
|
| 2109 | struct fuse_conn *fc = get_fuse_conn(inode);
|
| 2110 | struct fuse_writepage_args *wpa;
|
| 2111 | struct fuse_args_pages *ap;
|
| 2112 |
|
| 2113 | wpa = fuse_writepage_args_alloc();
|
| 2114 | if (!wpa)
|
| 2115 | return NULL;
|
| 2116 |
|
| 2117 | fuse_writepage_add_to_bucket(fc, wpa);
|
| 2118 | fuse_write_args_fill(&wpa->ia, ff, folio_pos(folio) + offset, 0);
|
| 2119 | wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE;
|
| 2120 | wpa->inode = inode;
|
| 2121 | wpa->ia.ff = ff;
|
| 2122 |
|
| 2123 | ap = &wpa->ia.ap;
|
| 2124 | ap->args.in_pages = true;
|
| 2125 | ap->args.end = fuse_writepage_end;
|
| 2126 |
|
| 2127 | return wpa;
|
| 2128 | }
|
| 2129 |
|
| 2130 | struct fuse_fill_wb_data {
|
| 2131 | struct fuse_writepage_args *wpa;
|
| 2132 | struct fuse_file *ff;
|
| 2133 | unsigned int max_folios;
|
| 2134 | |
| 2135 | * nr_bytes won't overflow since fuse_folios_need_send() caps
|
| 2136 | * wb requests to never exceed fc->max_pages (which has an upper bound
|
| 2137 | * of U16_MAX).
|
| 2138 |
|
| 2139 | unsigned int nr_bytes;
|
| 2140 | };
|
| 2141 |
|
| 2142 | static bool fuse_pages_realloc(struct fuse_fill_wb_data *data,
|
| 2143 | unsigned int max_pages)
|
| 2144 | {
|
| 2145 | struct fuse_args_pages *ap = &data->wpa->ia.ap;
|
| 2146 | struct folio **folios;
|
| 2147 | struct fuse_folio_desc *descs;
|
| 2148 | unsigned int nfolios = min_t(unsigned int,
|
| 2149 | max_t(unsigned int, data->max_folios * 2,
|
| 2150 | FUSE_DEFAULT_MAX_PAGES_PER_REQ),
|
| 2151 | max_pages);
|
| 2152 | WARN_ON(nfolios <= data->max_folios);
|
| 2153 |
|
| 2154 | folios = fuse_folios_alloc(nfolios, GFP_NOFS, &descs);
|
| 2155 | if (!folios)
|
| 2156 | return false;
|
| 2157 |
|
| 2158 | memcpy(folios, ap->folios, sizeof(struct folio *) * ap->num_folios);
|
| 2159 | memcpy(descs, ap->descs, sizeof(struct fuse_folio_desc) * ap->num_folios);
|
| 2160 | kfree(ap->folios);
|
| 2161 | ap->folios = folios;
|
| 2162 | ap->descs = descs;
|
| 2163 | data->max_folios = nfolios;
|
| 2164 |
|
| 2165 | return true;
|
| 2166 | }
|
| 2167 |
|
| 2168 | static void fuse_writepages_send(struct inode *inode,
|
| 2169 | struct fuse_fill_wb_data *data)
|
| 2170 | {
|
| 2171 | struct fuse_writepage_args *wpa = data->wpa;
|
| 2172 | struct fuse_inode *fi = get_fuse_inode(inode);
|
| 2173 |
|
| 2174 | spin_lock(&fi->lock);
|
| 2175 | list_add_tail(&wpa->queue_entry, &fi->queued_writes);
|
| 2176 | fuse_flush_writepages(inode);
|
| 2177 | spin_unlock(&fi->lock);
|
| 2178 | }
|
| 2179 |
|
| 2180 | static bool fuse_folios_need_send(struct fuse_conn *fc, loff_t pos,
|
| 2181 | unsigned len, struct fuse_args_pages *ap,
|
| 2182 | unsigned cur_bytes, bool write)
|
| 2183 | {
|
| 2184 | struct folio *prev_folio;
|
| 2185 | struct fuse_folio_desc prev_desc;
|
| 2186 | unsigned bytes = cur_bytes + len;
|
| 2187 | loff_t prev_pos;
|
| 2188 | size_t max_bytes = write ? fc->max_write : fc->max_read;
|
| 2189 |
|
| 2190 | WARN_ON(!ap->num_folios);
|
| 2191 |
|
| 2192 |
|
| 2193 | if (ap->num_folios >= fc->max_pages)
|
| 2194 | return true;
|
| 2195 |
|
| 2196 | if (DIV_ROUND_UP(bytes, PAGE_SIZE) > fc->max_pages)
|
| 2197 | return true;
|
| 2198 |
|
| 2199 | if (bytes > max_bytes)
|
| 2200 | return true;
|
| 2201 |
|
| 2202 |
|
| 2203 | prev_folio = ap->folios[ap->num_folios - 1];
|
| 2204 | prev_desc = ap->descs[ap->num_folios - 1];
|
| 2205 | prev_pos = folio_pos(prev_folio) + prev_desc.offset + prev_desc.length;
|
| 2206 | if (prev_pos != pos)
|
| 2207 | return true;
|
| 2208 |
|
| 2209 | return false;
|
| 2210 | }
|
| 2211 |
|
| 2212 | static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc,
|
| 2213 | struct folio *folio, u64 pos,
|
| 2214 | unsigned len, u64 end_pos)
|
| 2215 | {
|
| 2216 | struct fuse_fill_wb_data *data = wpc->wb_ctx;
|
| 2217 | struct fuse_writepage_args *wpa = data->wpa;
|
| 2218 | struct fuse_args_pages *ap = &wpa->ia.ap;
|
| 2219 | struct inode *inode = wpc->inode;
|
| 2220 | struct fuse_inode *fi = get_fuse_inode(inode);
|
| 2221 | struct fuse_conn *fc = get_fuse_conn(inode);
|
| 2222 | loff_t offset = offset_in_folio(folio, pos);
|
| 2223 |
|
| 2224 | WARN_ON_ONCE(!data);
|
| 2225 |
|
| 2226 | if (!data->ff) {
|
| 2227 | data->ff = fuse_write_file_get(fi);
|
| 2228 | if (!data->ff)
|
| 2229 | return -EIO;
|
| 2230 | }
|
| 2231 |
|
| 2232 | if (wpa) {
|
| 2233 | bool send = fuse_folios_need_send(fc, pos, len, ap,
|
| 2234 | data->nr_bytes, true);
|
| 2235 |
|
| 2236 | if (!send) {
|
| 2237 | |
| 2238 | * Need to grow the pages array? If so, did the
|
| 2239 | * expansion fail?
|
| 2240 |
|
| 2241 | send = (ap->num_folios == data->max_folios) &&
|
| 2242 | !fuse_pages_realloc(data, fc->max_pages);
|
| 2243 | }
|
| 2244 |
|
| 2245 | if (send) {
|
| 2246 | fuse_writepages_send(inode, data);
|
| 2247 | data->wpa = NULL;
|
| 2248 | data->nr_bytes = 0;
|
| 2249 | }
|
| 2250 | }
|
| 2251 |
|
| 2252 | if (data->wpa == NULL) {
|
| 2253 | wpa = fuse_writepage_args_setup(folio, offset, data->ff);
|
| 2254 | if (!wpa)
|
| 2255 | return -ENOMEM;
|
| 2256 | fuse_file_get(wpa->ia.ff);
|
| 2257 | data->max_folios = 1;
|
| 2258 | ap = &wpa->ia.ap;
|
| 2259 | }
|
| 2260 |
|
| 2261 | fuse_writepage_args_page_fill(wpa, folio, ap->num_folios,
|
| 2262 | offset, len);
|
| 2263 | data->nr_bytes += len;
|
| 2264 |
|
| 2265 | ap->num_folios++;
|
| 2266 | if (!data->wpa)
|
| 2267 | data->wpa = wpa;
|
| 2268 |
|
| 2269 | return len;
|
| 2270 | }
|
| 2271 |
|
| 2272 | static int fuse_iomap_writeback_submit(struct iomap_writepage_ctx *wpc,
|
| 2273 | int error)
|
| 2274 | {
|
| 2275 | struct fuse_fill_wb_data *data = wpc->wb_ctx;
|
| 2276 |
|
| 2277 | WARN_ON_ONCE(!data);
|
| 2278 |
|
| 2279 | if (data->wpa) {
|
| 2280 | WARN_ON(!data->wpa->ia.ap.num_folios);
|
| 2281 | fuse_writepages_send(wpc->inode, data);
|
| 2282 | }
|
| 2283 |
|
| 2284 | if (data->ff)
|
| 2285 | fuse_file_put(data->ff, false);
|
| 2286 |
|
| 2287 | return error;
|
| 2288 | }
|
| 2289 |
|
| 2290 | static const struct iomap_writeback_ops fuse_writeback_ops = {
|
| 2291 | .writeback_range = fuse_iomap_writeback_range,
|
| 2292 | .writeback_submit = fuse_iomap_writeback_submit,
|
| 2293 | };
|
| 2294 |
|
| 2295 | static int fuse_writepages(struct address_space *mapping,
|
| 2296 | struct writeback_control *wbc)
|
| 2297 | {
|
| 2298 | struct inode *inode = mapping->host;
|
| 2299 | struct fuse_conn *fc = get_fuse_conn(inode);
|
| 2300 | struct fuse_fill_wb_data data = {};
|
| 2301 | struct iomap_writepage_ctx wpc = {
|
| 2302 | .inode = inode,
|
| 2303 | .iomap.type = IOMAP_MAPPED,
|
| 2304 | .wbc = wbc,
|
| 2305 | .ops = &fuse_writeback_ops,
|
| 2306 | .wb_ctx = &data,
|
| 2307 | };
|
| 2308 |
|
| 2309 | if (fuse_is_bad(inode))
|
| 2310 | return -EIO;
|
| 2311 |
|
| 2312 | if (wbc->sync_mode == WB_SYNC_NONE &&
|
| 2313 | fuse_chan_num_background(fc->chan) >= fc->congestion_threshold)
|
| 2314 | return 0;
|
| 2315 |
|
| 2316 | return iomap_writepages(&wpc);
|
| 2317 | }
|
| 2318 |
|
| 2319 | static int fuse_launder_folio(struct folio *folio)
|
| 2320 | {
|
| 2321 | int err = 0;
|
| 2322 | struct fuse_fill_wb_data data = {};
|
| 2323 | struct iomap_writepage_ctx wpc = {
|
| 2324 | .inode = folio->mapping->host,
|
| 2325 | .iomap.type = IOMAP_MAPPED,
|
| 2326 | .ops = &fuse_writeback_ops,
|
| 2327 | .wb_ctx = &data,
|
| 2328 | };
|
| 2329 |
|
| 2330 | if (folio_clear_dirty_for_io(folio)) {
|
| 2331 | err = iomap_writeback_folio(&wpc, folio);
|
| 2332 | err = fuse_iomap_writeback_submit(&wpc, err);
|
| 2333 | if (!err)
|
| 2334 | folio_wait_writeback(folio);
|
| 2335 | }
|
| 2336 | return err;
|
| 2337 | }
|
| 2338 |
|
| 2339 | |
| 2340 | * Write back dirty data/metadata now (there may not be any suitable
|
| 2341 | * open files later for data)
|
| 2342 |
|
| 2343 | static void fuse_vma_close(struct vm_area_struct *vma)
|
| 2344 | {
|
| 2345 | int err;
|
| 2346 |
|
| 2347 | err = write_inode_now(vma->vm_file->f_mapping->host, 1);
|
| 2348 | mapping_set_error(vma->vm_file->f_mapping, err);
|
| 2349 | }
|
| 2350 |
|
| 2351 | |
| 2352 | * Wait for writeback against this page to complete before allowing it
|
| 2353 | * to be marked dirty again, and hence written back again, possibly
|
| 2354 | * before the previous writepage completed.
|
| 2355 | *
|
| 2356 | * Block here, instead of in ->writepage(), so that the userspace fs
|
| 2357 | * can only block processes actually operating on the filesystem.
|
| 2358 | *
|
| 2359 | * Otherwise unprivileged userspace fs would be able to block
|
| 2360 | * unrelated:
|
| 2361 | *
|
| 2362 | * - page migration
|
| 2363 | * - sync(2)
|
| 2364 | * - try_to_free_pages() with order > PAGE_ALLOC_COSTLY_ORDER
|
| 2365 |
|
| 2366 | static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf)
|
| 2367 | {
|
| 2368 | struct folio *folio = page_folio(vmf->page);
|
| 2369 | struct inode *inode = file_inode(vmf->vma->vm_file);
|
| 2370 |
|
| 2371 | file_update_time(vmf->vma->vm_file);
|
| 2372 | folio_lock(folio);
|
| 2373 | if (folio->mapping != inode->i_mapping) {
|
| 2374 | folio_unlock(folio);
|
| 2375 | return VM_FAULT_NOPAGE;
|
| 2376 | }
|
| 2377 |
|
| 2378 | folio_wait_writeback(folio);
|
| 2379 | return VM_FAULT_LOCKED;
|
| 2380 | }
|
| 2381 |
|
| 2382 | static const struct vm_operations_struct fuse_file_vm_ops = {
|
| 2383 | .close = fuse_vma_close,
|
| 2384 | .fault = filemap_fault,
|
| 2385 | .map_pages = filemap_map_pages,
|
| 2386 | .page_mkwrite = fuse_page_mkwrite,
|
| 2387 | };
|
| 2388 |
|
| 2389 | static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma)
|
| 2390 | {
|
| 2391 | struct fuse_file *ff = file->private_data;
|
| 2392 | struct fuse_conn *fc = ff->fm->fc;
|
| 2393 | struct inode *inode = file_inode(file);
|
| 2394 | int rc;
|
| 2395 |
|
| 2396 |
|
| 2397 | if (FUSE_IS_DAX(inode))
|
| 2398 | return fuse_dax_mmap(file, vma);
|
| 2399 |
|
| 2400 | |
| 2401 | * If inode is in passthrough io mode, because it has some file open
|
| 2402 | * in passthrough mode, either mmap to backing file or fail mmap,
|
| 2403 | * because mixing cached mmap and passthrough io mode is not allowed.
|
| 2404 |
|
| 2405 | if (fuse_file_passthrough(ff))
|
| 2406 | return fuse_passthrough_mmap(file, vma);
|
| 2407 | else if (fuse_inode_backing(get_fuse_inode(inode)))
|
| 2408 | return -ENODEV;
|
| 2409 |
|
| 2410 | |
| 2411 | * FOPEN_DIRECT_IO handling is special compared to O_DIRECT,
|
| 2412 | * as does not allow MAP_SHARED mmap without FUSE_DIRECT_IO_ALLOW_MMAP.
|
| 2413 |
|
| 2414 | if (ff->open_flags & FOPEN_DIRECT_IO) {
|
| 2415 | |
| 2416 | * Can't provide the coherency needed for MAP_SHARED
|
| 2417 | * if FUSE_DIRECT_IO_ALLOW_MMAP isn't set.
|
| 2418 |
|
| 2419 | if ((vma->vm_flags & VM_MAYSHARE) && !fc->direct_io_allow_mmap)
|
| 2420 | return -ENODEV;
|
| 2421 |
|
| 2422 | invalidate_inode_pages2(file->f_mapping);
|
| 2423 |
|
| 2424 | if (!(vma->vm_flags & VM_MAYSHARE)) {
|
| 2425 |
|
| 2426 | return generic_file_mmap(file, vma);
|
| 2427 | }
|
| 2428 |
|
| 2429 | |
| 2430 | * First mmap of direct_io file enters caching inode io mode.
|
| 2431 | * Also waits for parallel dio writers to go into serial mode
|
| 2432 | * (exclusive instead of shared lock).
|
| 2433 | * After first mmap, the inode stays in caching io mode until
|
| 2434 | * the direct_io file release.
|
| 2435 |
|
| 2436 | rc = fuse_file_cached_io_open(inode, ff);
|
| 2437 | if (rc)
|
| 2438 | return rc;
|
| 2439 | }
|
| 2440 |
|
| 2441 | if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
|
| 2442 | fuse_link_write_file(file);
|
| 2443 |
|
| 2444 | file_accessed(file);
|
| 2445 | vma->vm_ops = &fuse_file_vm_ops;
|
| 2446 | return 0;
|
| 2447 | }
|
| 2448 |
|
| 2449 | static int convert_fuse_file_lock(struct fuse_conn *fc,
|
| 2450 | const struct fuse_file_lock *ffl,
|
| 2451 | struct file_lock *fl)
|
| 2452 | {
|
| 2453 | switch (ffl->type) {
|
| 2454 | case F_UNLCK:
|
| 2455 | break;
|
| 2456 |
|
| 2457 | case F_RDLCK:
|
| 2458 | case F_WRLCK:
|
| 2459 | if (ffl->start > OFFSET_MAX || ffl->end > OFFSET_MAX ||
|
| 2460 | ffl->end < ffl->start)
|
| 2461 | return -EIO;
|
| 2462 |
|
| 2463 | fl->fl_start = ffl->start;
|
| 2464 | fl->fl_end = ffl->end;
|
| 2465 |
|
| 2466 | |
| 2467 | * Convert pid into init's pid namespace. The locks API will
|
| 2468 | * translate it into the caller's pid namespace.
|
| 2469 |
|
| 2470 | rcu_read_lock();
|
| 2471 | fl->c.flc_pid = pid_nr_ns(find_pid_ns(ffl->pid, fc->pid_ns), &init_pid_ns);
|
| 2472 | rcu_read_unlock();
|
| 2473 | break;
|
| 2474 |
|
| 2475 | default:
|
| 2476 | return -EIO;
|
| 2477 | }
|
| 2478 | fl->c.flc_type = ffl->type;
|
| 2479 | return 0;
|
| 2480 | }
|
| 2481 |
|
| 2482 | static void fuse_lk_fill(struct fuse_args *args, struct file *file,
|
| 2483 | const struct file_lock *fl, int opcode, pid_t pid,
|
| 2484 | int flock, struct fuse_lk_in *inarg)
|
| 2485 | {
|
| 2486 | struct inode *inode = file_inode(file);
|
| 2487 | struct fuse_conn *fc = get_fuse_conn(inode);
|
| 2488 | struct fuse_file *ff = file->private_data;
|
| 2489 |
|
| 2490 | memset(inarg, 0, sizeof(*inarg));
|
| 2491 | inarg->fh = ff->fh;
|
| 2492 | inarg->owner = fuse_lock_owner_id(fc, fl->c.flc_owner);
|
| 2493 | inarg->lk.start = fl->fl_start;
|
| 2494 | inarg->lk.end = fl->fl_end;
|
| 2495 | inarg->lk.type = fl->c.flc_type;
|
| 2496 | inarg->lk.pid = pid;
|
| 2497 | if (flock)
|
| 2498 | inarg->lk_flags |= FUSE_LK_FLOCK;
|
| 2499 | args->opcode = opcode;
|
| 2500 | args->nodeid = get_node_id(inode);
|
| 2501 | args->in_numargs = 1;
|
| 2502 | args->in_args[0].size = sizeof(*inarg);
|
| 2503 | args->in_args[0].value = inarg;
|
| 2504 | }
|
| 2505 |
|
| 2506 | static int fuse_getlk(struct file *file, struct file_lock *fl)
|
| 2507 | {
|
| 2508 | struct inode *inode = file_inode(file);
|
| 2509 | struct fuse_mount *fm = get_fuse_mount(inode);
|
| 2510 | FUSE_ARGS(args);
|
| 2511 | struct fuse_lk_in inarg;
|
| 2512 | struct fuse_lk_out outarg;
|
| 2513 | int err;
|
| 2514 |
|
| 2515 | fuse_lk_fill(&args, file, fl, FUSE_GETLK, 0, 0, &inarg);
|
| 2516 | args.out_numargs = 1;
|
| 2517 | args.out_args[0].size = sizeof(outarg);
|
| 2518 | args.out_args[0].value = &outarg;
|
| 2519 | err = fuse_simple_request(fm, &args);
|
| 2520 | if (!err)
|
| 2521 | err = convert_fuse_file_lock(fm->fc, &outarg.lk, fl);
|
| 2522 |
|
| 2523 | return err;
|
| 2524 | }
|
| 2525 |
|
| 2526 | static int fuse_setlk(struct file *file, struct file_lock *fl, int flock)
|
| 2527 | {
|
| 2528 | struct inode *inode = file_inode(file);
|
| 2529 | struct fuse_mount *fm = get_fuse_mount(inode);
|
| 2530 | FUSE_ARGS(args);
|
| 2531 | struct fuse_lk_in inarg;
|
| 2532 | int opcode = (fl->c.flc_flags & FL_SLEEP) ? FUSE_SETLKW : FUSE_SETLK;
|
| 2533 | struct pid *pid = fl->c.flc_type != F_UNLCK ? task_tgid(current) : NULL;
|
| 2534 | pid_t pid_nr = pid_nr_ns(pid, fm->fc->pid_ns);
|
| 2535 | int err;
|
| 2536 |
|
| 2537 | if (fl->fl_lmops && fl->fl_lmops->lm_grant) {
|
| 2538 |
|
| 2539 | return -ENOLCK;
|
| 2540 | }
|
| 2541 |
|
| 2542 | fuse_lk_fill(&args, file, fl, opcode, pid_nr, flock, &inarg);
|
| 2543 | err = fuse_simple_request(fm, &args);
|
| 2544 |
|
| 2545 |
|
| 2546 | if (err == -EINTR)
|
| 2547 | err = -ERESTARTSYS;
|
| 2548 |
|
| 2549 | return err;
|
| 2550 | }
|
| 2551 |
|
| 2552 | static int fuse_file_lock(struct file *file, int cmd, struct file_lock *fl)
|
| 2553 | {
|
| 2554 | struct inode *inode = file_inode(file);
|
| 2555 | struct fuse_conn *fc = get_fuse_conn(inode);
|
| 2556 | int err;
|
| 2557 |
|
| 2558 | if (cmd == F_CANCELLK) {
|
| 2559 | err = 0;
|
| 2560 | } else if (cmd == F_GETLK) {
|
| 2561 | if (fc->no_lock) {
|
| 2562 | posix_test_lock(file, fl);
|
| 2563 | err = 0;
|
| 2564 | } else
|
| 2565 | err = fuse_getlk(file, fl);
|
| 2566 | } else {
|
| 2567 | if (fc->no_lock)
|
| 2568 | err = posix_lock_file(file, fl, NULL);
|
| 2569 | else
|
| 2570 | err = fuse_setlk(file, fl, 0);
|
| 2571 | }
|
| 2572 | return err;
|
| 2573 | }
|
| 2574 |
|
| 2575 | static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl)
|
| 2576 | {
|
| 2577 | struct inode *inode = file_inode(file);
|
| 2578 | struct fuse_conn *fc = get_fuse_conn(inode);
|
| 2579 | int err;
|
| 2580 |
|
| 2581 | if (fc->no_flock) {
|
| 2582 | err = locks_lock_file_wait(file, fl);
|
| 2583 | } else {
|
| 2584 | struct fuse_file *ff = file->private_data;
|
| 2585 |
|
| 2586 |
|
| 2587 | err = fuse_setlk(file, fl, 1);
|
| 2588 | if (!err)
|
| 2589 | ff->flock = true;
|
| 2590 | }
|
| 2591 |
|
| 2592 | return err;
|
| 2593 | }
|
| 2594 |
|
| 2595 | static sector_t fuse_bmap(struct address_space *mapping, sector_t block)
|
| 2596 | {
|
| 2597 | struct inode *inode = mapping->host;
|
| 2598 | struct fuse_mount *fm = get_fuse_mount(inode);
|
| 2599 | FUSE_ARGS(args);
|
| 2600 | struct fuse_bmap_in inarg;
|
| 2601 | struct fuse_bmap_out outarg;
|
| 2602 | int err;
|
| 2603 |
|
| 2604 | if (!inode->i_sb->s_bdev || fm->fc->no_bmap)
|
| 2605 | return 0;
|
| 2606 |
|
| 2607 | memset(&inarg, 0, sizeof(inarg));
|
| 2608 | inarg.block = block;
|
| 2609 | inarg.blocksize = inode->i_sb->s_blocksize;
|
| 2610 | args.opcode = FUSE_BMAP;
|
| 2611 | args.nodeid = get_node_id(inode);
|
| 2612 | args.in_numargs = 1;
|
| 2613 | args.in_args[0].size = sizeof(inarg);
|
| 2614 | args.in_args[0].value = &inarg;
|
| 2615 | args.out_numargs = 1;
|
| 2616 | args.out_args[0].size = sizeof(outarg);
|
| 2617 | args.out_args[0].value = &outarg;
|
| 2618 | err = fuse_simple_request(fm, &args);
|
| 2619 | if (err == -ENOSYS)
|
| 2620 | fm->fc->no_bmap = 1;
|
| 2621 |
|
| 2622 | return err ? 0 : outarg.block;
|
| 2623 | }
|
| 2624 |
|
| 2625 | static loff_t fuse_lseek(struct file *file, loff_t offset, int whence)
|
| 2626 | {
|
| 2627 | struct inode *inode = file->f_mapping->host;
|
| 2628 | struct fuse_mount *fm = get_fuse_mount(inode);
|
| 2629 | struct fuse_file *ff = file->private_data;
|
| 2630 | FUSE_ARGS(args);
|
| 2631 | struct fuse_lseek_in inarg = {
|
| 2632 | .fh = ff->fh,
|
| 2633 | .offset = offset,
|
| 2634 | .whence = whence
|
| 2635 | };
|
| 2636 | struct fuse_lseek_out outarg;
|
| 2637 | int err;
|
| 2638 |
|
| 2639 | if (fm->fc->no_lseek)
|
| 2640 | goto fallback;
|
| 2641 |
|
| 2642 | args.opcode = FUSE_LSEEK;
|
| 2643 | args.nodeid = ff->nodeid;
|
| 2644 | args.in_numargs = 1;
|
| 2645 | args.in_args[0].size = sizeof(inarg);
|
| 2646 | args.in_args[0].value = &inarg;
|
| 2647 | args.out_numargs = 1;
|
| 2648 | args.out_args[0].size = sizeof(outarg);
|
| 2649 | args.out_args[0].value = &outarg;
|
| 2650 | err = fuse_simple_request(fm, &args);
|
| 2651 | if (err) {
|
| 2652 | if (err == -ENOSYS) {
|
| 2653 | fm->fc->no_lseek = 1;
|
| 2654 | goto fallback;
|
| 2655 | }
|
| 2656 | return err;
|
| 2657 | }
|
| 2658 |
|
| 2659 | return vfs_setpos(file, outarg.offset, inode->i_sb->s_maxbytes);
|
| 2660 |
|
| 2661 | fallback:
|
| 2662 | err = fuse_update_attributes(inode, file, STATX_SIZE);
|
| 2663 | if (!err)
|
| 2664 | return generic_file_llseek(file, offset, whence);
|
| 2665 | else
|
| 2666 | return err;
|
| 2667 | }
|
| 2668 |
|
| 2669 | static loff_t fuse_file_llseek(struct file *file, loff_t offset, int whence)
|
| 2670 | {
|
| 2671 | loff_t retval;
|
| 2672 | struct inode *inode = file_inode(file);
|
| 2673 |
|
| 2674 | switch (whence) {
|
| 2675 | case SEEK_SET:
|
| 2676 | case SEEK_CUR:
|
| 2677 |
|
| 2678 | retval = generic_file_llseek(file, offset, whence);
|
| 2679 | break;
|
| 2680 | case SEEK_END:
|
| 2681 | inode_lock(inode);
|
| 2682 | retval = fuse_update_attributes(inode, file, STATX_SIZE);
|
| 2683 | if (!retval)
|
| 2684 | retval = generic_file_llseek(file, offset, whence);
|
| 2685 | inode_unlock(inode);
|
| 2686 | break;
|
| 2687 | case SEEK_HOLE:
|
| 2688 | case SEEK_DATA:
|
| 2689 | inode_lock(inode);
|
| 2690 | retval = fuse_lseek(file, offset, whence);
|
| 2691 | inode_unlock(inode);
|
| 2692 | break;
|
| 2693 | default:
|
| 2694 | retval = -EINVAL;
|
| 2695 | }
|
| 2696 |
|
| 2697 | return retval;
|
| 2698 | }
|
| 2699 |
|
| 2700 | static void fuse_do_truncate(struct file *file)
|
| 2701 | {
|
| 2702 | struct inode *inode = file->f_mapping->host;
|
| 2703 | struct iattr attr;
|
| 2704 |
|
| 2705 | attr.ia_valid = ATTR_SIZE;
|
| 2706 | attr.ia_size = i_size_read(inode);
|
| 2707 |
|
| 2708 | attr.ia_file = file;
|
| 2709 | attr.ia_valid |= ATTR_FILE;
|
| 2710 |
|
| 2711 | fuse_do_setattr(file_mnt_idmap(file), file_dentry(file), &attr, file);
|
| 2712 | }
|
| 2713 |
|
| 2714 | static inline loff_t fuse_round_up(struct fuse_conn *fc, loff_t off)
|
| 2715 | {
|
| 2716 | return round_up(off, fc->max_pages << PAGE_SHIFT);
|
| 2717 | }
|
| 2718 |
|
| 2719 | static ssize_t
|
| 2720 | fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
|
| 2721 | {
|
| 2722 | DECLARE_COMPLETION_ONSTACK(wait);
|
| 2723 | ssize_t ret = 0;
|
| 2724 | struct file *file = iocb->ki_filp;
|
| 2725 | struct fuse_file *ff = file->private_data;
|
| 2726 | loff_t pos = 0;
|
| 2727 | struct inode *inode;
|
| 2728 | loff_t i_size;
|
| 2729 | size_t count = iov_iter_count(iter), shortened = 0;
|
| 2730 | loff_t offset = iocb->ki_pos;
|
| 2731 | struct fuse_io_priv *io;
|
| 2732 | bool async = ff->fm->fc->async_dio;
|
| 2733 |
|
| 2734 | pos = offset;
|
| 2735 | inode = file->f_mapping->host;
|
| 2736 | i_size = i_size_read(inode);
|
| 2737 |
|
| 2738 | if ((iov_iter_rw(iter) == READ) && (offset >= i_size))
|
| 2739 | return 0;
|
| 2740 |
|
| 2741 | if ((iov_iter_rw(iter) == WRITE) && async && !inode->i_sb->s_dio_done_wq) {
|
| 2742 | ret = sb_init_dio_done_wq(inode->i_sb);
|
| 2743 | if (ret < 0)
|
| 2744 | return ret;
|
| 2745 | }
|
| 2746 |
|
| 2747 | io = kmalloc_obj(struct fuse_io_priv);
|
| 2748 | if (!io)
|
| 2749 | return -ENOMEM;
|
| 2750 | spin_lock_init(&io->lock);
|
| 2751 | kref_init(&io->refcnt);
|
| 2752 | io->reqs = 1;
|
| 2753 | io->bytes = -1;
|
| 2754 | io->size = 0;
|
| 2755 | io->offset = offset;
|
| 2756 | io->write = (iov_iter_rw(iter) == WRITE);
|
| 2757 | io->err = 0;
|
| 2758 | |
| 2759 | * By default, we want to optimize all I/Os with async request
|
| 2760 | * submission to the client filesystem if supported.
|
| 2761 |
|
| 2762 | io->async = async;
|
| 2763 | io->iocb = iocb;
|
| 2764 | io->blocking = is_sync_kiocb(iocb);
|
| 2765 |
|
| 2766 |
|
| 2767 | if (io->async && !io->write && offset + count > i_size) {
|
| 2768 | iov_iter_truncate(iter, fuse_round_up(ff->fm->fc, i_size - offset));
|
| 2769 | shortened = count - iov_iter_count(iter);
|
| 2770 | count -= shortened;
|
| 2771 | }
|
| 2772 |
|
| 2773 | |
| 2774 | * We cannot asynchronously extend the size of a file.
|
| 2775 | * In such case the aio will behave exactly like sync io.
|
| 2776 |
|
| 2777 | if ((offset + count > i_size) && io->write)
|
| 2778 | io->blocking = true;
|
| 2779 |
|
| 2780 | if (io->async && io->blocking) {
|
| 2781 | |
| 2782 | * Additional reference to keep io around after
|
| 2783 | * calling fuse_aio_complete()
|
| 2784 |
|
| 2785 | kref_get(&io->refcnt);
|
| 2786 | io->done = &wait;
|
| 2787 | }
|
| 2788 |
|
| 2789 | if (iov_iter_rw(iter) == WRITE) {
|
| 2790 | ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE);
|
| 2791 | fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE);
|
| 2792 | } else {
|
| 2793 | ret = __fuse_direct_read(io, iter, &pos);
|
| 2794 | }
|
| 2795 | iov_iter_reexpand(iter, iov_iter_count(iter) + shortened);
|
| 2796 |
|
| 2797 | if (io->async) {
|
| 2798 | bool blocking = io->blocking;
|
| 2799 |
|
| 2800 | fuse_aio_complete(io, ret < 0 ? ret : 0, -1);
|
| 2801 |
|
| 2802 |
|
| 2803 | if (!blocking)
|
| 2804 | return -EIOCBQUEUED;
|
| 2805 |
|
| 2806 | wait_for_completion(&wait);
|
| 2807 | ret = fuse_get_res_by_io(io);
|
| 2808 | }
|
| 2809 |
|
| 2810 | kref_put(&io->refcnt, fuse_io_release);
|
| 2811 |
|
| 2812 | if (iov_iter_rw(iter) == WRITE) {
|
| 2813 | fuse_write_update_attr(inode, pos, ret);
|
| 2814 |
|
| 2815 | if (ret < 0 && offset + count > i_size)
|
| 2816 | fuse_do_truncate(file);
|
| 2817 | }
|
| 2818 |
|
| 2819 | return ret;
|
| 2820 | }
|
| 2821 |
|
| 2822 | static int fuse_writeback_range(struct inode *inode, loff_t start, loff_t end)
|
| 2823 | {
|
| 2824 | int err = filemap_write_and_wait_range(inode->i_mapping, start, LLONG_MAX);
|
| 2825 |
|
| 2826 | if (!err)
|
| 2827 | fuse_sync_writes(inode);
|
| 2828 |
|
| 2829 | return err;
|
| 2830 | }
|
| 2831 |
|
| 2832 | static long fuse_file_fallocate(struct file *file, int mode, loff_t offset,
|
| 2833 | loff_t length)
|
| 2834 | {
|
| 2835 | struct fuse_file *ff = file->private_data;
|
| 2836 | struct inode *inode = file_inode(file);
|
| 2837 | struct fuse_inode *fi = get_fuse_inode(inode);
|
| 2838 | struct fuse_mount *fm = ff->fm;
|
| 2839 | FUSE_ARGS(args);
|
| 2840 | struct fuse_fallocate_in inarg = {
|
| 2841 | .fh = ff->fh,
|
| 2842 | .offset = offset,
|
| 2843 | .length = length,
|
| 2844 | .mode = mode
|
| 2845 | };
|
| 2846 | int err;
|
| 2847 | bool block_faults = FUSE_IS_DAX(inode) &&
|
| 2848 | (!(mode & FALLOC_FL_KEEP_SIZE) ||
|
| 2849 | (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)));
|
| 2850 |
|
| 2851 | if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |
|
| 2852 | FALLOC_FL_ZERO_RANGE))
|
| 2853 | return -EOPNOTSUPP;
|
| 2854 |
|
| 2855 | if (fm->fc->no_fallocate)
|
| 2856 | return -EOPNOTSUPP;
|
| 2857 |
|
| 2858 | inode_lock(inode);
|
| 2859 | if (block_faults) {
|
| 2860 | filemap_invalidate_lock(inode->i_mapping);
|
| 2861 | err = fuse_dax_break_layouts(inode, 0, -1);
|
| 2862 | if (err)
|
| 2863 | goto out;
|
| 2864 | }
|
| 2865 |
|
| 2866 | if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)) {
|
| 2867 | loff_t endbyte = offset + length - 1;
|
| 2868 |
|
| 2869 | err = fuse_writeback_range(inode, offset, endbyte);
|
| 2870 | if (err)
|
| 2871 | goto out;
|
| 2872 | }
|
| 2873 |
|
| 2874 | if (!(mode & FALLOC_FL_KEEP_SIZE) &&
|
| 2875 | offset + length > i_size_read(inode)) {
|
| 2876 | err = inode_newsize_ok(inode, offset + length);
|
| 2877 | if (err)
|
| 2878 | goto out;
|
| 2879 | }
|
| 2880 |
|
| 2881 | err = file_modified(file);
|
| 2882 | if (err)
|
| 2883 | goto out;
|
| 2884 |
|
| 2885 | if (!(mode & FALLOC_FL_KEEP_SIZE))
|
| 2886 | set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
|
| 2887 |
|
| 2888 | args.opcode = FUSE_FALLOCATE;
|
| 2889 | args.nodeid = ff->nodeid;
|
| 2890 | args.in_numargs = 1;
|
| 2891 | args.in_args[0].size = sizeof(inarg);
|
| 2892 | args.in_args[0].value = &inarg;
|
| 2893 | err = fuse_simple_request(fm, &args);
|
| 2894 | if (err == -ENOSYS) {
|
| 2895 | fm->fc->no_fallocate = 1;
|
| 2896 | err = -EOPNOTSUPP;
|
| 2897 | }
|
| 2898 | if (err)
|
| 2899 | goto out;
|
| 2900 |
|
| 2901 |
|
| 2902 | if (!(mode & FALLOC_FL_KEEP_SIZE)) {
|
| 2903 | loff_t oldsize = i_size_read(inode);
|
| 2904 |
|
| 2905 | if (offset + length > oldsize)
|
| 2906 | truncate_pagecache_range(inode, oldsize,
|
| 2907 | offset + length - 1);
|
| 2908 | if (fuse_write_update_attr(inode, offset + length, length))
|
| 2909 | file_update_time(file);
|
| 2910 | }
|
| 2911 |
|
| 2912 | if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE))
|
| 2913 | truncate_pagecache_range(inode, offset, offset + length - 1);
|
| 2914 |
|
| 2915 | fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE);
|
| 2916 |
|
| 2917 | out:
|
| 2918 | if (!(mode & FALLOC_FL_KEEP_SIZE))
|
| 2919 | clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
|
| 2920 |
|
| 2921 | if (block_faults)
|
| 2922 | filemap_invalidate_unlock(inode->i_mapping);
|
| 2923 |
|
| 2924 | inode_unlock(inode);
|
| 2925 |
|
| 2926 | fuse_flush_time_update(inode);
|
| 2927 |
|
| 2928 | return err;
|
| 2929 | }
|
| 2930 |
|
| 2931 | static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in,
|
| 2932 | struct file *file_out, loff_t pos_out,
|
| 2933 | size_t len, unsigned int flags)
|
| 2934 | {
|
| 2935 | struct fuse_file *ff_in = file_in->private_data;
|
| 2936 | struct fuse_file *ff_out = file_out->private_data;
|
| 2937 | struct inode *inode_in = file_inode(file_in);
|
| 2938 | struct inode *inode_out = file_inode(file_out);
|
| 2939 | struct fuse_inode *fi_out = get_fuse_inode(inode_out);
|
| 2940 | struct fuse_mount *fm = ff_in->fm;
|
| 2941 | struct fuse_conn *fc = fm->fc;
|
| 2942 | FUSE_ARGS(args);
|
| 2943 | struct fuse_copy_file_range_in inarg = {
|
| 2944 | .fh_in = ff_in->fh,
|
| 2945 | .off_in = pos_in,
|
| 2946 | .nodeid_out = ff_out->nodeid,
|
| 2947 | .fh_out = ff_out->fh,
|
| 2948 | .off_out = pos_out,
|
| 2949 | .len = len,
|
| 2950 | .flags = flags
|
| 2951 | };
|
| 2952 | struct fuse_write_out outarg;
|
| 2953 | struct fuse_copy_file_range_out outarg_64;
|
| 2954 | u64 bytes_copied;
|
| 2955 | ssize_t err;
|
| 2956 | |
| 2957 | * extended
|
| 2958 | bool is_unstable = (!fc->writeback_cache) &&
|
| 2959 | ((pos_out + len) > inode_out->i_size);
|
| 2960 |
|
| 2961 | if (fc->no_copy_file_range)
|
| 2962 | return -EOPNOTSUPP;
|
| 2963 |
|
| 2964 | if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb)
|
| 2965 | return -EXDEV;
|
| 2966 |
|
| 2967 | inode_lock(inode_in);
|
| 2968 | err = fuse_writeback_range(inode_in, pos_in, pos_in + len - 1);
|
| 2969 | inode_unlock(inode_in);
|
| 2970 | if (err)
|
| 2971 | return err;
|
| 2972 |
|
| 2973 | inode_lock(inode_out);
|
| 2974 |
|
| 2975 | err = file_modified(file_out);
|
| 2976 | if (err)
|
| 2977 | goto out;
|
| 2978 |
|
| 2979 | |
| 2980 | * Write out dirty pages in the destination file before sending the COPY
|
| 2981 | * request to userspace. After the request is completed, truncate off
|
| 2982 | * pages (including partial ones) from the cache that have been copied,
|
| 2983 | * since these contain stale data at that point.
|
| 2984 | *
|
| 2985 | * This should be mostly correct, but if the COPY writes to partial
|
| 2986 | * pages (at the start or end) and the parts not covered by the COPY are
|
| 2987 | * written through a memory map after calling fuse_writeback_range(),
|
| 2988 | * then these partial page modifications will be lost on truncation.
|
| 2989 | *
|
| 2990 | * It is unlikely that someone would rely on such mixed style
|
| 2991 | * modifications. Yet this does give less guarantees than if the
|
| 2992 | * copying was performed with write(2).
|
| 2993 | *
|
| 2994 | * To fix this a mapping->invalidate_lock could be used to prevent new
|
| 2995 | * faults while the copy is ongoing.
|
| 2996 |
|
| 2997 | err = fuse_writeback_range(inode_out, pos_out, pos_out + len - 1);
|
| 2998 | if (err)
|
| 2999 | goto out;
|
| 3000 |
|
| 3001 | if (is_unstable)
|
| 3002 | set_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state);
|
| 3003 |
|
| 3004 | args.opcode = FUSE_COPY_FILE_RANGE_64;
|
| 3005 | args.nodeid = ff_in->nodeid;
|
| 3006 | args.in_numargs = 1;
|
| 3007 | args.in_args[0].size = sizeof(inarg);
|
| 3008 | args.in_args[0].value = &inarg;
|
| 3009 | args.out_numargs = 1;
|
| 3010 | args.out_args[0].size = sizeof(outarg_64);
|
| 3011 | args.out_args[0].value = &outarg_64;
|
| 3012 | if (fc->no_copy_file_range_64) {
|
| 3013 | fallback:
|
| 3014 |
|
| 3015 | args.opcode = FUSE_COPY_FILE_RANGE;
|
| 3016 | args.out_args[0].size = sizeof(outarg);
|
| 3017 | args.out_args[0].value = &outarg;
|
| 3018 | inarg.len = len = min_t(size_t, len, UINT_MAX & PAGE_MASK);
|
| 3019 | }
|
| 3020 | err = fuse_simple_request(fm, &args);
|
| 3021 | if (err == -ENOSYS) {
|
| 3022 | if (fc->no_copy_file_range_64) {
|
| 3023 | fc->no_copy_file_range = 1;
|
| 3024 | err = -EOPNOTSUPP;
|
| 3025 | } else {
|
| 3026 | fc->no_copy_file_range_64 = 1;
|
| 3027 | goto fallback;
|
| 3028 | }
|
| 3029 | }
|
| 3030 | if (err)
|
| 3031 | goto out;
|
| 3032 |
|
| 3033 | bytes_copied = fc->no_copy_file_range_64 ?
|
| 3034 | outarg.size : outarg_64.bytes_copied;
|
| 3035 |
|
| 3036 | if (bytes_copied > len) {
|
| 3037 | err = -EIO;
|
| 3038 | goto out;
|
| 3039 | }
|
| 3040 |
|
| 3041 | truncate_inode_pages_range(inode_out->i_mapping,
|
| 3042 | ALIGN_DOWN(pos_out, PAGE_SIZE),
|
| 3043 | ALIGN(pos_out + bytes_copied, PAGE_SIZE) - 1);
|
| 3044 |
|
| 3045 | file_update_time(file_out);
|
| 3046 | fuse_write_update_attr(inode_out, pos_out + bytes_copied, bytes_copied);
|
| 3047 |
|
| 3048 | err = bytes_copied;
|
| 3049 | out:
|
| 3050 | if (is_unstable)
|
| 3051 | clear_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state);
|
| 3052 |
|
| 3053 | inode_unlock(inode_out);
|
| 3054 | file_accessed(file_in);
|
| 3055 |
|
| 3056 | fuse_flush_time_update(inode_out);
|
| 3057 |
|
| 3058 | return err;
|
| 3059 | }
|
| 3060 |
|
| 3061 | static ssize_t fuse_copy_file_range(struct file *src_file, loff_t src_off,
|
| 3062 | struct file *dst_file, loff_t dst_off,
|
| 3063 | size_t len, unsigned int flags)
|
| 3064 | {
|
| 3065 | ssize_t ret;
|
| 3066 |
|
| 3067 | ret = __fuse_copy_file_range(src_file, src_off, dst_file, dst_off,
|
| 3068 | len, flags);
|
| 3069 |
|
| 3070 | if (ret == -EOPNOTSUPP || ret == -EXDEV)
|
| 3071 | ret = splice_copy_file_range(src_file, src_off, dst_file,
|
| 3072 | dst_off, len);
|
| 3073 | return ret;
|
| 3074 | }
|
| 3075 |
|
| 3076 | static const struct file_operations fuse_file_operations = {
|
| 3077 | .llseek = fuse_file_llseek,
|
| 3078 | .read_iter = fuse_file_read_iter,
|
| 3079 | .write_iter = fuse_file_write_iter,
|
| 3080 | .mmap = fuse_file_mmap,
|
| 3081 | .open = fuse_open,
|
| 3082 | .flush = fuse_flush,
|
| 3083 | .release = fuse_release,
|
| 3084 | .fsync = fuse_fsync,
|
| 3085 | .lock = fuse_file_lock,
|
| 3086 | .get_unmapped_area = thp_get_unmapped_area,
|
| 3087 | .flock = fuse_file_flock,
|
| 3088 | .splice_read = fuse_splice_read,
|
| 3089 | .splice_write = fuse_splice_write,
|
| 3090 | .unlocked_ioctl = fuse_file_ioctl,
|
| 3091 | .compat_ioctl = fuse_file_compat_ioctl,
|
| 3092 | .poll = fuse_file_poll,
|
| 3093 | .fallocate = fuse_file_fallocate,
|
| 3094 | .copy_file_range = fuse_copy_file_range,
|
| 3095 | .setlease = generic_setlease,
|
| 3096 | };
|
| 3097 |
|
| 3098 | static const struct address_space_operations fuse_file_aops = {
|
| 3099 | .read_folio = fuse_read_folio,
|
| 3100 | .readahead = fuse_readahead,
|
| 3101 | .writepages = fuse_writepages,
|
| 3102 | .launder_folio = fuse_launder_folio,
|
| 3103 | .dirty_folio = iomap_dirty_folio,
|
| 3104 | .release_folio = iomap_release_folio,
|
| 3105 | .invalidate_folio = iomap_invalidate_folio,
|
| 3106 | .is_partially_uptodate = iomap_is_partially_uptodate,
|
| 3107 | .migrate_folio = filemap_migrate_folio,
|
| 3108 | .bmap = fuse_bmap,
|
| 3109 | .direct_IO = fuse_direct_IO,
|
| 3110 | };
|
| 3111 |
|
| 3112 | void fuse_init_file_inode(struct inode *inode, unsigned int flags)
|
| 3113 | {
|
| 3114 | struct fuse_inode *fi = get_fuse_inode(inode);
|
| 3115 | struct fuse_conn *fc = get_fuse_conn(inode);
|
| 3116 |
|
| 3117 | inode->i_fop = &fuse_file_operations;
|
| 3118 | inode->i_data.a_ops = &fuse_file_aops;
|
| 3119 | if (fc->writeback_cache)
|
| 3120 | mapping_set_writeback_may_deadlock_on_reclaim(&inode->i_data);
|
| 3121 |
|
| 3122 | INIT_LIST_HEAD(&fi->write_files);
|
| 3123 | INIT_LIST_HEAD(&fi->queued_writes);
|
| 3124 | fi->writectr = 0;
|
| 3125 | fi->iocachectr = 0;
|
| 3126 | init_waitqueue_head(&fi->page_waitq);
|
| 3127 | init_waitqueue_head(&fi->direct_io_waitq);
|
| 3128 |
|
| 3129 | if (IS_ENABLED(CONFIG_FUSE_DAX))
|
| 3130 | fuse_dax_inode_init(inode, flags);
|
| 3131 | }
|