| 1 | |
| 2 | #include <linux/ceph/ceph_debug.h>
|
| 3 | #include <linux/ceph/striper.h>
|
| 4 |
|
| 5 | #include <linux/module.h>
|
| 6 | #include <linux/sched.h>
|
| 7 | #include <linux/slab.h>
|
| 8 | #include <linux/file.h>
|
| 9 | #include <linux/mount.h>
|
| 10 | #include <linux/namei.h>
|
| 11 | #include <linux/writeback.h>
|
| 12 | #include <linux/falloc.h>
|
| 13 | #include <linux/iversion.h>
|
| 14 | #include <linux/ktime.h>
|
| 15 | #include <linux/splice.h>
|
| 16 |
|
| 17 | #include "super.h"
|
| 18 | #include "mds_client.h"
|
| 19 | #include "cache.h"
|
| 20 | #include "io.h"
|
| 21 | #include "metric.h"
|
| 22 | #include "subvolume_metrics.h"
|
| 23 |
|
| 24 | |
| 25 | * Record I/O for subvolume metrics tracking.
|
| 26 | *
|
| 27 | * Callers must ensure bytes > 0 for reads (ret > 0 check) to avoid counting
|
| 28 | * EOF as an I/O operation. For writes, the condition is (ret >= 0 && len > 0).
|
| 29 |
|
| 30 | static inline void ceph_record_subvolume_io(struct inode *inode, bool is_write,
|
| 31 | ktime_t start, ktime_t end,
|
| 32 | size_t bytes)
|
| 33 | {
|
| 34 | if (!bytes)
|
| 35 | return;
|
| 36 |
|
| 37 | ceph_subvolume_metrics_record_io(ceph_sb_to_mdsc(inode->i_sb),
|
| 38 | ceph_inode(inode),
|
| 39 | is_write, bytes, start, end);
|
| 40 | }
|
| 41 |
|
| 42 | static __le32 ceph_flags_sys2wire(struct ceph_mds_client *mdsc, u32 flags)
|
| 43 | {
|
| 44 | struct ceph_client *cl = mdsc->fsc->client;
|
| 45 | u32 wire_flags = 0;
|
| 46 |
|
| 47 | switch (flags & O_ACCMODE) {
|
| 48 | case O_RDONLY:
|
| 49 | wire_flags |= CEPH_O_RDONLY;
|
| 50 | break;
|
| 51 | case O_WRONLY:
|
| 52 | wire_flags |= CEPH_O_WRONLY;
|
| 53 | break;
|
| 54 | case O_RDWR:
|
| 55 | wire_flags |= CEPH_O_RDWR;
|
| 56 | break;
|
| 57 | }
|
| 58 |
|
| 59 | flags &= ~O_ACCMODE;
|
| 60 |
|
| 61 | #define ceph_sys2wire(a) if (flags & a) { wire_flags |= CEPH_##a; flags &= ~a; }
|
| 62 |
|
| 63 | ceph_sys2wire(O_CREAT);
|
| 64 | ceph_sys2wire(O_EXCL);
|
| 65 | ceph_sys2wire(O_TRUNC);
|
| 66 | ceph_sys2wire(O_DIRECTORY);
|
| 67 | ceph_sys2wire(O_NOFOLLOW);
|
| 68 |
|
| 69 | #undef ceph_sys2wire
|
| 70 |
|
| 71 | if (flags)
|
| 72 | doutc(cl, "unused open flags: %x\n", flags);
|
| 73 |
|
| 74 | return cpu_to_le32(wire_flags);
|
| 75 | }
|
| 76 |
|
| 77 | |
| 78 | * Ceph file operations
|
| 79 | *
|
| 80 | * Implement basic open/close functionality, and implement
|
| 81 | * read/write.
|
| 82 | *
|
| 83 | * We implement three modes of file I/O:
|
| 84 | * - buffered uses the generic_file_aio_{read,write} helpers
|
| 85 | *
|
| 86 | * - synchronous is used when there is multi-client read/write
|
| 87 | * sharing, avoids the page cache, and synchronously waits for an
|
| 88 | * ack from the OSD.
|
| 89 | *
|
| 90 | * - direct io takes the variant of the sync path that references
|
| 91 | * user pages directly.
|
| 92 | *
|
| 93 | * fsync() flushes and waits on dirty pages, but just queues metadata
|
| 94 | * for writeback: since the MDS can recover size and mtime there is no
|
| 95 | * need to wait for MDS acknowledgement.
|
| 96 |
|
| 97 |
|
| 98 | |
| 99 | * How many pages to get in one call to iov_iter_get_pages(). This
|
| 100 | * determines the size of the on-stack array used as a buffer.
|
| 101 |
|
| 102 | #define ITER_GET_BVECS_PAGES 64
|
| 103 |
|
| 104 | static ssize_t __iter_get_bvecs(struct iov_iter *iter, size_t maxsize,
|
| 105 | struct bio_vec *bvecs)
|
| 106 | {
|
| 107 | size_t size = 0;
|
| 108 | int bvec_idx = 0;
|
| 109 |
|
| 110 | if (maxsize > iov_iter_count(iter))
|
| 111 | maxsize = iov_iter_count(iter);
|
| 112 |
|
| 113 | while (size < maxsize) {
|
| 114 | struct page *pages[ITER_GET_BVECS_PAGES];
|
| 115 | ssize_t bytes;
|
| 116 | size_t start;
|
| 117 | int idx = 0;
|
| 118 |
|
| 119 | bytes = iov_iter_get_pages2(iter, pages, maxsize - size,
|
| 120 | ITER_GET_BVECS_PAGES, &start);
|
| 121 | if (bytes < 0)
|
| 122 | return size ?: bytes;
|
| 123 |
|
| 124 | size += bytes;
|
| 125 |
|
| 126 | for ( ; bytes; idx++, bvec_idx++) {
|
| 127 | int len = min_t(int, bytes, PAGE_SIZE - start);
|
| 128 |
|
| 129 | bvec_set_page(&bvecs[bvec_idx], pages[idx], len, start);
|
| 130 | bytes -= len;
|
| 131 | start = 0;
|
| 132 | }
|
| 133 | }
|
| 134 |
|
| 135 | return size;
|
| 136 | }
|
| 137 |
|
| 138 | |
| 139 | * iov_iter_get_pages() only considers one iov_iter segment, no matter
|
| 140 | * what maxsize or maxpages are given. For ITER_BVEC that is a single
|
| 141 | * page.
|
| 142 | *
|
| 143 | * Attempt to get up to @maxsize bytes worth of pages from @iter.
|
| 144 | * Return the number of bytes in the created bio_vec array, or an error.
|
| 145 |
|
| 146 | static ssize_t iter_get_bvecs_alloc(struct iov_iter *iter, size_t maxsize,
|
| 147 | struct bio_vec **bvecs, int *num_bvecs)
|
| 148 | {
|
| 149 | struct bio_vec *bv;
|
| 150 | size_t orig_count = iov_iter_count(iter);
|
| 151 | ssize_t bytes;
|
| 152 | int npages;
|
| 153 |
|
| 154 | iov_iter_truncate(iter, maxsize);
|
| 155 | npages = iov_iter_npages(iter, INT_MAX);
|
| 156 | iov_iter_reexpand(iter, orig_count);
|
| 157 |
|
| 158 | |
| 159 | * __iter_get_bvecs() may populate only part of the array -- zero it
|
| 160 | * out.
|
| 161 |
|
| 162 | bv = kvmalloc_objs(*bv, npages, GFP_KERNEL | __GFP_ZERO);
|
| 163 | if (!bv)
|
| 164 | return -ENOMEM;
|
| 165 |
|
| 166 | bytes = __iter_get_bvecs(iter, maxsize, bv);
|
| 167 | if (bytes < 0) {
|
| 168 | |
| 169 | * No pages were pinned -- just free the array.
|
| 170 |
|
| 171 | kvfree(bv);
|
| 172 | return bytes;
|
| 173 | }
|
| 174 |
|
| 175 | *bvecs = bv;
|
| 176 | *num_bvecs = npages;
|
| 177 | return bytes;
|
| 178 | }
|
| 179 |
|
| 180 | static void put_bvecs(struct bio_vec *bvecs, int num_bvecs, bool should_dirty)
|
| 181 | {
|
| 182 | int i;
|
| 183 |
|
| 184 | for (i = 0; i < num_bvecs; i++) {
|
| 185 | if (bvecs[i].bv_page) {
|
| 186 | if (should_dirty)
|
| 187 | set_page_dirty_lock(bvecs[i].bv_page);
|
| 188 | put_page(bvecs[i].bv_page);
|
| 189 | }
|
| 190 | }
|
| 191 | kvfree(bvecs);
|
| 192 | }
|
| 193 |
|
| 194 | |
| 195 | * Prepare an open request. Preallocate ceph_cap to avoid an
|
| 196 | * inopportune ENOMEM later.
|
| 197 |
|
| 198 | static struct ceph_mds_request *
|
| 199 | prepare_open_request(struct super_block *sb, int flags, int create_mode)
|
| 200 | {
|
| 201 | struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(sb);
|
| 202 | struct ceph_mds_request *req;
|
| 203 | int want_auth = USE_ANY_MDS;
|
| 204 | int op = (flags & O_CREAT) ? CEPH_MDS_OP_CREATE : CEPH_MDS_OP_OPEN;
|
| 205 |
|
| 206 | if (flags & (O_WRONLY|O_RDWR|O_CREAT|O_TRUNC))
|
| 207 | want_auth = USE_AUTH_MDS;
|
| 208 |
|
| 209 | req = ceph_mdsc_create_request(mdsc, op, want_auth);
|
| 210 | if (IS_ERR(req))
|
| 211 | goto out;
|
| 212 | req->r_fmode = ceph_flags_to_mode(flags);
|
| 213 | req->r_args.open.flags = ceph_flags_sys2wire(mdsc, flags);
|
| 214 | req->r_args.open.mode = cpu_to_le32(create_mode);
|
| 215 | out:
|
| 216 | return req;
|
| 217 | }
|
| 218 |
|
| 219 | static int ceph_init_file_info(struct inode *inode, struct file *file,
|
| 220 | int fmode, bool isdir)
|
| 221 | {
|
| 222 | struct ceph_inode_info *ci = ceph_inode(inode);
|
| 223 | struct ceph_mount_options *opt =
|
| 224 | ceph_inode_to_fs_client(&ci->netfs.inode)->mount_options;
|
| 225 | struct ceph_client *cl = ceph_inode_to_client(inode);
|
| 226 | struct ceph_file_info *fi;
|
| 227 | int ret;
|
| 228 |
|
| 229 | doutc(cl, "%p %llx.%llx %p 0%o (%s)\n", inode, ceph_vinop(inode),
|
| 230 | file, inode->i_mode, isdir ? "dir" : "regular");
|
| 231 | BUG_ON(inode->i_fop->release != ceph_release);
|
| 232 |
|
| 233 | if (isdir) {
|
| 234 | struct ceph_dir_file_info *dfi =
|
| 235 | kmem_cache_zalloc(ceph_dir_file_cachep, GFP_KERNEL);
|
| 236 | if (!dfi)
|
| 237 | return -ENOMEM;
|
| 238 |
|
| 239 | file->private_data = dfi;
|
| 240 | fi = &dfi->file_info;
|
| 241 | dfi->next_offset = 2;
|
| 242 | dfi->readdir_cache_idx = -1;
|
| 243 | } else {
|
| 244 | fi = kmem_cache_zalloc(ceph_file_cachep, GFP_KERNEL);
|
| 245 | if (!fi)
|
| 246 | return -ENOMEM;
|
| 247 |
|
| 248 | if (opt->flags & CEPH_MOUNT_OPT_NOPAGECACHE)
|
| 249 | fi->flags |= CEPH_F_SYNC;
|
| 250 |
|
| 251 | file->private_data = fi;
|
| 252 | }
|
| 253 |
|
| 254 | ceph_get_fmode(ci, fmode, 1);
|
| 255 | fi->fmode = fmode;
|
| 256 |
|
| 257 | spin_lock_init(&fi->rw_contexts_lock);
|
| 258 | INIT_LIST_HEAD(&fi->rw_contexts);
|
| 259 | fi->filp_gen = READ_ONCE(ceph_inode_to_fs_client(inode)->filp_gen);
|
| 260 |
|
| 261 | if ((file->f_mode & FMODE_WRITE) && ceph_has_inline_data(ci)) {
|
| 262 | ret = ceph_uninline_data(file);
|
| 263 | if (ret < 0)
|
| 264 | goto error;
|
| 265 | }
|
| 266 |
|
| 267 | return 0;
|
| 268 |
|
| 269 | error:
|
| 270 | ceph_fscache_unuse_cookie(inode, file->f_mode & FMODE_WRITE);
|
| 271 | ceph_put_fmode(ci, fi->fmode, 1);
|
| 272 | kmem_cache_free(ceph_file_cachep, fi);
|
| 273 |
|
| 274 | wake_up_all(&ci->i_cap_wq);
|
| 275 | return ret;
|
| 276 | }
|
| 277 |
|
| 278 | |
| 279 | * initialize private struct file data.
|
| 280 | * if we fail, clean up by dropping fmode reference on the ceph_inode
|
| 281 |
|
| 282 | static int ceph_init_file(struct inode *inode, struct file *file, int fmode)
|
| 283 | {
|
| 284 | struct ceph_client *cl = ceph_inode_to_client(inode);
|
| 285 | int ret = 0;
|
| 286 |
|
| 287 | switch (inode->i_mode & S_IFMT) {
|
| 288 | case S_IFREG:
|
| 289 | ceph_fscache_use_cookie(inode, file->f_mode & FMODE_WRITE);
|
| 290 | fallthrough;
|
| 291 | case S_IFDIR:
|
| 292 | ret = ceph_init_file_info(inode, file, fmode,
|
| 293 | S_ISDIR(inode->i_mode));
|
| 294 | break;
|
| 295 |
|
| 296 | case S_IFLNK:
|
| 297 | doutc(cl, "%p %llx.%llx %p 0%o (symlink)\n", inode,
|
| 298 | ceph_vinop(inode), file, inode->i_mode);
|
| 299 | break;
|
| 300 |
|
| 301 | default:
|
| 302 | doutc(cl, "%p %llx.%llx %p 0%o (special)\n", inode,
|
| 303 | ceph_vinop(inode), file, inode->i_mode);
|
| 304 | |
| 305 | * we need to drop the open ref now, since we don't
|
| 306 | * have .release set to ceph_release.
|
| 307 |
|
| 308 | BUG_ON(inode->i_fop->release == ceph_release);
|
| 309 |
|
| 310 |
|
| 311 | ret = inode->i_fop->open(inode, file);
|
| 312 | }
|
| 313 | return ret;
|
| 314 | }
|
| 315 |
|
| 316 | |
| 317 | * Retry cap acquisition after a stale session or a lost cap update.
|
| 318 |
|
| 319 | int ceph_renew_caps(struct inode *inode, int fmode)
|
| 320 | {
|
| 321 | struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb);
|
| 322 | struct ceph_client *cl = mdsc->fsc->client;
|
| 323 | struct ceph_inode_info *ci = ceph_inode(inode);
|
| 324 | struct ceph_mds_request *req;
|
| 325 | int err, flags, wanted, issued;
|
| 326 |
|
| 327 | spin_lock(&ci->i_ceph_lock);
|
| 328 | __ceph_touch_fmode(ci, mdsc, fmode);
|
| 329 | wanted = __ceph_caps_file_wanted(ci);
|
| 330 | issued = __ceph_caps_issued(ci, NULL);
|
| 331 | if (__ceph_is_any_real_caps(ci) &&
|
| 332 | (!(wanted & CEPH_CAP_ANY_WR) || ci->i_auth_cap) &&
|
| 333 | (issued & wanted) == wanted) {
|
| 334 | spin_unlock(&ci->i_ceph_lock);
|
| 335 | doutc(cl, "%p %llx.%llx want %s issued %s updating mds_wanted\n",
|
| 336 | inode, ceph_vinop(inode), ceph_cap_string(wanted),
|
| 337 | ceph_cap_string(issued));
|
| 338 | ceph_check_caps(ci, 0);
|
| 339 | return 0;
|
| 340 | }
|
| 341 | spin_unlock(&ci->i_ceph_lock);
|
| 342 |
|
| 343 | flags = 0;
|
| 344 | if ((wanted & CEPH_CAP_FILE_RD) && (wanted & CEPH_CAP_FILE_WR))
|
| 345 | flags = O_RDWR;
|
| 346 | else if (wanted & CEPH_CAP_FILE_RD)
|
| 347 | flags = O_RDONLY;
|
| 348 | else if (wanted & CEPH_CAP_FILE_WR)
|
| 349 | flags = O_WRONLY;
|
| 350 | #ifdef O_LAZY
|
| 351 | if (wanted & CEPH_CAP_FILE_LAZYIO)
|
| 352 | flags |= O_LAZY;
|
| 353 | #endif
|
| 354 |
|
| 355 | req = prepare_open_request(inode->i_sb, flags, 0);
|
| 356 | if (IS_ERR(req)) {
|
| 357 | err = PTR_ERR(req);
|
| 358 | goto out;
|
| 359 | }
|
| 360 |
|
| 361 | req->r_inode = inode;
|
| 362 | ihold(inode);
|
| 363 | req->r_num_caps = 1;
|
| 364 |
|
| 365 | err = ceph_mdsc_do_request(mdsc, NULL, req);
|
| 366 | ceph_mdsc_put_request(req);
|
| 367 | out:
|
| 368 | doutc(cl, "%p %llx.%llx open result=%d\n", inode, ceph_vinop(inode),
|
| 369 | err);
|
| 370 | return err < 0 ? err : 0;
|
| 371 | }
|
| 372 |
|
| 373 | |
| 374 | * If we already have the requisite capabilities, we can satisfy
|
| 375 | * the open request locally (no need to request new caps from the
|
| 376 | * MDS). We do, however, need to inform the MDS (asynchronously)
|
| 377 | * if our wanted caps set expands.
|
| 378 |
|
| 379 | int ceph_open(struct inode *inode, struct file *file)
|
| 380 | {
|
| 381 | struct ceph_inode_info *ci = ceph_inode(inode);
|
| 382 | struct ceph_fs_client *fsc = ceph_sb_to_fs_client(inode->i_sb);
|
| 383 | struct ceph_client *cl = fsc->client;
|
| 384 | struct ceph_mds_client *mdsc = fsc->mdsc;
|
| 385 | struct ceph_mds_request *req;
|
| 386 | struct ceph_file_info *fi = file->private_data;
|
| 387 | int err;
|
| 388 | int flags, fmode, wanted;
|
| 389 | struct dentry *dentry;
|
| 390 | char *path;
|
| 391 | bool do_sync = false;
|
| 392 | int mask = MAY_READ;
|
| 393 |
|
| 394 | if (fi) {
|
| 395 | doutc(cl, "file %p is already opened\n", file);
|
| 396 | return 0;
|
| 397 | }
|
| 398 |
|
| 399 |
|
| 400 | flags = file->f_flags & ~(O_CREAT|O_EXCL);
|
| 401 | if (S_ISDIR(inode->i_mode)) {
|
| 402 | flags = O_DIRECTORY;
|
| 403 | } else if (S_ISREG(inode->i_mode)) {
|
| 404 | err = fscrypt_file_open(inode, file);
|
| 405 | if (err)
|
| 406 | return err;
|
| 407 | }
|
| 408 |
|
| 409 | doutc(cl, "%p %llx.%llx file %p flags %d (%d)\n", inode,
|
| 410 | ceph_vinop(inode), file, flags, file->f_flags);
|
| 411 | fmode = ceph_flags_to_mode(flags);
|
| 412 | wanted = ceph_caps_for_mode(fmode);
|
| 413 |
|
| 414 | if (fmode & CEPH_FILE_MODE_WR)
|
| 415 | mask |= MAY_WRITE;
|
| 416 | dentry = d_find_alias(inode);
|
| 417 | if (!dentry) {
|
| 418 | do_sync = true;
|
| 419 | } else {
|
| 420 | struct ceph_path_info path_info = {0};
|
| 421 | path = ceph_mdsc_build_path(mdsc, dentry, &path_info, 0);
|
| 422 | if (IS_ERR(path)) {
|
| 423 | do_sync = true;
|
| 424 | err = 0;
|
| 425 | } else {
|
| 426 | err = ceph_mds_check_access(mdsc, path, mask);
|
| 427 | }
|
| 428 | ceph_mdsc_free_path_info(&path_info);
|
| 429 | dput(dentry);
|
| 430 |
|
| 431 |
|
| 432 | if (err == -EACCES) {
|
| 433 | return err;
|
| 434 | } else if (err < 0) {
|
| 435 | do_sync = true;
|
| 436 | err = 0;
|
| 437 | }
|
| 438 | }
|
| 439 |
|
| 440 |
|
| 441 | if (ceph_snap(inode) != CEPH_NOSNAP && (file->f_mode & FMODE_WRITE))
|
| 442 | return -EROFS;
|
| 443 |
|
| 444 |
|
| 445 | if (ceph_snap(inode) == CEPH_SNAPDIR) {
|
| 446 | return ceph_init_file(inode, file, fmode);
|
| 447 | }
|
| 448 |
|
| 449 | |
| 450 | * No need to block if we have caps on the auth MDS (for
|
| 451 | * write) or any MDS (for read). Update wanted set
|
| 452 | * asynchronously.
|
| 453 |
|
| 454 | spin_lock(&ci->i_ceph_lock);
|
| 455 | if (!do_sync && __ceph_is_any_real_caps(ci) &&
|
| 456 | (((fmode & CEPH_FILE_MODE_WR) == 0) || ci->i_auth_cap)) {
|
| 457 | int mds_wanted = __ceph_caps_mds_wanted(ci, true);
|
| 458 | int issued = __ceph_caps_issued(ci, NULL);
|
| 459 |
|
| 460 | doutc(cl, "open %p fmode %d want %s issued %s using existing\n",
|
| 461 | inode, fmode, ceph_cap_string(wanted),
|
| 462 | ceph_cap_string(issued));
|
| 463 | __ceph_touch_fmode(ci, mdsc, fmode);
|
| 464 | spin_unlock(&ci->i_ceph_lock);
|
| 465 |
|
| 466 |
|
| 467 | if ((issued & wanted) != wanted &&
|
| 468 | (mds_wanted & wanted) != wanted &&
|
| 469 | ceph_snap(inode) != CEPH_SNAPDIR)
|
| 470 | ceph_check_caps(ci, 0);
|
| 471 |
|
| 472 | return ceph_init_file(inode, file, fmode);
|
| 473 | } else if (!do_sync && ceph_snap(inode) != CEPH_NOSNAP &&
|
| 474 | (ci->i_snap_caps & wanted) == wanted) {
|
| 475 | __ceph_touch_fmode(ci, mdsc, fmode);
|
| 476 | spin_unlock(&ci->i_ceph_lock);
|
| 477 | return ceph_init_file(inode, file, fmode);
|
| 478 | }
|
| 479 |
|
| 480 | spin_unlock(&ci->i_ceph_lock);
|
| 481 |
|
| 482 | doutc(cl, "open fmode %d wants %s\n", fmode, ceph_cap_string(wanted));
|
| 483 | req = prepare_open_request(inode->i_sb, flags, 0);
|
| 484 | if (IS_ERR(req)) {
|
| 485 | err = PTR_ERR(req);
|
| 486 | goto out;
|
| 487 | }
|
| 488 | req->r_inode = inode;
|
| 489 | ihold(inode);
|
| 490 |
|
| 491 | req->r_num_caps = 1;
|
| 492 | err = ceph_mdsc_do_request(mdsc, NULL, req);
|
| 493 | if (!err)
|
| 494 | err = ceph_init_file(inode, file, req->r_fmode);
|
| 495 | ceph_mdsc_put_request(req);
|
| 496 | doutc(cl, "open result=%d on %llx.%llx\n", err, ceph_vinop(inode));
|
| 497 | out:
|
| 498 | return err;
|
| 499 | }
|
| 500 |
|
| 501 |
|
| 502 | static void
|
| 503 | cache_file_layout(struct inode *dst, struct inode *src)
|
| 504 | {
|
| 505 | struct ceph_inode_info *cdst = ceph_inode(dst);
|
| 506 | struct ceph_inode_info *csrc = ceph_inode(src);
|
| 507 |
|
| 508 | spin_lock(&cdst->i_ceph_lock);
|
| 509 | if ((__ceph_caps_issued(cdst, NULL) & CEPH_CAP_DIR_CREATE) &&
|
| 510 | !ceph_file_layout_is_valid(&cdst->i_cached_layout)) {
|
| 511 | memcpy(&cdst->i_cached_layout, &csrc->i_layout,
|
| 512 | sizeof(cdst->i_cached_layout));
|
| 513 | rcu_assign_pointer(cdst->i_cached_layout.pool_ns,
|
| 514 | ceph_try_get_string(csrc->i_layout.pool_ns));
|
| 515 | }
|
| 516 | spin_unlock(&cdst->i_ceph_lock);
|
| 517 | }
|
| 518 |
|
| 519 | |
| 520 | * Try to set up an async create. We need caps, a file layout, and inode number,
|
| 521 | * and either a lease on the dentry or complete dir info. If any of those
|
| 522 | * criteria are not satisfied, then return false and the caller can go
|
| 523 | * synchronous.
|
| 524 |
|
| 525 | static int try_prep_async_create(struct inode *dir, struct dentry *dentry,
|
| 526 | struct ceph_file_layout *lo, u64 *pino)
|
| 527 | {
|
| 528 | struct ceph_inode_info *ci = ceph_inode(dir);
|
| 529 | struct ceph_dentry_info *di = ceph_dentry(dentry);
|
| 530 | int got = 0, want = CEPH_CAP_FILE_EXCL | CEPH_CAP_DIR_CREATE;
|
| 531 | u64 ino;
|
| 532 |
|
| 533 | spin_lock(&ci->i_ceph_lock);
|
| 534 |
|
| 535 | if (!ci->i_auth_cap)
|
| 536 | goto no_async;
|
| 537 |
|
| 538 |
|
| 539 | if (xa_empty(&ci->i_auth_cap->session->s_delegated_inos))
|
| 540 | goto no_async;
|
| 541 |
|
| 542 | if (!ceph_file_layout_is_valid(&ci->i_cached_layout))
|
| 543 | goto no_async;
|
| 544 |
|
| 545 | if ((__ceph_caps_issued(ci, NULL) & want) != want)
|
| 546 | goto no_async;
|
| 547 |
|
| 548 | if (d_in_lookup(dentry)) {
|
| 549 | if (!__ceph_dir_is_complete(ci))
|
| 550 | goto no_async;
|
| 551 | spin_lock(&dentry->d_lock);
|
| 552 | di->lease_shared_gen = atomic_read(&ci->i_shared_gen);
|
| 553 | spin_unlock(&dentry->d_lock);
|
| 554 | } else if (atomic_read(&ci->i_shared_gen) !=
|
| 555 | READ_ONCE(di->lease_shared_gen)) {
|
| 556 | goto no_async;
|
| 557 | }
|
| 558 |
|
| 559 | ino = ceph_get_deleg_ino(ci->i_auth_cap->session);
|
| 560 | if (!ino)
|
| 561 | goto no_async;
|
| 562 |
|
| 563 | *pino = ino;
|
| 564 | ceph_take_cap_refs(ci, want, false);
|
| 565 | memcpy(lo, &ci->i_cached_layout, sizeof(*lo));
|
| 566 | rcu_assign_pointer(lo->pool_ns,
|
| 567 | ceph_try_get_string(ci->i_cached_layout.pool_ns));
|
| 568 | got = want;
|
| 569 | no_async:
|
| 570 | spin_unlock(&ci->i_ceph_lock);
|
| 571 | return got;
|
| 572 | }
|
| 573 |
|
| 574 | static void restore_deleg_ino(struct inode *dir, u64 ino)
|
| 575 | {
|
| 576 | struct ceph_client *cl = ceph_inode_to_client(dir);
|
| 577 | struct ceph_inode_info *ci = ceph_inode(dir);
|
| 578 | struct ceph_mds_session *s = NULL;
|
| 579 |
|
| 580 | spin_lock(&ci->i_ceph_lock);
|
| 581 | if (ci->i_auth_cap)
|
| 582 | s = ceph_get_mds_session(ci->i_auth_cap->session);
|
| 583 | spin_unlock(&ci->i_ceph_lock);
|
| 584 | if (s) {
|
| 585 | int err = ceph_restore_deleg_ino(s, ino);
|
| 586 | if (err)
|
| 587 | pr_warn_client(cl,
|
| 588 | "unable to restore delegated ino 0x%llx to session: %d\n",
|
| 589 | ino, err);
|
| 590 | ceph_put_mds_session(s);
|
| 591 | }
|
| 592 | }
|
| 593 |
|
| 594 | static void wake_async_create_waiters(struct inode *inode,
|
| 595 | struct ceph_mds_session *session)
|
| 596 | {
|
| 597 | struct ceph_inode_info *ci = ceph_inode(inode);
|
| 598 | bool check_cap = false;
|
| 599 |
|
| 600 | spin_lock(&ci->i_ceph_lock);
|
| 601 | if (ci->i_ceph_flags & CEPH_I_ASYNC_CREATE) {
|
| 602 |
|
| 603 | clear_and_wake_up_bit(CEPH_I_ASYNC_CREATE_BIT, &ci->i_ceph_flags);
|
| 604 |
|
| 605 | if (test_and_clear_bit(CEPH_I_ASYNC_CHECK_CAPS_BIT,
|
| 606 | &ci->i_ceph_flags))
|
| 607 | check_cap = true;
|
| 608 | }
|
| 609 | ceph_kick_flushing_inode_caps(session, ci);
|
| 610 | spin_unlock(&ci->i_ceph_lock);
|
| 611 |
|
| 612 | if (check_cap)
|
| 613 | ceph_check_caps(ci, CHECK_CAPS_FLUSH);
|
| 614 | }
|
| 615 |
|
| 616 | static void ceph_async_create_cb(struct ceph_mds_client *mdsc,
|
| 617 | struct ceph_mds_request *req)
|
| 618 | {
|
| 619 | struct ceph_client *cl = mdsc->fsc->client;
|
| 620 | struct dentry *dentry = req->r_dentry;
|
| 621 | struct inode *dinode = d_inode(dentry);
|
| 622 | struct inode *tinode = req->r_target_inode;
|
| 623 | int result = req->r_err ? req->r_err :
|
| 624 | le32_to_cpu(req->r_reply_info.head->result);
|
| 625 |
|
| 626 | WARN_ON_ONCE(dinode && tinode && dinode != tinode);
|
| 627 |
|
| 628 |
|
| 629 | if (result == -EJUKEBOX)
|
| 630 | goto out;
|
| 631 |
|
| 632 | mapping_set_error(req->r_parent->i_mapping, result);
|
| 633 |
|
| 634 | if (result) {
|
| 635 | struct ceph_path_info path_info = {0};
|
| 636 | char *path = ceph_mdsc_build_path(mdsc, req->r_dentry, &path_info, 0);
|
| 637 |
|
| 638 | pr_warn_client(cl,
|
| 639 | "async create failure path=(%llx)%s result=%d!\n",
|
| 640 | path_info.vino.ino, IS_ERR(path) ? "<<bad>>" : path, result);
|
| 641 | ceph_mdsc_free_path_info(&path_info);
|
| 642 |
|
| 643 | ceph_dir_clear_complete(req->r_parent);
|
| 644 | if (!d_unhashed(dentry))
|
| 645 | d_drop(dentry);
|
| 646 |
|
| 647 | if (dinode) {
|
| 648 | mapping_set_error(dinode->i_mapping, result);
|
| 649 | ceph_inode_shutdown(dinode);
|
| 650 | wake_async_create_waiters(dinode, req->r_session);
|
| 651 | }
|
| 652 | }
|
| 653 |
|
| 654 | if (tinode) {
|
| 655 | u64 ino = ceph_vino(tinode).ino;
|
| 656 |
|
| 657 | if (req->r_deleg_ino != ino)
|
| 658 | pr_warn_client(cl,
|
| 659 | "inode number mismatch! err=%d deleg_ino=0x%llx target=0x%llx\n",
|
| 660 | req->r_err, req->r_deleg_ino, ino);
|
| 661 |
|
| 662 | mapping_set_error(tinode->i_mapping, result);
|
| 663 | wake_async_create_waiters(tinode, req->r_session);
|
| 664 | } else if (!result) {
|
| 665 | pr_warn_client(cl, "no req->r_target_inode for 0x%llx\n",
|
| 666 | req->r_deleg_ino);
|
| 667 | }
|
| 668 | out:
|
| 669 | ceph_mdsc_release_dir_caps(req);
|
| 670 | }
|
| 671 |
|
| 672 | static int ceph_finish_async_create(struct inode *dir, struct inode *inode,
|
| 673 | struct dentry *dentry,
|
| 674 | struct file *file, umode_t mode,
|
| 675 | struct ceph_mds_request *req,
|
| 676 | struct ceph_acl_sec_ctx *as_ctx,
|
| 677 | struct ceph_file_layout *lo)
|
| 678 | {
|
| 679 | int ret;
|
| 680 | char xattr_buf[4];
|
| 681 | struct ceph_mds_reply_inode in = { };
|
| 682 | struct ceph_mds_reply_info_in iinfo = { .in = &in };
|
| 683 | struct ceph_inode_info *ci = ceph_inode(dir);
|
| 684 | struct ceph_dentry_info *di = ceph_dentry(dentry);
|
| 685 | struct timespec64 now;
|
| 686 | struct ceph_string *pool_ns;
|
| 687 | struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(dir->i_sb);
|
| 688 | struct ceph_client *cl = mdsc->fsc->client;
|
| 689 | struct ceph_vino vino = { .ino = req->r_deleg_ino,
|
| 690 | .snap = CEPH_NOSNAP };
|
| 691 |
|
| 692 | ktime_get_real_ts64(&now);
|
| 693 |
|
| 694 | iinfo.inline_version = CEPH_INLINE_NONE;
|
| 695 | iinfo.change_attr = 1;
|
| 696 | ceph_encode_timespec64(&iinfo.btime, &now);
|
| 697 |
|
| 698 | if (req->r_pagelist) {
|
| 699 | iinfo.xattr_len = req->r_pagelist->length;
|
| 700 | iinfo.xattr_data = req->r_pagelist->mapped_tail;
|
| 701 | } else {
|
| 702 |
|
| 703 | iinfo.xattr_len = ARRAY_SIZE(xattr_buf);
|
| 704 | iinfo.xattr_data = xattr_buf;
|
| 705 | memset(iinfo.xattr_data, 0, iinfo.xattr_len);
|
| 706 | }
|
| 707 |
|
| 708 | in.ino = cpu_to_le64(vino.ino);
|
| 709 | in.snapid = cpu_to_le64(CEPH_NOSNAP);
|
| 710 | in.version = cpu_to_le64(1); |
| 711 | in.cap.caps = in.cap.wanted = cpu_to_le32(CEPH_CAP_ALL_FILE);
|
| 712 | in.cap.cap_id = cpu_to_le64(1);
|
| 713 | in.cap.realm = cpu_to_le64(ci->i_snap_realm->ino);
|
| 714 | in.cap.flags = CEPH_CAP_FLAG_AUTH;
|
| 715 | in.ctime = in.mtime = in.atime = iinfo.btime;
|
| 716 | in.truncate_seq = cpu_to_le32(1);
|
| 717 | in.truncate_size = cpu_to_le64(-1ULL);
|
| 718 | in.xattr_version = cpu_to_le64(1);
|
| 719 | in.uid = cpu_to_le32(from_kuid(&init_user_ns,
|
| 720 | mapped_fsuid(req->r_mnt_idmap,
|
| 721 | &init_user_ns)));
|
| 722 | if (dir->i_mode & S_ISGID) {
|
| 723 | in.gid = cpu_to_le32(from_kgid(&init_user_ns, dir->i_gid));
|
| 724 |
|
| 725 |
|
| 726 | if (S_ISDIR(mode))
|
| 727 | mode |= S_ISGID;
|
| 728 | } else {
|
| 729 | in.gid = cpu_to_le32(from_kgid(&init_user_ns,
|
| 730 | mapped_fsgid(req->r_mnt_idmap,
|
| 731 | &init_user_ns)));
|
| 732 | }
|
| 733 | in.mode = cpu_to_le32((u32)mode);
|
| 734 |
|
| 735 | in.nlink = cpu_to_le32(1);
|
| 736 | in.max_size = cpu_to_le64(lo->stripe_unit);
|
| 737 |
|
| 738 | ceph_file_layout_to_legacy(lo, &in.layout);
|
| 739 |
|
| 740 | pool_ns = rcu_dereference_raw(lo->pool_ns);
|
| 741 | if (pool_ns) {
|
| 742 | iinfo.pool_ns_len = pool_ns->len;
|
| 743 | iinfo.pool_ns_data = pool_ns->str;
|
| 744 | }
|
| 745 |
|
| 746 | down_read(&mdsc->snap_rwsem);
|
| 747 | ret = ceph_fill_inode(inode, NULL, &iinfo, NULL, req->r_session,
|
| 748 | req->r_fmode, NULL);
|
| 749 | up_read(&mdsc->snap_rwsem);
|
| 750 | if (ret) {
|
| 751 | doutc(cl, "failed to fill inode: %d\n", ret);
|
| 752 | ceph_dir_clear_complete(dir);
|
| 753 | if (!d_unhashed(dentry))
|
| 754 | d_drop(dentry);
|
| 755 | discard_new_inode(inode);
|
| 756 | } else {
|
| 757 | struct dentry *dn;
|
| 758 |
|
| 759 | doutc(cl, "d_adding new inode 0x%llx to 0x%llx/%s\n",
|
| 760 | vino.ino, ceph_ino(dir), dentry->d_name.name);
|
| 761 | ceph_dir_clear_ordered(dir);
|
| 762 | ceph_init_inode_acls(inode, as_ctx);
|
| 763 | if (inode_state_read_once(inode) & I_NEW) {
|
| 764 | |
| 765 | * If it's not I_NEW, then someone created this before
|
| 766 | * we got here. Assume the server is aware of it at
|
| 767 | * that point and don't worry about setting
|
| 768 | * CEPH_I_ASYNC_CREATE.
|
| 769 |
|
| 770 | set_bit(CEPH_I_ASYNC_CREATE_BIT,
|
| 771 | &ceph_inode(inode)->i_ceph_flags);
|
| 772 | unlock_new_inode(inode);
|
| 773 | }
|
| 774 | if (d_in_lookup(dentry) || d_really_is_negative(dentry)) {
|
| 775 | if (!d_unhashed(dentry))
|
| 776 | d_drop(dentry);
|
| 777 | dn = d_splice_alias(inode, dentry);
|
| 778 | WARN_ON_ONCE(dn && dn != dentry);
|
| 779 | }
|
| 780 | file->f_mode |= FMODE_CREATED;
|
| 781 | ret = finish_open(file, dentry, ceph_open);
|
| 782 | }
|
| 783 |
|
| 784 | spin_lock(&dentry->d_lock);
|
| 785 | clear_and_wake_up_bit(CEPH_DENTRY_ASYNC_CREATE_BIT, &di->flags);
|
| 786 | spin_unlock(&dentry->d_lock);
|
| 787 |
|
| 788 | return ret;
|
| 789 | }
|
| 790 |
|
| 791 | |
| 792 | * Do a lookup + open with a single request. If we get a non-existent
|
| 793 | * file or symlink, return 1 so the VFS can retry.
|
| 794 |
|
| 795 | int ceph_atomic_open(struct inode *dir, struct dentry *dentry,
|
| 796 | struct file *file, unsigned flags, umode_t mode)
|
| 797 | {
|
| 798 | const struct mnt_idmap *idmap = file_mnt_idmap(file);
|
| 799 | struct ceph_fs_client *fsc = ceph_sb_to_fs_client(dir->i_sb);
|
| 800 | struct ceph_client *cl = fsc->client;
|
| 801 | struct ceph_mds_client *mdsc = fsc->mdsc;
|
| 802 | struct ceph_mds_request *req;
|
| 803 | struct inode *new_inode = NULL;
|
| 804 | struct dentry *dn;
|
| 805 | struct ceph_acl_sec_ctx as_ctx = {};
|
| 806 | bool try_async = ceph_test_mount_opt(fsc, ASYNC_DIROPS);
|
| 807 | int mask;
|
| 808 | int err;
|
| 809 | char *path;
|
| 810 |
|
| 811 | doutc(cl, "%p %llx.%llx dentry %p '%pd' %s flags %d mode 0%o\n",
|
| 812 | dir, ceph_vinop(dir), dentry, dentry,
|
| 813 | d_unhashed(dentry) ? "unhashed" : "hashed", flags, mode);
|
| 814 |
|
| 815 | if (dentry->d_name.len > NAME_MAX)
|
| 816 | return -ENAMETOOLONG;
|
| 817 |
|
| 818 | err = ceph_wait_on_conflict_unlink(dentry);
|
| 819 | if (err)
|
| 820 | return err;
|
| 821 | |
| 822 | * Do not truncate the file, since atomic_open is called before the
|
| 823 | * permission check. The caller will do the truncation afterward.
|
| 824 |
|
| 825 | flags &= ~O_TRUNC;
|
| 826 |
|
| 827 | dn = d_find_alias(dir);
|
| 828 | if (!dn) {
|
| 829 | try_async = false;
|
| 830 | } else {
|
| 831 | struct ceph_path_info path_info = {0};
|
| 832 | path = ceph_mdsc_build_path(mdsc, dn, &path_info, 0);
|
| 833 | if (IS_ERR(path)) {
|
| 834 | try_async = false;
|
| 835 | err = 0;
|
| 836 | } else {
|
| 837 | int fmode = ceph_flags_to_mode(flags);
|
| 838 |
|
| 839 | mask = MAY_READ;
|
| 840 | if (fmode & CEPH_FILE_MODE_WR)
|
| 841 | mask |= MAY_WRITE;
|
| 842 | err = ceph_mds_check_access(mdsc, path, mask);
|
| 843 | }
|
| 844 | ceph_mdsc_free_path_info(&path_info);
|
| 845 | dput(dn);
|
| 846 |
|
| 847 |
|
| 848 | if (err == -EACCES) {
|
| 849 | return err;
|
| 850 | } else if (err < 0) {
|
| 851 | try_async = false;
|
| 852 | err = 0;
|
| 853 | }
|
| 854 | }
|
| 855 |
|
| 856 | retry:
|
| 857 | if (flags & O_CREAT) {
|
| 858 | if (ceph_quota_is_max_files_exceeded(dir))
|
| 859 | return -EDQUOT;
|
| 860 |
|
| 861 | new_inode = ceph_new_inode(dir, dentry, &mode, &as_ctx);
|
| 862 | if (IS_ERR(new_inode)) {
|
| 863 | err = PTR_ERR(new_inode);
|
| 864 | goto out_ctx;
|
| 865 | }
|
| 866 |
|
| 867 | if (as_ctx.pagelist &&
|
| 868 | !list_is_singular(&as_ctx.pagelist->head))
|
| 869 | try_async = false;
|
| 870 | } else if (!d_in_lookup(dentry)) {
|
| 871 |
|
| 872 | return -ENOENT;
|
| 873 | }
|
| 874 |
|
| 875 |
|
| 876 | req = prepare_open_request(dir->i_sb, flags, mode);
|
| 877 | if (IS_ERR(req)) {
|
| 878 | err = PTR_ERR(req);
|
| 879 | goto out_ctx;
|
| 880 | }
|
| 881 | req->r_dentry = dget(dentry);
|
| 882 | req->r_num_caps = 2;
|
| 883 | mask = CEPH_STAT_CAP_INODE | CEPH_CAP_AUTH_SHARED;
|
| 884 | if (ceph_security_xattr_wanted(dir))
|
| 885 | mask |= CEPH_CAP_XATTR_SHARED;
|
| 886 | req->r_args.open.mask = cpu_to_le32(mask);
|
| 887 | req->r_parent = dir;
|
| 888 | if (req->r_op == CEPH_MDS_OP_CREATE)
|
| 889 | req->r_mnt_idmap = mnt_idmap_get(idmap);
|
| 890 | ihold(dir);
|
| 891 | if (IS_ENCRYPTED(dir)) {
|
| 892 | set_bit(CEPH_MDS_R_FSCRYPT_FILE, &req->r_req_flags);
|
| 893 | err = fscrypt_prepare_lookup_partial(dir, dentry);
|
| 894 | if (err < 0)
|
| 895 | goto out_req;
|
| 896 | }
|
| 897 |
|
| 898 | if (flags & O_CREAT) {
|
| 899 | struct ceph_file_layout lo;
|
| 900 |
|
| 901 | req->r_dentry_drop = CEPH_CAP_FILE_SHARED | CEPH_CAP_AUTH_EXCL |
|
| 902 | CEPH_CAP_XATTR_EXCL;
|
| 903 | req->r_dentry_unless = CEPH_CAP_FILE_EXCL;
|
| 904 |
|
| 905 | ceph_as_ctx_to_req(req, &as_ctx);
|
| 906 |
|
| 907 | if (try_async && (req->r_dir_caps =
|
| 908 | try_prep_async_create(dir, dentry, &lo,
|
| 909 | &req->r_deleg_ino))) {
|
| 910 | struct ceph_vino vino = { .ino = req->r_deleg_ino,
|
| 911 | .snap = CEPH_NOSNAP };
|
| 912 | struct ceph_dentry_info *di = ceph_dentry(dentry);
|
| 913 |
|
| 914 | set_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags);
|
| 915 | req->r_args.open.flags |= cpu_to_le32(CEPH_O_EXCL);
|
| 916 | req->r_callback = ceph_async_create_cb;
|
| 917 |
|
| 918 |
|
| 919 | new_inode = ceph_get_inode(dir->i_sb, vino, new_inode);
|
| 920 | if (IS_ERR(new_inode)) {
|
| 921 | err = PTR_ERR(new_inode);
|
| 922 | new_inode = NULL;
|
| 923 | goto out_req;
|
| 924 | }
|
| 925 | WARN_ON_ONCE(!(inode_state_read_once(new_inode) & I_NEW));
|
| 926 |
|
| 927 | spin_lock(&dentry->d_lock);
|
| 928 | di->flags |= CEPH_DENTRY_ASYNC_CREATE;
|
| 929 | spin_unlock(&dentry->d_lock);
|
| 930 |
|
| 931 | err = ceph_mdsc_submit_request(mdsc, dir, req);
|
| 932 | if (!err) {
|
| 933 | err = ceph_finish_async_create(dir, new_inode,
|
| 934 | dentry, file,
|
| 935 | mode, req,
|
| 936 | &as_ctx, &lo);
|
| 937 | new_inode = NULL;
|
| 938 | } else if (err == -EJUKEBOX) {
|
| 939 | restore_deleg_ino(dir, req->r_deleg_ino);
|
| 940 | ceph_mdsc_put_request(req);
|
| 941 | discard_new_inode(new_inode);
|
| 942 | ceph_release_acl_sec_ctx(&as_ctx);
|
| 943 | memset(&as_ctx, 0, sizeof(as_ctx));
|
| 944 | new_inode = NULL;
|
| 945 | try_async = false;
|
| 946 | ceph_put_string(rcu_dereference_raw(lo.pool_ns));
|
| 947 | goto retry;
|
| 948 | }
|
| 949 | ceph_put_string(rcu_dereference_raw(lo.pool_ns));
|
| 950 | goto out_req;
|
| 951 | }
|
| 952 | }
|
| 953 |
|
| 954 | set_bit(CEPH_MDS_R_PARENT_LOCKED, &req->r_req_flags);
|
| 955 | req->r_new_inode = new_inode;
|
| 956 | new_inode = NULL;
|
| 957 | err = ceph_mdsc_do_request(mdsc, (flags & O_CREAT) ? dir : NULL, req);
|
| 958 | if (err == -ENOENT) {
|
| 959 | dentry = ceph_handle_snapdir(req, dentry);
|
| 960 | if (IS_ERR(dentry)) {
|
| 961 | err = PTR_ERR(dentry);
|
| 962 | goto out_req;
|
| 963 | }
|
| 964 | err = 0;
|
| 965 | }
|
| 966 |
|
| 967 | if (!err && (flags & O_CREAT) && !req->r_reply_info.head->is_dentry)
|
| 968 | err = ceph_handle_notrace_create(dir, dentry);
|
| 969 |
|
| 970 | if (d_in_lookup(dentry)) {
|
| 971 | dn = ceph_finish_lookup(req, dentry, err);
|
| 972 | if (IS_ERR(dn))
|
| 973 | err = PTR_ERR(dn);
|
| 974 | } else {
|
| 975 |
|
| 976 | dn = NULL;
|
| 977 | }
|
| 978 | if (err)
|
| 979 | goto out_req;
|
| 980 | if (dn || d_really_is_negative(dentry) || d_is_symlink(dentry)) {
|
| 981 |
|
| 982 | doutc(cl, "finish_no_open on dn %p\n", dn);
|
| 983 | err = finish_no_open(file, dn);
|
| 984 | } else {
|
| 985 | if (IS_ENCRYPTED(dir) &&
|
| 986 | !fscrypt_has_permitted_context(dir, d_inode(dentry))) {
|
| 987 | pr_warn_client(cl,
|
| 988 | "Inconsistent encryption context (parent %llx:%llx child %llx:%llx)\n",
|
| 989 | ceph_vinop(dir), ceph_vinop(d_inode(dentry)));
|
| 990 | goto out_req;
|
| 991 | }
|
| 992 |
|
| 993 | doutc(cl, "finish_open on dn %p\n", dn);
|
| 994 | if (req->r_op == CEPH_MDS_OP_CREATE && req->r_reply_info.has_create_ino) {
|
| 995 | struct inode *newino = d_inode(dentry);
|
| 996 |
|
| 997 | cache_file_layout(dir, newino);
|
| 998 | ceph_init_inode_acls(newino, &as_ctx);
|
| 999 | file->f_mode |= FMODE_CREATED;
|
| 1000 | }
|
| 1001 | if ((flags & __O_REGULAR) && !d_is_reg(dentry)) {
|
| 1002 | err = -EFTYPE;
|
| 1003 | goto out_req;
|
| 1004 | }
|
| 1005 | err = finish_open(file, dentry, ceph_open);
|
| 1006 | }
|
| 1007 | out_req:
|
| 1008 | ceph_mdsc_put_request(req);
|
| 1009 | iput(new_inode);
|
| 1010 | out_ctx:
|
| 1011 | ceph_release_acl_sec_ctx(&as_ctx);
|
| 1012 | doutc(cl, "result=%d\n", err);
|
| 1013 | return err;
|
| 1014 | }
|
| 1015 |
|
| 1016 | int ceph_release(struct inode *inode, struct file *file)
|
| 1017 | {
|
| 1018 | struct ceph_client *cl = ceph_inode_to_client(inode);
|
| 1019 | struct ceph_inode_info *ci = ceph_inode(inode);
|
| 1020 |
|
| 1021 | if (S_ISDIR(inode->i_mode)) {
|
| 1022 | struct ceph_dir_file_info *dfi = file->private_data;
|
| 1023 | doutc(cl, "%p %llx.%llx dir file %p\n", inode,
|
| 1024 | ceph_vinop(inode), file);
|
| 1025 | WARN_ON(!list_empty(&dfi->file_info.rw_contexts));
|
| 1026 |
|
| 1027 | ceph_put_fmode(ci, dfi->file_info.fmode, 1);
|
| 1028 |
|
| 1029 | if (dfi->last_readdir)
|
| 1030 | ceph_mdsc_put_request(dfi->last_readdir);
|
| 1031 | kfree(dfi->last_name);
|
| 1032 | kfree(dfi->dir_info);
|
| 1033 | kmem_cache_free(ceph_dir_file_cachep, dfi);
|
| 1034 | } else {
|
| 1035 | struct ceph_file_info *fi = file->private_data;
|
| 1036 | doutc(cl, "%p %llx.%llx regular file %p\n", inode,
|
| 1037 | ceph_vinop(inode), file);
|
| 1038 | WARN_ON(!list_empty(&fi->rw_contexts));
|
| 1039 |
|
| 1040 | ceph_fscache_unuse_cookie(inode, file->f_mode & FMODE_WRITE);
|
| 1041 | ceph_put_fmode(ci, fi->fmode, 1);
|
| 1042 |
|
| 1043 | kmem_cache_free(ceph_file_cachep, fi);
|
| 1044 | }
|
| 1045 |
|
| 1046 |
|
| 1047 | wake_up_all(&ci->i_cap_wq);
|
| 1048 | return 0;
|
| 1049 | }
|
| 1050 |
|
| 1051 | enum {
|
| 1052 | HAVE_RETRIED = 1,
|
| 1053 | CHECK_EOF = 2,
|
| 1054 | READ_INLINE = 3,
|
| 1055 | };
|
| 1056 |
|
| 1057 | |
| 1058 | * Completely synchronous read and write methods. Direct from __user
|
| 1059 | * buffer to osd, or directly to user pages (if O_DIRECT).
|
| 1060 | *
|
| 1061 | * If the read spans object boundary, just do multiple reads. (That's not
|
| 1062 | * atomic, but good enough for now.)
|
| 1063 | *
|
| 1064 | * If we get a short result from the OSD, check against i_size; we need to
|
| 1065 | * only return a short read to the caller if we hit EOF.
|
| 1066 |
|
| 1067 | ssize_t __ceph_sync_read(struct inode *inode, loff_t *ki_pos,
|
| 1068 | struct iov_iter *to, int *retry_op,
|
| 1069 | u64 *last_objver)
|
| 1070 | {
|
| 1071 | struct ceph_inode_info *ci = ceph_inode(inode);
|
| 1072 | struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
|
| 1073 | struct ceph_client *cl = fsc->client;
|
| 1074 | struct ceph_osd_client *osdc = &fsc->client->osdc;
|
| 1075 | ssize_t ret;
|
| 1076 | u64 off = *ki_pos;
|
| 1077 | u64 len = iov_iter_count(to);
|
| 1078 | u64 i_size = i_size_read(inode);
|
| 1079 | bool sparse = IS_ENCRYPTED(inode) || ceph_test_mount_opt(fsc, SPARSEREAD);
|
| 1080 | u64 objver = 0;
|
| 1081 |
|
| 1082 | doutc(cl, "on inode %p %llx.%llx %llx~%llx\n", inode,
|
| 1083 | ceph_vinop(inode), *ki_pos, len);
|
| 1084 |
|
| 1085 | if (ceph_inode_is_shutdown(inode))
|
| 1086 | return -EIO;
|
| 1087 |
|
| 1088 | if (!len || !i_size)
|
| 1089 | return 0;
|
| 1090 | |
| 1091 | * flush any page cache pages in this range. this
|
| 1092 | * will make concurrent normal and sync io slow,
|
| 1093 | * but it will at least behave sensibly when they are
|
| 1094 | * in sequence.
|
| 1095 |
|
| 1096 | ret = filemap_write_and_wait_range(inode->i_mapping,
|
| 1097 | off, off + len - 1);
|
| 1098 | if (ret < 0)
|
| 1099 | return ret;
|
| 1100 |
|
| 1101 | ret = 0;
|
| 1102 | while ((len = iov_iter_count(to)) > 0) {
|
| 1103 | struct ceph_osd_request *req;
|
| 1104 | struct page **pages;
|
| 1105 | int num_pages;
|
| 1106 | size_t page_off;
|
| 1107 | bool more;
|
| 1108 | int idx = 0;
|
| 1109 | size_t left;
|
| 1110 | struct ceph_osd_req_op *op;
|
| 1111 | u64 read_off = off;
|
| 1112 | u64 read_len = len;
|
| 1113 | int extent_cnt;
|
| 1114 |
|
| 1115 |
|
| 1116 | ceph_fscrypt_adjust_off_and_len(inode, &read_off, &read_len);
|
| 1117 |
|
| 1118 | doutc(cl, "orig %llu~%llu reading %llu~%llu", off, len,
|
| 1119 | read_off, read_len);
|
| 1120 |
|
| 1121 | req = ceph_osdc_new_request(osdc, &ci->i_layout,
|
| 1122 | ci->i_vino, read_off, &read_len, 0, 1,
|
| 1123 | sparse ? CEPH_OSD_OP_SPARSE_READ :
|
| 1124 | CEPH_OSD_OP_READ,
|
| 1125 | CEPH_OSD_FLAG_READ,
|
| 1126 | NULL, ci->i_truncate_seq,
|
| 1127 | ci->i_truncate_size, false);
|
| 1128 | if (IS_ERR(req)) {
|
| 1129 | ret = PTR_ERR(req);
|
| 1130 | break;
|
| 1131 | }
|
| 1132 |
|
| 1133 |
|
| 1134 | if (off + len > read_off + read_len)
|
| 1135 | len = read_off + read_len - off;
|
| 1136 | more = len < iov_iter_count(to);
|
| 1137 |
|
| 1138 | op = &req->r_ops[0];
|
| 1139 | if (sparse) {
|
| 1140 | extent_cnt = __ceph_sparse_read_ext_count(inode, read_len);
|
| 1141 | ret = ceph_alloc_sparse_ext_map(op, extent_cnt);
|
| 1142 | if (ret) {
|
| 1143 | ceph_osdc_put_request(req);
|
| 1144 | break;
|
| 1145 | }
|
| 1146 | }
|
| 1147 |
|
| 1148 | num_pages = calc_pages_for(read_off, read_len);
|
| 1149 | page_off = offset_in_page(off);
|
| 1150 | pages = ceph_alloc_page_vector(num_pages, GFP_KERNEL);
|
| 1151 | if (IS_ERR(pages)) {
|
| 1152 | ceph_osdc_put_request(req);
|
| 1153 | ret = PTR_ERR(pages);
|
| 1154 | break;
|
| 1155 | }
|
| 1156 |
|
| 1157 | osd_req_op_extent_osd_data_pages(req, 0, pages, read_len,
|
| 1158 | offset_in_page(read_off),
|
| 1159 | false, true);
|
| 1160 |
|
| 1161 | ceph_osdc_start_request(osdc, req);
|
| 1162 | ret = ceph_osdc_wait_request(osdc, req);
|
| 1163 |
|
| 1164 | ceph_update_read_metrics(&fsc->mdsc->metric,
|
| 1165 | req->r_start_latency,
|
| 1166 | req->r_end_latency,
|
| 1167 | read_len, ret);
|
| 1168 | |
| 1169 | * Only record subvolume metrics for actual bytes read.
|
| 1170 | * ret == 0 means EOF (no data), not an I/O operation.
|
| 1171 |
|
| 1172 | if (ret > 0)
|
| 1173 | ceph_record_subvolume_io(inode, false,
|
| 1174 | req->r_start_latency,
|
| 1175 | req->r_end_latency,
|
| 1176 | ret);
|
| 1177 |
|
| 1178 | if (ret > 0)
|
| 1179 | objver = req->r_version;
|
| 1180 |
|
| 1181 | i_size = i_size_read(inode);
|
| 1182 | doutc(cl, "%llu~%llu got %zd i_size %llu%s\n", off, len,
|
| 1183 | ret, i_size, (more ? " MORE" : ""));
|
| 1184 |
|
| 1185 |
|
| 1186 | if (sparse && ret >= 0)
|
| 1187 | ret = ceph_sparse_ext_map_end(op);
|
| 1188 | else if (ret == -ENOENT)
|
| 1189 | ret = 0;
|
| 1190 |
|
| 1191 | if (ret < 0) {
|
| 1192 | ceph_osdc_put_request(req);
|
| 1193 | if (ret == -EBLOCKLISTED)
|
| 1194 | fsc->blocklisted = true;
|
| 1195 | break;
|
| 1196 | }
|
| 1197 |
|
| 1198 | if (IS_ENCRYPTED(inode)) {
|
| 1199 | int fret;
|
| 1200 |
|
| 1201 | fret = ceph_fscrypt_decrypt_extents(inode, pages,
|
| 1202 | read_off, op->extent.sparse_ext,
|
| 1203 | op->extent.sparse_ext_cnt);
|
| 1204 | if (fret < 0) {
|
| 1205 | ret = fret;
|
| 1206 | ceph_osdc_put_request(req);
|
| 1207 | break;
|
| 1208 | }
|
| 1209 |
|
| 1210 |
|
| 1211 | fret -= (off - read_off);
|
| 1212 |
|
| 1213 | |
| 1214 | * Short read after big offset adjustment?
|
| 1215 | * Nothing is usable, just call it a zero
|
| 1216 | * len read.
|
| 1217 |
|
| 1218 | fret = max(fret, 0);
|
| 1219 |
|
| 1220 |
|
| 1221 | ret = min_t(ssize_t, fret, len);
|
| 1222 | }
|
| 1223 |
|
| 1224 |
|
| 1225 | if (ret < len && (off + ret < i_size)) {
|
| 1226 | int zlen = min(len - ret, i_size - off - ret);
|
| 1227 | int zoff = page_off + ret;
|
| 1228 |
|
| 1229 | doutc(cl, "zero gap %llu~%llu\n", off + ret,
|
| 1230 | off + ret + zlen);
|
| 1231 | ceph_zero_page_vector_range(zoff, zlen, pages);
|
| 1232 | ret += zlen;
|
| 1233 | }
|
| 1234 |
|
| 1235 | if (off + ret > i_size)
|
| 1236 | left = (i_size > off) ? i_size - off : 0;
|
| 1237 | else
|
| 1238 | left = ret;
|
| 1239 |
|
| 1240 | while (left > 0) {
|
| 1241 | size_t plen, copied;
|
| 1242 |
|
| 1243 | plen = min_t(size_t, left, PAGE_SIZE - page_off);
|
| 1244 | SetPageUptodate(pages[idx]);
|
| 1245 | copied = copy_page_to_iter(pages[idx++],
|
| 1246 | page_off, plen, to);
|
| 1247 | off += copied;
|
| 1248 | left -= copied;
|
| 1249 | page_off = 0;
|
| 1250 | if (copied < plen) {
|
| 1251 | ret = -EFAULT;
|
| 1252 | break;
|
| 1253 | }
|
| 1254 | }
|
| 1255 |
|
| 1256 | ceph_osdc_put_request(req);
|
| 1257 |
|
| 1258 | if (off >= i_size || !more)
|
| 1259 | break;
|
| 1260 | }
|
| 1261 |
|
| 1262 | if (ret > 0) {
|
| 1263 | if (off >= i_size) {
|
| 1264 | *retry_op = CHECK_EOF;
|
| 1265 | ret = i_size - *ki_pos;
|
| 1266 | *ki_pos = i_size;
|
| 1267 | } else {
|
| 1268 | ret = off - *ki_pos;
|
| 1269 | *ki_pos = off;
|
| 1270 | }
|
| 1271 |
|
| 1272 | if (last_objver)
|
| 1273 | *last_objver = objver;
|
| 1274 | }
|
| 1275 | doutc(cl, "result %zd retry_op %d\n", ret, *retry_op);
|
| 1276 | return ret;
|
| 1277 | }
|
| 1278 |
|
| 1279 | static ssize_t ceph_sync_read(struct kiocb *iocb, struct iov_iter *to,
|
| 1280 | int *retry_op)
|
| 1281 | {
|
| 1282 | struct file *file = iocb->ki_filp;
|
| 1283 | struct inode *inode = file_inode(file);
|
| 1284 | struct ceph_client *cl = ceph_inode_to_client(inode);
|
| 1285 |
|
| 1286 | doutc(cl, "on file %p %llx~%zx %s\n", file, iocb->ki_pos,
|
| 1287 | iov_iter_count(to),
|
| 1288 | (file->f_flags & O_DIRECT) ? "O_DIRECT" : "");
|
| 1289 |
|
| 1290 | return __ceph_sync_read(inode, &iocb->ki_pos, to, retry_op, NULL);
|
| 1291 | }
|
| 1292 |
|
| 1293 | struct ceph_aio_request {
|
| 1294 | struct kiocb *iocb;
|
| 1295 | size_t total_len;
|
| 1296 | bool write;
|
| 1297 | bool should_dirty;
|
| 1298 | int error;
|
| 1299 | struct list_head osd_reqs;
|
| 1300 | unsigned num_reqs;
|
| 1301 | atomic_t pending_reqs;
|
| 1302 | struct timespec64 mtime;
|
| 1303 | struct ceph_cap_flush *prealloc_cf;
|
| 1304 | };
|
| 1305 |
|
| 1306 | struct ceph_aio_work {
|
| 1307 | struct work_struct work;
|
| 1308 | struct ceph_osd_request *req;
|
| 1309 | };
|
| 1310 |
|
| 1311 | static void ceph_aio_retry_work(struct work_struct *work);
|
| 1312 |
|
| 1313 | static void ceph_aio_complete(struct inode *inode,
|
| 1314 | struct ceph_aio_request *aio_req)
|
| 1315 | {
|
| 1316 | struct ceph_client *cl = ceph_inode_to_client(inode);
|
| 1317 | struct ceph_inode_info *ci = ceph_inode(inode);
|
| 1318 | int ret;
|
| 1319 |
|
| 1320 | if (!atomic_dec_and_test(&aio_req->pending_reqs))
|
| 1321 | return;
|
| 1322 |
|
| 1323 | if (aio_req->iocb->ki_flags & IOCB_DIRECT)
|
| 1324 | inode_dio_end(inode);
|
| 1325 |
|
| 1326 | ret = aio_req->error;
|
| 1327 | if (!ret)
|
| 1328 | ret = aio_req->total_len;
|
| 1329 |
|
| 1330 | doutc(cl, "%p %llx.%llx rc %d\n", inode, ceph_vinop(inode), ret);
|
| 1331 |
|
| 1332 | if (ret >= 0 && aio_req->write) {
|
| 1333 | int dirty;
|
| 1334 |
|
| 1335 | loff_t endoff = aio_req->iocb->ki_pos + aio_req->total_len;
|
| 1336 | if (endoff > i_size_read(inode)) {
|
| 1337 | if (ceph_inode_set_size(inode, endoff))
|
| 1338 | ceph_check_caps(ci, CHECK_CAPS_AUTHONLY);
|
| 1339 | }
|
| 1340 |
|
| 1341 | spin_lock(&ci->i_ceph_lock);
|
| 1342 | dirty = __ceph_mark_dirty_caps(ci, CEPH_CAP_FILE_WR,
|
| 1343 | &aio_req->prealloc_cf);
|
| 1344 | spin_unlock(&ci->i_ceph_lock);
|
| 1345 | if (dirty)
|
| 1346 | __mark_inode_dirty(inode, dirty);
|
| 1347 |
|
| 1348 | }
|
| 1349 |
|
| 1350 | ceph_put_cap_refs(ci, (aio_req->write ? CEPH_CAP_FILE_WR :
|
| 1351 | CEPH_CAP_FILE_RD));
|
| 1352 |
|
| 1353 | aio_req->iocb->ki_complete(aio_req->iocb, ret);
|
| 1354 |
|
| 1355 | ceph_free_cap_flush(aio_req->prealloc_cf);
|
| 1356 | kfree(aio_req);
|
| 1357 | }
|
| 1358 |
|
| 1359 | static void ceph_aio_complete_req(struct ceph_osd_request *req)
|
| 1360 | {
|
| 1361 | int rc = req->r_result;
|
| 1362 | struct inode *inode = req->r_inode;
|
| 1363 | struct ceph_aio_request *aio_req = req->r_priv;
|
| 1364 | struct ceph_osd_data *osd_data = osd_req_op_extent_osd_data(req, 0);
|
| 1365 | struct ceph_osd_req_op *op = &req->r_ops[0];
|
| 1366 | struct ceph_client_metric *metric = &ceph_sb_to_mdsc(inode->i_sb)->metric;
|
| 1367 | unsigned int len = osd_data->bvec_pos.iter.bi_size;
|
| 1368 | bool sparse = (op->op == CEPH_OSD_OP_SPARSE_READ);
|
| 1369 | struct ceph_client *cl = ceph_inode_to_client(inode);
|
| 1370 |
|
| 1371 | BUG_ON(osd_data->type != CEPH_OSD_DATA_TYPE_BVECS);
|
| 1372 | BUG_ON(!osd_data->num_bvecs);
|
| 1373 |
|
| 1374 | doutc(cl, "req %p inode %p %llx.%llx, rc %d bytes %u\n", req,
|
| 1375 | inode, ceph_vinop(inode), rc, len);
|
| 1376 |
|
| 1377 | if (rc == -EOLDSNAPC) {
|
| 1378 | struct ceph_aio_work *aio_work;
|
| 1379 | BUG_ON(!aio_req->write);
|
| 1380 |
|
| 1381 | aio_work = kmalloc_obj(*aio_work, GFP_NOFS);
|
| 1382 | if (aio_work) {
|
| 1383 | INIT_WORK(&aio_work->work, ceph_aio_retry_work);
|
| 1384 | aio_work->req = req;
|
| 1385 | queue_work(ceph_inode_to_fs_client(inode)->inode_wq,
|
| 1386 | &aio_work->work);
|
| 1387 | return;
|
| 1388 | }
|
| 1389 | rc = -ENOMEM;
|
| 1390 | } else if (!aio_req->write) {
|
| 1391 | if (sparse && rc >= 0)
|
| 1392 | rc = ceph_sparse_ext_map_end(op);
|
| 1393 | if (rc == -ENOENT)
|
| 1394 | rc = 0;
|
| 1395 | if (rc >= 0 && len > rc) {
|
| 1396 | struct iov_iter i;
|
| 1397 | int zlen = len - rc;
|
| 1398 |
|
| 1399 | |
| 1400 | * If read is satisfied by single OSD request,
|
| 1401 | * it can pass EOF. Otherwise read is within
|
| 1402 | * i_size.
|
| 1403 |
|
| 1404 | if (aio_req->num_reqs == 1) {
|
| 1405 | loff_t i_size = i_size_read(inode);
|
| 1406 | loff_t endoff = aio_req->iocb->ki_pos + rc;
|
| 1407 | if (endoff < i_size)
|
| 1408 | zlen = min_t(size_t, zlen,
|
| 1409 | i_size - endoff);
|
| 1410 | aio_req->total_len = rc + zlen;
|
| 1411 | }
|
| 1412 |
|
| 1413 | iov_iter_bvec(&i, ITER_DEST, osd_data->bvec_pos.bvecs,
|
| 1414 | osd_data->num_bvecs, len);
|
| 1415 | iov_iter_advance(&i, rc);
|
| 1416 | iov_iter_zero(zlen, &i);
|
| 1417 | }
|
| 1418 | }
|
| 1419 |
|
| 1420 |
|
| 1421 | if (req->r_start_latency) {
|
| 1422 | if (aio_req->write) {
|
| 1423 | ceph_update_write_metrics(metric, req->r_start_latency,
|
| 1424 | req->r_end_latency, len, rc);
|
| 1425 | if (rc >= 0 && len)
|
| 1426 | ceph_record_subvolume_io(inode, true,
|
| 1427 | req->r_start_latency,
|
| 1428 | req->r_end_latency,
|
| 1429 | len);
|
| 1430 | } else {
|
| 1431 | ceph_update_read_metrics(metric, req->r_start_latency,
|
| 1432 | req->r_end_latency, len, rc);
|
| 1433 | if (rc > 0)
|
| 1434 | ceph_record_subvolume_io(inode, false,
|
| 1435 | req->r_start_latency,
|
| 1436 | req->r_end_latency,
|
| 1437 | rc);
|
| 1438 | }
|
| 1439 | }
|
| 1440 |
|
| 1441 | put_bvecs(osd_data->bvec_pos.bvecs, osd_data->num_bvecs,
|
| 1442 | aio_req->should_dirty);
|
| 1443 | ceph_osdc_put_request(req);
|
| 1444 |
|
| 1445 | if (rc < 0)
|
| 1446 | cmpxchg(&aio_req->error, 0, rc);
|
| 1447 |
|
| 1448 | ceph_aio_complete(inode, aio_req);
|
| 1449 | return;
|
| 1450 | }
|
| 1451 |
|
| 1452 | static void ceph_aio_retry_work(struct work_struct *work)
|
| 1453 | {
|
| 1454 | struct ceph_aio_work *aio_work =
|
| 1455 | container_of(work, struct ceph_aio_work, work);
|
| 1456 | struct ceph_osd_request *orig_req = aio_work->req;
|
| 1457 | struct ceph_aio_request *aio_req = orig_req->r_priv;
|
| 1458 | struct inode *inode = orig_req->r_inode;
|
| 1459 | struct ceph_inode_info *ci = ceph_inode(inode);
|
| 1460 | struct ceph_snap_context *snapc;
|
| 1461 | struct ceph_osd_request *req;
|
| 1462 | int ret;
|
| 1463 |
|
| 1464 | spin_lock(&ci->i_ceph_lock);
|
| 1465 | if (__ceph_have_pending_cap_snap(ci)) {
|
| 1466 | struct ceph_cap_snap *capsnap =
|
| 1467 | list_last_entry(&ci->i_cap_snaps,
|
| 1468 | struct ceph_cap_snap,
|
| 1469 | ci_item);
|
| 1470 | snapc = ceph_get_snap_context(capsnap->context);
|
| 1471 | } else {
|
| 1472 | BUG_ON(!ci->i_head_snapc);
|
| 1473 | snapc = ceph_get_snap_context(ci->i_head_snapc);
|
| 1474 | }
|
| 1475 | spin_unlock(&ci->i_ceph_lock);
|
| 1476 |
|
| 1477 | req = ceph_osdc_alloc_request(orig_req->r_osdc, snapc, 1,
|
| 1478 | false, GFP_NOFS);
|
| 1479 | if (!req) {
|
| 1480 | ret = -ENOMEM;
|
| 1481 | req = orig_req;
|
| 1482 | goto out;
|
| 1483 | }
|
| 1484 |
|
| 1485 | req->r_flags = CEPH_OSD_FLAG_WRITE;
|
| 1486 | ceph_oloc_copy(&req->r_base_oloc, &orig_req->r_base_oloc);
|
| 1487 | ceph_oid_copy(&req->r_base_oid, &orig_req->r_base_oid);
|
| 1488 |
|
| 1489 | req->r_ops[0] = orig_req->r_ops[0];
|
| 1490 |
|
| 1491 | req->r_mtime = aio_req->mtime;
|
| 1492 | req->r_data_offset = req->r_ops[0].extent.offset;
|
| 1493 |
|
| 1494 | ret = ceph_osdc_alloc_messages(req, GFP_NOFS);
|
| 1495 | if (ret) {
|
| 1496 | ceph_osdc_put_request(req);
|
| 1497 | req = orig_req;
|
| 1498 | goto out;
|
| 1499 | }
|
| 1500 |
|
| 1501 | ceph_osdc_put_request(orig_req);
|
| 1502 |
|
| 1503 | req->r_callback = ceph_aio_complete_req;
|
| 1504 | req->r_inode = inode;
|
| 1505 | req->r_priv = aio_req;
|
| 1506 |
|
| 1507 | ceph_osdc_start_request(req->r_osdc, req);
|
| 1508 | out:
|
| 1509 | if (ret < 0) {
|
| 1510 | req->r_result = ret;
|
| 1511 | ceph_aio_complete_req(req);
|
| 1512 | }
|
| 1513 |
|
| 1514 | ceph_put_snap_context(snapc);
|
| 1515 | kfree(aio_work);
|
| 1516 | }
|
| 1517 |
|
| 1518 | static ssize_t
|
| 1519 | ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter,
|
| 1520 | struct ceph_snap_context *snapc,
|
| 1521 | struct ceph_cap_flush **pcf)
|
| 1522 | {
|
| 1523 | struct file *file = iocb->ki_filp;
|
| 1524 | struct inode *inode = file_inode(file);
|
| 1525 | struct ceph_inode_info *ci = ceph_inode(inode);
|
| 1526 | struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
|
| 1527 | struct ceph_client *cl = fsc->client;
|
| 1528 | struct ceph_client_metric *metric = &fsc->mdsc->metric;
|
| 1529 | struct ceph_vino vino;
|
| 1530 | struct ceph_osd_request *req;
|
| 1531 | struct bio_vec *bvecs;
|
| 1532 | struct ceph_aio_request *aio_req = NULL;
|
| 1533 | int num_pages = 0;
|
| 1534 | int flags;
|
| 1535 | int ret = 0;
|
| 1536 | struct timespec64 mtime = current_time(inode);
|
| 1537 | size_t count = iov_iter_count(iter);
|
| 1538 | loff_t pos = iocb->ki_pos;
|
| 1539 | bool write = iov_iter_rw(iter) == WRITE;
|
| 1540 | bool should_dirty = !write && user_backed_iter(iter);
|
| 1541 | bool sparse = ceph_test_mount_opt(fsc, SPARSEREAD);
|
| 1542 |
|
| 1543 | if (write && ceph_snap(file_inode(file)) != CEPH_NOSNAP)
|
| 1544 | return -EROFS;
|
| 1545 |
|
| 1546 | doutc(cl, "sync_direct_%s on file %p %lld~%u snapc %p seq %lld\n",
|
| 1547 | (write ? "write" : "read"), file, pos, (unsigned)count,
|
| 1548 | snapc, snapc ? snapc->seq : 0);
|
| 1549 |
|
| 1550 | if (write) {
|
| 1551 | int ret2;
|
| 1552 |
|
| 1553 | ceph_fscache_invalidate(inode, true);
|
| 1554 |
|
| 1555 | ret2 = invalidate_inode_pages2_range(inode->i_mapping,
|
| 1556 | pos >> PAGE_SHIFT,
|
| 1557 | (pos + count - 1) >> PAGE_SHIFT);
|
| 1558 | if (ret2 < 0)
|
| 1559 | doutc(cl, "invalidate_inode_pages2_range returned %d\n",
|
| 1560 | ret2);
|
| 1561 |
|
| 1562 | flags = CEPH_OSD_FLAG_WRITE;
|
| 1563 | } else {
|
| 1564 | flags = CEPH_OSD_FLAG_READ;
|
| 1565 | }
|
| 1566 |
|
| 1567 | while (iov_iter_count(iter) > 0) {
|
| 1568 | u64 size = iov_iter_count(iter);
|
| 1569 | ssize_t len;
|
| 1570 | struct ceph_osd_req_op *op;
|
| 1571 | int readop = sparse ? CEPH_OSD_OP_SPARSE_READ : CEPH_OSD_OP_READ;
|
| 1572 | int extent_cnt;
|
| 1573 |
|
| 1574 | if (write)
|
| 1575 | size = min_t(u64, size, fsc->mount_options->wsize);
|
| 1576 | else
|
| 1577 | size = min_t(u64, size, fsc->mount_options->rsize);
|
| 1578 |
|
| 1579 | vino = ceph_vino(inode);
|
| 1580 | req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout,
|
| 1581 | vino, pos, &size, 0,
|
| 1582 | 1,
|
| 1583 | write ? CEPH_OSD_OP_WRITE : readop,
|
| 1584 | flags, snapc,
|
| 1585 | ci->i_truncate_seq,
|
| 1586 | ci->i_truncate_size,
|
| 1587 | false);
|
| 1588 | if (IS_ERR(req)) {
|
| 1589 | ret = PTR_ERR(req);
|
| 1590 | break;
|
| 1591 | }
|
| 1592 |
|
| 1593 | op = &req->r_ops[0];
|
| 1594 | if (!write && sparse) {
|
| 1595 | extent_cnt = __ceph_sparse_read_ext_count(inode, size);
|
| 1596 | ret = ceph_alloc_sparse_ext_map(op, extent_cnt);
|
| 1597 | if (ret) {
|
| 1598 | ceph_osdc_put_request(req);
|
| 1599 | break;
|
| 1600 | }
|
| 1601 | }
|
| 1602 |
|
| 1603 | len = iter_get_bvecs_alloc(iter, size, &bvecs, &num_pages);
|
| 1604 | if (len < 0) {
|
| 1605 | ceph_osdc_put_request(req);
|
| 1606 | ret = len;
|
| 1607 | break;
|
| 1608 | }
|
| 1609 | if (len != size)
|
| 1610 | osd_req_op_extent_update(req, 0, len);
|
| 1611 |
|
| 1612 | osd_req_op_extent_osd_data_bvecs(req, 0, bvecs, num_pages, len);
|
| 1613 |
|
| 1614 | |
| 1615 | * To simplify error handling, allow AIO when IO within i_size
|
| 1616 | * or IO can be satisfied by single OSD request.
|
| 1617 |
|
| 1618 | if (pos == iocb->ki_pos && !is_sync_kiocb(iocb) &&
|
| 1619 | (len == count || pos + count <= i_size_read(inode))) {
|
| 1620 | aio_req = kzalloc_obj(*aio_req);
|
| 1621 | if (aio_req) {
|
| 1622 | aio_req->iocb = iocb;
|
| 1623 | aio_req->write = write;
|
| 1624 | aio_req->should_dirty = should_dirty;
|
| 1625 | INIT_LIST_HEAD(&aio_req->osd_reqs);
|
| 1626 | if (write) {
|
| 1627 | aio_req->mtime = mtime;
|
| 1628 | swap(aio_req->prealloc_cf, *pcf);
|
| 1629 | }
|
| 1630 | }
|
| 1631 |
|
| 1632 | }
|
| 1633 |
|
| 1634 | if (write) {
|
| 1635 | |
| 1636 | * throw out any page cache pages in this range. this
|
| 1637 | * may block.
|
| 1638 |
|
| 1639 | truncate_inode_pages_range(inode->i_mapping, pos,
|
| 1640 | PAGE_ALIGN(pos + len) - 1);
|
| 1641 |
|
| 1642 | req->r_mtime = mtime;
|
| 1643 | }
|
| 1644 |
|
| 1645 | if (aio_req) {
|
| 1646 | aio_req->total_len += len;
|
| 1647 | aio_req->num_reqs++;
|
| 1648 | atomic_inc(&aio_req->pending_reqs);
|
| 1649 |
|
| 1650 | req->r_callback = ceph_aio_complete_req;
|
| 1651 | req->r_inode = inode;
|
| 1652 | req->r_priv = aio_req;
|
| 1653 | list_add_tail(&req->r_private_item, &aio_req->osd_reqs);
|
| 1654 |
|
| 1655 | pos += len;
|
| 1656 | continue;
|
| 1657 | }
|
| 1658 |
|
| 1659 | ceph_osdc_start_request(req->r_osdc, req);
|
| 1660 | ret = ceph_osdc_wait_request(&fsc->client->osdc, req);
|
| 1661 |
|
| 1662 | if (write) {
|
| 1663 | ceph_update_write_metrics(metric, req->r_start_latency,
|
| 1664 | req->r_end_latency, len, ret);
|
| 1665 | if (ret >= 0 && len)
|
| 1666 | ceph_record_subvolume_io(inode, true,
|
| 1667 | req->r_start_latency,
|
| 1668 | req->r_end_latency,
|
| 1669 | len);
|
| 1670 | } else {
|
| 1671 | ceph_update_read_metrics(metric, req->r_start_latency,
|
| 1672 | req->r_end_latency, len, ret);
|
| 1673 | if (ret > 0)
|
| 1674 | ceph_record_subvolume_io(inode, false,
|
| 1675 | req->r_start_latency,
|
| 1676 | req->r_end_latency,
|
| 1677 | ret);
|
| 1678 | }
|
| 1679 |
|
| 1680 | size = i_size_read(inode);
|
| 1681 | if (!write) {
|
| 1682 | if (sparse && ret >= 0)
|
| 1683 | ret = ceph_sparse_ext_map_end(op);
|
| 1684 | else if (ret == -ENOENT)
|
| 1685 | ret = 0;
|
| 1686 |
|
| 1687 | if (ret >= 0 && ret < len && pos + ret < size) {
|
| 1688 | struct iov_iter i;
|
| 1689 | int zlen = min_t(size_t, len - ret,
|
| 1690 | size - pos - ret);
|
| 1691 |
|
| 1692 | iov_iter_bvec(&i, ITER_DEST, bvecs, num_pages, len);
|
| 1693 | iov_iter_advance(&i, ret);
|
| 1694 | iov_iter_zero(zlen, &i);
|
| 1695 | ret += zlen;
|
| 1696 | }
|
| 1697 | if (ret >= 0)
|
| 1698 | len = ret;
|
| 1699 | }
|
| 1700 |
|
| 1701 | put_bvecs(bvecs, num_pages, should_dirty);
|
| 1702 | ceph_osdc_put_request(req);
|
| 1703 | if (ret < 0)
|
| 1704 | break;
|
| 1705 |
|
| 1706 | pos += len;
|
| 1707 | if (!write && pos >= size)
|
| 1708 | break;
|
| 1709 |
|
| 1710 | if (write && pos > size) {
|
| 1711 | if (ceph_inode_set_size(inode, pos))
|
| 1712 | ceph_check_caps(ceph_inode(inode),
|
| 1713 | CHECK_CAPS_AUTHONLY);
|
| 1714 | }
|
| 1715 | }
|
| 1716 |
|
| 1717 | if (aio_req) {
|
| 1718 | LIST_HEAD(osd_reqs);
|
| 1719 |
|
| 1720 | if (aio_req->num_reqs == 0) {
|
| 1721 | kfree(aio_req);
|
| 1722 | return ret;
|
| 1723 | }
|
| 1724 |
|
| 1725 | ceph_get_cap_refs(ci, write ? CEPH_CAP_FILE_WR :
|
| 1726 | CEPH_CAP_FILE_RD);
|
| 1727 |
|
| 1728 | list_splice(&aio_req->osd_reqs, &osd_reqs);
|
| 1729 | inode_dio_begin(inode);
|
| 1730 | while (!list_empty(&osd_reqs)) {
|
| 1731 | req = list_first_entry(&osd_reqs,
|
| 1732 | struct ceph_osd_request,
|
| 1733 | r_private_item);
|
| 1734 | list_del_init(&req->r_private_item);
|
| 1735 | if (ret >= 0)
|
| 1736 | ceph_osdc_start_request(req->r_osdc, req);
|
| 1737 | if (ret < 0) {
|
| 1738 | req->r_result = ret;
|
| 1739 | ceph_aio_complete_req(req);
|
| 1740 | }
|
| 1741 | }
|
| 1742 | return -EIOCBQUEUED;
|
| 1743 | }
|
| 1744 |
|
| 1745 | if (ret != -EOLDSNAPC && pos > iocb->ki_pos) {
|
| 1746 | ret = pos - iocb->ki_pos;
|
| 1747 | iocb->ki_pos = pos;
|
| 1748 | }
|
| 1749 | return ret;
|
| 1750 | }
|
| 1751 |
|
| 1752 | |
| 1753 | * Synchronous write, straight from __user pointer or user pages.
|
| 1754 | *
|
| 1755 | * If write spans object boundary, just do multiple writes. (For a
|
| 1756 | * correct atomic write, we should e.g. take write locks on all
|
| 1757 | * objects, rollback on failure, etc.)
|
| 1758 |
|
| 1759 | static ssize_t
|
| 1760 | ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,
|
| 1761 | struct ceph_snap_context *snapc)
|
| 1762 | {
|
| 1763 | struct file *file = iocb->ki_filp;
|
| 1764 | struct inode *inode = file_inode(file);
|
| 1765 | struct ceph_inode_info *ci = ceph_inode(inode);
|
| 1766 | struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
|
| 1767 | struct ceph_client *cl = fsc->client;
|
| 1768 | struct ceph_osd_client *osdc = &fsc->client->osdc;
|
| 1769 | struct ceph_osd_request *req;
|
| 1770 | struct page **pages;
|
| 1771 | u64 len;
|
| 1772 | int num_pages;
|
| 1773 | int written = 0;
|
| 1774 | int ret;
|
| 1775 | bool check_caps = false;
|
| 1776 | struct timespec64 mtime = current_time(inode);
|
| 1777 | size_t count = iov_iter_count(from);
|
| 1778 |
|
| 1779 | if (ceph_snap(file_inode(file)) != CEPH_NOSNAP)
|
| 1780 | return -EROFS;
|
| 1781 |
|
| 1782 | doutc(cl, "on file %p %lld~%u snapc %p seq %lld\n", file, pos,
|
| 1783 | (unsigned)count, snapc, snapc->seq);
|
| 1784 |
|
| 1785 | ret = filemap_write_and_wait_range(inode->i_mapping,
|
| 1786 | pos, pos + count - 1);
|
| 1787 | if (ret < 0)
|
| 1788 | return ret;
|
| 1789 |
|
| 1790 | ceph_fscache_invalidate(inode, false);
|
| 1791 |
|
| 1792 | while ((len = iov_iter_count(from)) > 0) {
|
| 1793 | size_t left;
|
| 1794 | int n;
|
| 1795 | u64 write_pos = pos;
|
| 1796 | u64 write_len = len;
|
| 1797 | u64 objnum, objoff;
|
| 1798 | u32 xlen;
|
| 1799 | u64 assert_ver = 0;
|
| 1800 | bool rmw;
|
| 1801 | bool first, last;
|
| 1802 | struct iov_iter saved_iter = *from;
|
| 1803 | size_t off;
|
| 1804 |
|
| 1805 | ceph_fscrypt_adjust_off_and_len(inode, &write_pos, &write_len);
|
| 1806 |
|
| 1807 |
|
| 1808 | ceph_calc_file_object_mapping(&ci->i_layout, write_pos,
|
| 1809 | write_len, &objnum, &objoff,
|
| 1810 | &xlen);
|
| 1811 | write_len = xlen;
|
| 1812 |
|
| 1813 |
|
| 1814 | if (pos + len > write_pos + write_len)
|
| 1815 | len = write_pos + write_len - pos;
|
| 1816 |
|
| 1817 | |
| 1818 | * If we had to adjust the length or position to align with a
|
| 1819 | * crypto block, then we must do a read/modify/write cycle. We
|
| 1820 | * use a version assertion to redrive the thing if something
|
| 1821 | * changes in between.
|
| 1822 |
|
| 1823 | first = pos != write_pos;
|
| 1824 | last = (pos + len) != (write_pos + write_len);
|
| 1825 | rmw = first || last;
|
| 1826 |
|
| 1827 | doutc(cl, "ino %llx %lld~%llu adjusted %lld~%llu -- %srmw\n",
|
| 1828 | ci->i_vino.ino, pos, len, write_pos, write_len,
|
| 1829 | rmw ? "" : "no ");
|
| 1830 |
|
| 1831 | |
| 1832 | * The data is emplaced into the page as it would be if it were
|
| 1833 | * in an array of pagecache pages.
|
| 1834 |
|
| 1835 | num_pages = calc_pages_for(write_pos, write_len);
|
| 1836 | pages = ceph_alloc_page_vector(num_pages, GFP_KERNEL);
|
| 1837 | if (IS_ERR(pages)) {
|
| 1838 | ret = PTR_ERR(pages);
|
| 1839 | break;
|
| 1840 | }
|
| 1841 |
|
| 1842 |
|
| 1843 | if (rmw) {
|
| 1844 | u64 first_pos = write_pos;
|
| 1845 | u64 last_pos = (write_pos + write_len) - CEPH_FSCRYPT_BLOCK_SIZE;
|
| 1846 | u64 read_len = CEPH_FSCRYPT_BLOCK_SIZE;
|
| 1847 | struct ceph_osd_req_op *op;
|
| 1848 |
|
| 1849 |
|
| 1850 | WARN_ON_ONCE(!IS_ENCRYPTED(inode));
|
| 1851 |
|
| 1852 |
|
| 1853 | if (first && last_pos == first_pos)
|
| 1854 | last = false;
|
| 1855 |
|
| 1856 | |
| 1857 | * Allocate a read request for one or two extents,
|
| 1858 | * depending on how the request was aligned.
|
| 1859 |
|
| 1860 | req = ceph_osdc_new_request(osdc, &ci->i_layout,
|
| 1861 | ci->i_vino, first ? first_pos : last_pos,
|
| 1862 | &read_len, 0, (first && last) ? 2 : 1,
|
| 1863 | CEPH_OSD_OP_SPARSE_READ, CEPH_OSD_FLAG_READ,
|
| 1864 | NULL, ci->i_truncate_seq,
|
| 1865 | ci->i_truncate_size, false);
|
| 1866 | if (IS_ERR(req)) {
|
| 1867 | ceph_release_page_vector(pages, num_pages);
|
| 1868 | ret = PTR_ERR(req);
|
| 1869 | break;
|
| 1870 | }
|
| 1871 |
|
| 1872 |
|
| 1873 | if (read_len != CEPH_FSCRYPT_BLOCK_SIZE) {
|
| 1874 | ceph_osdc_put_request(req);
|
| 1875 | ceph_release_page_vector(pages, num_pages);
|
| 1876 | ret = -EIO;
|
| 1877 | break;
|
| 1878 | }
|
| 1879 |
|
| 1880 |
|
| 1881 | op = &req->r_ops[0];
|
| 1882 |
|
| 1883 | if (first) {
|
| 1884 | osd_req_op_extent_osd_data_pages(req, 0, pages,
|
| 1885 | CEPH_FSCRYPT_BLOCK_SIZE,
|
| 1886 | offset_in_page(first_pos),
|
| 1887 | false, false);
|
| 1888 |
|
| 1889 | ret = __ceph_alloc_sparse_ext_map(op, 1);
|
| 1890 | if (ret) {
|
| 1891 | ceph_osdc_put_request(req);
|
| 1892 | ceph_release_page_vector(pages, num_pages);
|
| 1893 | break;
|
| 1894 | }
|
| 1895 | }
|
| 1896 |
|
| 1897 |
|
| 1898 | if (last) {
|
| 1899 |
|
| 1900 | if (first) {
|
| 1901 | op = &req->r_ops[1];
|
| 1902 | osd_req_op_extent_init(req, 1,
|
| 1903 | CEPH_OSD_OP_SPARSE_READ,
|
| 1904 | last_pos, CEPH_FSCRYPT_BLOCK_SIZE,
|
| 1905 | ci->i_truncate_size,
|
| 1906 | ci->i_truncate_seq);
|
| 1907 | }
|
| 1908 |
|
| 1909 | ret = __ceph_alloc_sparse_ext_map(op, 1);
|
| 1910 | if (ret) {
|
| 1911 | ceph_osdc_put_request(req);
|
| 1912 | ceph_release_page_vector(pages, num_pages);
|
| 1913 | break;
|
| 1914 | }
|
| 1915 |
|
| 1916 | osd_req_op_extent_osd_data_pages(req, first ? 1 : 0,
|
| 1917 | &pages[num_pages - 1],
|
| 1918 | CEPH_FSCRYPT_BLOCK_SIZE,
|
| 1919 | offset_in_page(last_pos),
|
| 1920 | false, false);
|
| 1921 | }
|
| 1922 |
|
| 1923 | ceph_osdc_start_request(osdc, req);
|
| 1924 | ret = ceph_osdc_wait_request(osdc, req);
|
| 1925 |
|
| 1926 |
|
| 1927 | ceph_update_read_metrics(&fsc->mdsc->metric,
|
| 1928 | req->r_start_latency,
|
| 1929 | req->r_end_latency,
|
| 1930 | read_len, ret);
|
| 1931 | if (ret > 0)
|
| 1932 | ceph_record_subvolume_io(inode, false,
|
| 1933 | req->r_start_latency,
|
| 1934 | req->r_end_latency,
|
| 1935 | ret);
|
| 1936 |
|
| 1937 |
|
| 1938 | if (ret == -ENOENT) {
|
| 1939 | |
| 1940 | * If there is no object, then we can't assert
|
| 1941 | * on its version. Set it to 0, and we'll use an
|
| 1942 | * exclusive create instead.
|
| 1943 |
|
| 1944 | ceph_osdc_put_request(req);
|
| 1945 | ret = 0;
|
| 1946 |
|
| 1947 | |
| 1948 | * zero out the soon-to-be uncopied parts of the
|
| 1949 | * first and last pages.
|
| 1950 |
|
| 1951 | if (first)
|
| 1952 | zero_user_segment(pages[0], 0,
|
| 1953 | offset_in_page(first_pos));
|
| 1954 | if (last)
|
| 1955 | zero_user_segment(pages[num_pages - 1],
|
| 1956 | offset_in_page(last_pos),
|
| 1957 | PAGE_SIZE);
|
| 1958 | } else {
|
| 1959 | if (ret < 0) {
|
| 1960 | ceph_osdc_put_request(req);
|
| 1961 | ceph_release_page_vector(pages, num_pages);
|
| 1962 | break;
|
| 1963 | }
|
| 1964 |
|
| 1965 | op = &req->r_ops[0];
|
| 1966 | if (op->extent.sparse_ext_cnt == 0) {
|
| 1967 | if (first)
|
| 1968 | zero_user_segment(pages[0], 0,
|
| 1969 | offset_in_page(first_pos));
|
| 1970 | else
|
| 1971 | zero_user_segment(pages[num_pages - 1],
|
| 1972 | offset_in_page(last_pos),
|
| 1973 | PAGE_SIZE);
|
| 1974 | } else if (op->extent.sparse_ext_cnt != 1 ||
|
| 1975 | ceph_sparse_ext_map_end(op) !=
|
| 1976 | CEPH_FSCRYPT_BLOCK_SIZE) {
|
| 1977 | ret = -EIO;
|
| 1978 | ceph_osdc_put_request(req);
|
| 1979 | ceph_release_page_vector(pages, num_pages);
|
| 1980 | break;
|
| 1981 | }
|
| 1982 |
|
| 1983 | if (first && last) {
|
| 1984 | op = &req->r_ops[1];
|
| 1985 | if (op->extent.sparse_ext_cnt == 0) {
|
| 1986 | zero_user_segment(pages[num_pages - 1],
|
| 1987 | offset_in_page(last_pos),
|
| 1988 | PAGE_SIZE);
|
| 1989 | } else if (op->extent.sparse_ext_cnt != 1 ||
|
| 1990 | ceph_sparse_ext_map_end(op) !=
|
| 1991 | CEPH_FSCRYPT_BLOCK_SIZE) {
|
| 1992 | ret = -EIO;
|
| 1993 | ceph_osdc_put_request(req);
|
| 1994 | ceph_release_page_vector(pages, num_pages);
|
| 1995 | break;
|
| 1996 | }
|
| 1997 | }
|
| 1998 |
|
| 1999 |
|
| 2000 | assert_ver = req->r_version;
|
| 2001 | WARN_ON_ONCE(ret > 0 && assert_ver == 0);
|
| 2002 |
|
| 2003 | ceph_osdc_put_request(req);
|
| 2004 | if (first) {
|
| 2005 | ret = ceph_fscrypt_decrypt_block_inplace(inode,
|
| 2006 | pages[0], CEPH_FSCRYPT_BLOCK_SIZE,
|
| 2007 | offset_in_page(first_pos),
|
| 2008 | first_pos >> CEPH_FSCRYPT_BLOCK_SHIFT);
|
| 2009 | if (ret < 0) {
|
| 2010 | ceph_release_page_vector(pages, num_pages);
|
| 2011 | break;
|
| 2012 | }
|
| 2013 | }
|
| 2014 | if (last) {
|
| 2015 | ret = ceph_fscrypt_decrypt_block_inplace(inode,
|
| 2016 | pages[num_pages - 1],
|
| 2017 | CEPH_FSCRYPT_BLOCK_SIZE,
|
| 2018 | offset_in_page(last_pos),
|
| 2019 | last_pos >> CEPH_FSCRYPT_BLOCK_SHIFT);
|
| 2020 | if (ret < 0) {
|
| 2021 | ceph_release_page_vector(pages, num_pages);
|
| 2022 | break;
|
| 2023 | }
|
| 2024 | }
|
| 2025 | }
|
| 2026 | }
|
| 2027 |
|
| 2028 | left = len;
|
| 2029 | off = offset_in_page(pos);
|
| 2030 | for (n = 0; n < num_pages; n++) {
|
| 2031 | size_t plen = min_t(size_t, left, PAGE_SIZE - off);
|
| 2032 |
|
| 2033 |
|
| 2034 | ret = copy_page_from_iter(pages[n], off, plen, from);
|
| 2035 | if (ret != plen) {
|
| 2036 | ret = -EFAULT;
|
| 2037 | break;
|
| 2038 | }
|
| 2039 | off = 0;
|
| 2040 | left -= ret;
|
| 2041 | }
|
| 2042 | if (ret < 0) {
|
| 2043 | doutc(cl, "write failed with %d\n", ret);
|
| 2044 | ceph_release_page_vector(pages, num_pages);
|
| 2045 | break;
|
| 2046 | }
|
| 2047 |
|
| 2048 | if (IS_ENCRYPTED(inode)) {
|
| 2049 | ret = ceph_fscrypt_encrypt_pages(inode, pages,
|
| 2050 | write_pos, write_len);
|
| 2051 | if (ret < 0) {
|
| 2052 | doutc(cl, "encryption failed with %d\n", ret);
|
| 2053 | ceph_release_page_vector(pages, num_pages);
|
| 2054 | break;
|
| 2055 | }
|
| 2056 | }
|
| 2057 |
|
| 2058 | req = ceph_osdc_new_request(osdc, &ci->i_layout,
|
| 2059 | ci->i_vino, write_pos, &write_len,
|
| 2060 | rmw ? 1 : 0, rmw ? 2 : 1,
|
| 2061 | CEPH_OSD_OP_WRITE,
|
| 2062 | CEPH_OSD_FLAG_WRITE,
|
| 2063 | snapc, ci->i_truncate_seq,
|
| 2064 | ci->i_truncate_size, false);
|
| 2065 | if (IS_ERR(req)) {
|
| 2066 | ret = PTR_ERR(req);
|
| 2067 | ceph_release_page_vector(pages, num_pages);
|
| 2068 | break;
|
| 2069 | }
|
| 2070 |
|
| 2071 | doutc(cl, "write op %lld~%llu\n", write_pos, write_len);
|
| 2072 | osd_req_op_extent_osd_data_pages(req, rmw ? 1 : 0, pages, write_len,
|
| 2073 | offset_in_page(write_pos), false,
|
| 2074 | true);
|
| 2075 | req->r_inode = inode;
|
| 2076 | req->r_mtime = mtime;
|
| 2077 |
|
| 2078 |
|
| 2079 | if (rmw) {
|
| 2080 | |
| 2081 | * Set up the assertion. If we don't have a version
|
| 2082 | * number, then the object doesn't exist yet. Use an
|
| 2083 | * exclusive create instead of a version assertion in
|
| 2084 | * that case.
|
| 2085 |
|
| 2086 | if (assert_ver) {
|
| 2087 | osd_req_op_init(req, 0, CEPH_OSD_OP_ASSERT_VER, 0);
|
| 2088 | req->r_ops[0].assert_ver.ver = assert_ver;
|
| 2089 | } else {
|
| 2090 | osd_req_op_init(req, 0, CEPH_OSD_OP_CREATE,
|
| 2091 | CEPH_OSD_OP_FLAG_EXCL);
|
| 2092 | }
|
| 2093 | }
|
| 2094 |
|
| 2095 | ceph_osdc_start_request(osdc, req);
|
| 2096 | ret = ceph_osdc_wait_request(osdc, req);
|
| 2097 |
|
| 2098 | ceph_update_write_metrics(&fsc->mdsc->metric, req->r_start_latency,
|
| 2099 | req->r_end_latency, len, ret);
|
| 2100 | if (ret >= 0 && write_len)
|
| 2101 | ceph_record_subvolume_io(inode, true,
|
| 2102 | req->r_start_latency,
|
| 2103 | req->r_end_latency,
|
| 2104 | write_len);
|
| 2105 | ceph_osdc_put_request(req);
|
| 2106 | if (ret != 0) {
|
| 2107 | doutc(cl, "osd write returned %d\n", ret);
|
| 2108 |
|
| 2109 | if ((assert_ver && (ret == -ERANGE || ret == -EOVERFLOW)) ||
|
| 2110 | (!assert_ver && ret == -EEXIST)) {
|
| 2111 |
|
| 2112 | WARN_ON_ONCE(!rmw);
|
| 2113 |
|
| 2114 |
|
| 2115 | WARN_ON_ONCE(ret == -EOVERFLOW);
|
| 2116 |
|
| 2117 | *from = saved_iter;
|
| 2118 |
|
| 2119 |
|
| 2120 | continue;
|
| 2121 | }
|
| 2122 | ceph_set_error_write(ci);
|
| 2123 | break;
|
| 2124 | }
|
| 2125 |
|
| 2126 | ceph_clear_error_write(ci);
|
| 2127 |
|
| 2128 | |
| 2129 | * We successfully wrote to a range of the file. Declare
|
| 2130 | * that region of the pagecache invalid.
|
| 2131 |
|
| 2132 | ret = invalidate_inode_pages2_range(
|
| 2133 | inode->i_mapping,
|
| 2134 | pos >> PAGE_SHIFT,
|
| 2135 | (pos + len - 1) >> PAGE_SHIFT);
|
| 2136 | if (ret < 0) {
|
| 2137 | doutc(cl, "invalidate_inode_pages2_range returned %d\n",
|
| 2138 | ret);
|
| 2139 | ret = 0;
|
| 2140 | }
|
| 2141 | pos += len;
|
| 2142 | written += len;
|
| 2143 | doutc(cl, "written %d\n", written);
|
| 2144 | if (pos > i_size_read(inode)) {
|
| 2145 | check_caps = ceph_inode_set_size(inode, pos);
|
| 2146 | if (check_caps)
|
| 2147 | ceph_check_caps(ceph_inode(inode),
|
| 2148 | CHECK_CAPS_AUTHONLY);
|
| 2149 | }
|
| 2150 |
|
| 2151 | }
|
| 2152 |
|
| 2153 | if (ret != -EOLDSNAPC && written > 0) {
|
| 2154 | ret = written;
|
| 2155 | iocb->ki_pos = pos;
|
| 2156 | }
|
| 2157 | doutc(cl, "returning %d\n", ret);
|
| 2158 | return ret;
|
| 2159 | }
|
| 2160 |
|
| 2161 | |
| 2162 | * Wrap generic_file_aio_read with checks for cap bits on the inode.
|
| 2163 | * Atomically grab references, so that those bits are not released
|
| 2164 | * back to the MDS mid-read.
|
| 2165 | *
|
| 2166 | * Hmm, the sync read case isn't actually async... should it be?
|
| 2167 |
|
| 2168 | static ssize_t ceph_read_iter(struct kiocb *iocb, struct iov_iter *to)
|
| 2169 | {
|
| 2170 | struct file *filp = iocb->ki_filp;
|
| 2171 | struct ceph_file_info *fi = filp->private_data;
|
| 2172 | size_t len = iov_iter_count(to);
|
| 2173 | struct inode *inode = file_inode(filp);
|
| 2174 | struct ceph_inode_info *ci = ceph_inode(inode);
|
| 2175 | bool direct_lock = iocb->ki_flags & IOCB_DIRECT;
|
| 2176 | struct ceph_client *cl = ceph_inode_to_client(inode);
|
| 2177 | ssize_t ret;
|
| 2178 | int want = 0, got = 0;
|
| 2179 | int retry_op = 0, read = 0;
|
| 2180 |
|
| 2181 | again:
|
| 2182 | doutc(cl, "%llu~%u trying to get caps on %p %llx.%llx\n",
|
| 2183 | iocb->ki_pos, (unsigned)len, inode, ceph_vinop(inode));
|
| 2184 |
|
| 2185 | if (ceph_inode_is_shutdown(inode))
|
| 2186 | return -ESTALE;
|
| 2187 |
|
| 2188 | ret = direct_lock ? ceph_start_io_direct(inode) :
|
| 2189 | ceph_start_io_read(inode);
|
| 2190 | if (ret)
|
| 2191 | return ret;
|
| 2192 |
|
| 2193 | if (!(fi->flags & CEPH_F_SYNC) && !direct_lock)
|
| 2194 | want |= CEPH_CAP_FILE_CACHE;
|
| 2195 | if (fi->fmode & CEPH_FILE_MODE_LAZY)
|
| 2196 | want |= CEPH_CAP_FILE_LAZYIO;
|
| 2197 |
|
| 2198 | ret = ceph_get_caps(filp, CEPH_CAP_FILE_RD, want, -1, &got);
|
| 2199 | if (ret < 0) {
|
| 2200 | if (direct_lock)
|
| 2201 | ceph_end_io_direct(inode);
|
| 2202 | else
|
| 2203 | ceph_end_io_read(inode);
|
| 2204 | return ret;
|
| 2205 | }
|
| 2206 |
|
| 2207 | if ((got & (CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO)) == 0 ||
|
| 2208 | (iocb->ki_flags & IOCB_DIRECT) ||
|
| 2209 | (fi->flags & CEPH_F_SYNC)) {
|
| 2210 |
|
| 2211 | doutc(cl, "sync %p %llx.%llx %llu~%u got cap refs on %s\n",
|
| 2212 | inode, ceph_vinop(inode), iocb->ki_pos, (unsigned)len,
|
| 2213 | ceph_cap_string(got));
|
| 2214 |
|
| 2215 | if (!ceph_has_inline_data(ci)) {
|
| 2216 | if (!retry_op &&
|
| 2217 | (iocb->ki_flags & IOCB_DIRECT) &&
|
| 2218 | !IS_ENCRYPTED(inode)) {
|
| 2219 | ret = ceph_direct_read_write(iocb, to,
|
| 2220 | NULL, NULL);
|
| 2221 | if (ret >= 0 && ret < len)
|
| 2222 | retry_op = CHECK_EOF;
|
| 2223 | } else {
|
| 2224 | ret = ceph_sync_read(iocb, to, &retry_op);
|
| 2225 | }
|
| 2226 | } else {
|
| 2227 | retry_op = READ_INLINE;
|
| 2228 | }
|
| 2229 | } else {
|
| 2230 | CEPH_DEFINE_RW_CONTEXT(rw_ctx, got);
|
| 2231 | doutc(cl, "async %p %llx.%llx %llu~%u got cap refs on %s\n",
|
| 2232 | inode, ceph_vinop(inode), iocb->ki_pos, (unsigned)len,
|
| 2233 | ceph_cap_string(got));
|
| 2234 | ceph_add_rw_context(fi, &rw_ctx);
|
| 2235 | ret = generic_file_read_iter(iocb, to);
|
| 2236 | ceph_del_rw_context(fi, &rw_ctx);
|
| 2237 | }
|
| 2238 |
|
| 2239 | doutc(cl, "%p %llx.%llx dropping cap refs on %s = %d\n",
|
| 2240 | inode, ceph_vinop(inode), ceph_cap_string(got), (int)ret);
|
| 2241 | ceph_put_cap_refs(ci, got);
|
| 2242 |
|
| 2243 | if (direct_lock)
|
| 2244 | ceph_end_io_direct(inode);
|
| 2245 | else
|
| 2246 | ceph_end_io_read(inode);
|
| 2247 |
|
| 2248 | if (retry_op > HAVE_RETRIED && ret >= 0) {
|
| 2249 | int statret;
|
| 2250 | struct page *page = NULL;
|
| 2251 | loff_t i_size;
|
| 2252 | int mask = CEPH_STAT_CAP_SIZE;
|
| 2253 | if (retry_op == READ_INLINE) {
|
| 2254 | page = __page_cache_alloc(GFP_KERNEL);
|
| 2255 | if (!page)
|
| 2256 | return -ENOMEM;
|
| 2257 |
|
| 2258 | mask = CEPH_STAT_CAP_INLINE_DATA;
|
| 2259 | }
|
| 2260 |
|
| 2261 | statret = __ceph_do_getattr(inode, page, mask, !!page);
|
| 2262 | if (statret < 0) {
|
| 2263 | if (page)
|
| 2264 | __free_page(page);
|
| 2265 | if (statret == -ENODATA) {
|
| 2266 | BUG_ON(retry_op != READ_INLINE);
|
| 2267 | goto again;
|
| 2268 | }
|
| 2269 | return statret;
|
| 2270 | }
|
| 2271 |
|
| 2272 | i_size = i_size_read(inode);
|
| 2273 | if (retry_op == READ_INLINE) {
|
| 2274 | BUG_ON(ret > 0 || read > 0);
|
| 2275 | if (iocb->ki_pos < i_size &&
|
| 2276 | iocb->ki_pos < PAGE_SIZE) {
|
| 2277 | loff_t end = min_t(loff_t, i_size,
|
| 2278 | iocb->ki_pos + len);
|
| 2279 | end = min_t(loff_t, end, PAGE_SIZE);
|
| 2280 | if (statret < end)
|
| 2281 | zero_user_segment(page, statret, end);
|
| 2282 | ret = copy_page_to_iter(page,
|
| 2283 | iocb->ki_pos & ~PAGE_MASK,
|
| 2284 | end - iocb->ki_pos, to);
|
| 2285 | iocb->ki_pos += ret;
|
| 2286 | read += ret;
|
| 2287 | }
|
| 2288 | if (iocb->ki_pos < i_size && read < len) {
|
| 2289 | size_t zlen = min_t(size_t, len - read,
|
| 2290 | i_size - iocb->ki_pos);
|
| 2291 | ret = iov_iter_zero(zlen, to);
|
| 2292 | iocb->ki_pos += ret;
|
| 2293 | read += ret;
|
| 2294 | }
|
| 2295 | __free_pages(page, 0);
|
| 2296 | return read;
|
| 2297 | }
|
| 2298 |
|
| 2299 |
|
| 2300 | if (retry_op == CHECK_EOF && iocb->ki_pos < i_size &&
|
| 2301 | ret < len) {
|
| 2302 | doutc(cl, "may hit hole, ppos %lld < size %lld, reading more\n",
|
| 2303 | iocb->ki_pos, i_size);
|
| 2304 |
|
| 2305 | read += ret;
|
| 2306 | len -= ret;
|
| 2307 | retry_op = HAVE_RETRIED;
|
| 2308 | goto again;
|
| 2309 | }
|
| 2310 | }
|
| 2311 |
|
| 2312 | if (ret >= 0)
|
| 2313 | ret += read;
|
| 2314 |
|
| 2315 | return ret;
|
| 2316 | }
|
| 2317 |
|
| 2318 | |
| 2319 | * Wrap filemap_splice_read with checks for cap bits on the inode.
|
| 2320 | * Atomically grab references, so that those bits are not released
|
| 2321 | * back to the MDS mid-read.
|
| 2322 |
|
| 2323 | static ssize_t ceph_splice_read(struct file *in, loff_t *ppos,
|
| 2324 | struct pipe_inode_info *pipe,
|
| 2325 | size_t len, unsigned int flags)
|
| 2326 | {
|
| 2327 | struct ceph_file_info *fi = in->private_data;
|
| 2328 | struct inode *inode = file_inode(in);
|
| 2329 | struct ceph_inode_info *ci = ceph_inode(inode);
|
| 2330 | ssize_t ret;
|
| 2331 | int want = 0, got = 0;
|
| 2332 | CEPH_DEFINE_RW_CONTEXT(rw_ctx, 0);
|
| 2333 |
|
| 2334 | dout("splice_read %p %llx.%llx %llu~%zu trying to get caps on %p\n",
|
| 2335 | inode, ceph_vinop(inode), *ppos, len, inode);
|
| 2336 |
|
| 2337 | if (ceph_inode_is_shutdown(inode))
|
| 2338 | return -ESTALE;
|
| 2339 |
|
| 2340 | if (ceph_has_inline_data(ci) ||
|
| 2341 | (fi->flags & CEPH_F_SYNC))
|
| 2342 | return copy_splice_read(in, ppos, pipe, len, flags);
|
| 2343 |
|
| 2344 | ret = ceph_start_io_read(inode);
|
| 2345 | if (ret)
|
| 2346 | return ret;
|
| 2347 |
|
| 2348 | want = CEPH_CAP_FILE_CACHE;
|
| 2349 | if (fi->fmode & CEPH_FILE_MODE_LAZY)
|
| 2350 | want |= CEPH_CAP_FILE_LAZYIO;
|
| 2351 |
|
| 2352 | ret = ceph_get_caps(in, CEPH_CAP_FILE_RD, want, -1, &got);
|
| 2353 | if (ret < 0)
|
| 2354 | goto out_end;
|
| 2355 |
|
| 2356 | if ((got & (CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO)) == 0) {
|
| 2357 | dout("splice_read/sync %p %llx.%llx %llu~%zu got cap refs on %s\n",
|
| 2358 | inode, ceph_vinop(inode), *ppos, len,
|
| 2359 | ceph_cap_string(got));
|
| 2360 |
|
| 2361 | ceph_put_cap_refs(ci, got);
|
| 2362 | ceph_end_io_read(inode);
|
| 2363 | return copy_splice_read(in, ppos, pipe, len, flags);
|
| 2364 | }
|
| 2365 |
|
| 2366 | dout("splice_read %p %llx.%llx %llu~%zu got cap refs on %s\n",
|
| 2367 | inode, ceph_vinop(inode), *ppos, len, ceph_cap_string(got));
|
| 2368 |
|
| 2369 | rw_ctx.caps = got;
|
| 2370 | ceph_add_rw_context(fi, &rw_ctx);
|
| 2371 | ret = filemap_splice_read(in, ppos, pipe, len, flags);
|
| 2372 | ceph_del_rw_context(fi, &rw_ctx);
|
| 2373 |
|
| 2374 | dout("splice_read %p %llx.%llx dropping cap refs on %s = %zd\n",
|
| 2375 | inode, ceph_vinop(inode), ceph_cap_string(got), ret);
|
| 2376 |
|
| 2377 | ceph_put_cap_refs(ci, got);
|
| 2378 | out_end:
|
| 2379 | ceph_end_io_read(inode);
|
| 2380 | return ret;
|
| 2381 | }
|
| 2382 |
|
| 2383 | |
| 2384 | * Take cap references to avoid releasing caps to MDS mid-write.
|
| 2385 | *
|
| 2386 | * If we are synchronous, and write with an old snap context, the OSD
|
| 2387 | * may return EOLDSNAPC. In that case, retry the write.. _after_
|
| 2388 | * dropping our cap refs and allowing the pending snap to logically
|
| 2389 | * complete _before_ this write occurs.
|
| 2390 | *
|
| 2391 | * If requested, nearfull writes are synced to preserve the legacy
|
| 2392 | * client-side backpressure behavior.
|
| 2393 |
|
| 2394 | static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from)
|
| 2395 | {
|
| 2396 | struct file *file = iocb->ki_filp;
|
| 2397 | struct ceph_file_info *fi = file->private_data;
|
| 2398 | struct inode *inode = file_inode(file);
|
| 2399 | struct ceph_inode_info *ci = ceph_inode(inode);
|
| 2400 | struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
|
| 2401 | struct ceph_client *cl = fsc->client;
|
| 2402 | struct ceph_osd_client *osdc = &fsc->client->osdc;
|
| 2403 | struct ceph_cap_flush *prealloc_cf;
|
| 2404 | ssize_t count, written = 0;
|
| 2405 | int err, want = 0, got;
|
| 2406 | bool direct_lock = false;
|
| 2407 | u32 map_flags;
|
| 2408 | u64 pool_flags;
|
| 2409 | loff_t pos;
|
| 2410 | loff_t limit = max(i_size_read(inode), fsc->max_file_size);
|
| 2411 |
|
| 2412 | if (ceph_inode_is_shutdown(inode))
|
| 2413 | return -ESTALE;
|
| 2414 |
|
| 2415 | if (ceph_snap(inode) != CEPH_NOSNAP)
|
| 2416 | return -EROFS;
|
| 2417 |
|
| 2418 | prealloc_cf = ceph_alloc_cap_flush();
|
| 2419 | if (!prealloc_cf)
|
| 2420 | return -ENOMEM;
|
| 2421 |
|
| 2422 | if ((iocb->ki_flags & (IOCB_DIRECT | IOCB_APPEND)) == IOCB_DIRECT)
|
| 2423 | direct_lock = true;
|
| 2424 |
|
| 2425 | retry_snap:
|
| 2426 | err = direct_lock ? ceph_start_io_direct(inode) :
|
| 2427 | ceph_start_io_write(inode);
|
| 2428 | if (err)
|
| 2429 | goto out_unlocked;
|
| 2430 |
|
| 2431 | if (iocb->ki_flags & IOCB_APPEND) {
|
| 2432 | err = ceph_do_getattr(inode, CEPH_STAT_CAP_SIZE, false);
|
| 2433 | if (err < 0)
|
| 2434 | goto out;
|
| 2435 | }
|
| 2436 |
|
| 2437 | err = generic_write_checks(iocb, from);
|
| 2438 | if (err <= 0)
|
| 2439 | goto out;
|
| 2440 |
|
| 2441 | pos = iocb->ki_pos;
|
| 2442 | if (unlikely(pos >= limit)) {
|
| 2443 | err = -EFBIG;
|
| 2444 | goto out;
|
| 2445 | } else {
|
| 2446 | iov_iter_truncate(from, limit - pos);
|
| 2447 | }
|
| 2448 |
|
| 2449 | count = iov_iter_count(from);
|
| 2450 | if (ceph_quota_is_max_bytes_exceeded(inode, pos + count)) {
|
| 2451 | err = -EDQUOT;
|
| 2452 | goto out;
|
| 2453 | }
|
| 2454 |
|
| 2455 | down_read(&osdc->lock);
|
| 2456 | map_flags = osdc->osdmap->flags;
|
| 2457 | pool_flags = ceph_pg_pool_flags(osdc->osdmap, ci->i_layout.pool_id);
|
| 2458 | up_read(&osdc->lock);
|
| 2459 | if ((map_flags & CEPH_OSDMAP_FULL) ||
|
| 2460 | (pool_flags & CEPH_POOL_FLAG_FULL)) {
|
| 2461 | err = -ENOSPC;
|
| 2462 | goto out;
|
| 2463 | }
|
| 2464 |
|
| 2465 | err = file_remove_privs(file);
|
| 2466 | if (err)
|
| 2467 | goto out;
|
| 2468 |
|
| 2469 | doutc(cl, "%p %llx.%llx %llu~%zd getting caps. i_size %llu\n",
|
| 2470 | inode, ceph_vinop(inode), pos, count,
|
| 2471 | i_size_read(inode));
|
| 2472 | if (!(fi->flags & CEPH_F_SYNC) && !direct_lock)
|
| 2473 | want |= CEPH_CAP_FILE_BUFFER;
|
| 2474 | if (fi->fmode & CEPH_FILE_MODE_LAZY)
|
| 2475 | want |= CEPH_CAP_FILE_LAZYIO;
|
| 2476 | got = 0;
|
| 2477 | err = ceph_get_caps(file, CEPH_CAP_FILE_WR, want, pos + count, &got);
|
| 2478 | if (err < 0)
|
| 2479 | goto out;
|
| 2480 |
|
| 2481 | |
| 2482 | * For O_APPEND writes we may have waited for Fwx exclusive caps
|
| 2483 | * while the previous Fwx holder (another client) extended the
|
| 2484 | * file. i_size has been updated via the cap grant message from
|
| 2485 | * the MDS, but ki_pos is still the old EOF. Re-read i_size here
|
| 2486 | * (no extra MDS round-trip needed) and adjust ki_pos to the true
|
| 2487 | * EOF. Since we hold Fwx, no other client can change the file.
|
| 2488 |
|
| 2489 | if (iocb->ki_flags & IOCB_APPEND) {
|
| 2490 | loff_t cur_eof = i_size_read(inode);
|
| 2491 |
|
| 2492 | if (cur_eof != pos) {
|
| 2493 | doutc(cl,
|
| 2494 | "%p %llx.%llx O_APPEND: pos adjusted %lld -> %lld\n",
|
| 2495 | inode, ceph_vinop(inode), pos, cur_eof);
|
| 2496 | iocb->ki_pos = cur_eof;
|
| 2497 | pos = cur_eof;
|
| 2498 | if (pos >= limit) {
|
| 2499 | err = -EFBIG;
|
| 2500 | goto out_caps;
|
| 2501 | }
|
| 2502 | iov_iter_truncate(from, limit - pos);
|
| 2503 | count = iov_iter_count(from);
|
| 2504 |
|
| 2505 | |
| 2506 | * ceph_get_caps() validated the old endoff
|
| 2507 | * against i_max_size; adjusting ki_pos forward
|
| 2508 | * may have shifted the write range beyond the
|
| 2509 | * granted max_size. Re-check and truncate if
|
| 2510 | * necessary.
|
| 2511 |
|
| 2512 | spin_lock(&ci->i_ceph_lock);
|
| 2513 | if (pos + count > (loff_t)ci->i_max_size) {
|
| 2514 | loff_t max_size = ci->i_max_size;
|
| 2515 |
|
| 2516 | spin_unlock(&ci->i_ceph_lock);
|
| 2517 | if (pos >= max_size) {
|
| 2518 | err = -EFBIG;
|
| 2519 | goto out_caps;
|
| 2520 | }
|
| 2521 | iov_iter_truncate(from, max_size - pos);
|
| 2522 | count = iov_iter_count(from);
|
| 2523 | } else {
|
| 2524 | spin_unlock(&ci->i_ceph_lock);
|
| 2525 | }
|
| 2526 | }
|
| 2527 | }
|
| 2528 |
|
| 2529 | err = file_update_time(file);
|
| 2530 | if (err)
|
| 2531 | goto out_caps;
|
| 2532 |
|
| 2533 | inode_inc_iversion_raw(inode);
|
| 2534 |
|
| 2535 | doutc(cl, "%p %llx.%llx %llu~%zd got cap refs on %s\n",
|
| 2536 | inode, ceph_vinop(inode), pos, count, ceph_cap_string(got));
|
| 2537 |
|
| 2538 | if ((got & (CEPH_CAP_FILE_BUFFER|CEPH_CAP_FILE_LAZYIO)) == 0 ||
|
| 2539 | (iocb->ki_flags & IOCB_DIRECT) || (fi->flags & CEPH_F_SYNC) ||
|
| 2540 | test_bit(CEPH_I_ERROR_WRITE_BIT, &ci->i_ceph_flags)) {
|
| 2541 | struct ceph_snap_context *snapc;
|
| 2542 | struct iov_iter data;
|
| 2543 |
|
| 2544 | spin_lock(&ci->i_ceph_lock);
|
| 2545 | if (__ceph_have_pending_cap_snap(ci)) {
|
| 2546 | struct ceph_cap_snap *capsnap =
|
| 2547 | list_last_entry(&ci->i_cap_snaps,
|
| 2548 | struct ceph_cap_snap,
|
| 2549 | ci_item);
|
| 2550 | snapc = ceph_get_snap_context(capsnap->context);
|
| 2551 | } else {
|
| 2552 | BUG_ON(!ci->i_head_snapc);
|
| 2553 | snapc = ceph_get_snap_context(ci->i_head_snapc);
|
| 2554 | }
|
| 2555 | spin_unlock(&ci->i_ceph_lock);
|
| 2556 |
|
| 2557 |
|
| 2558 | data = *from;
|
| 2559 | if ((iocb->ki_flags & IOCB_DIRECT) && !IS_ENCRYPTED(inode))
|
| 2560 | written = ceph_direct_read_write(iocb, &data, snapc,
|
| 2561 | &prealloc_cf);
|
| 2562 | else
|
| 2563 | written = ceph_sync_write(iocb, &data, pos, snapc);
|
| 2564 | if (direct_lock)
|
| 2565 | ceph_end_io_direct(inode);
|
| 2566 | else
|
| 2567 | ceph_end_io_write(inode);
|
| 2568 | if (written > 0)
|
| 2569 | iov_iter_advance(from, written);
|
| 2570 | ceph_put_snap_context(snapc);
|
| 2571 | } else {
|
| 2572 | |
| 2573 | * No need to acquire the i_truncate_mutex. Because
|
| 2574 | * the MDS revokes Fwb caps before sending truncate
|
| 2575 | * message to us. We can't get Fwb cap while there
|
| 2576 | * are pending vmtruncate. So write and vmtruncate
|
| 2577 | * can not run at the same time
|
| 2578 |
|
| 2579 | written = generic_perform_write(iocb, from);
|
| 2580 | ceph_end_io_write(inode);
|
| 2581 | }
|
| 2582 |
|
| 2583 | if (written >= 0) {
|
| 2584 | int dirty;
|
| 2585 |
|
| 2586 | spin_lock(&ci->i_ceph_lock);
|
| 2587 | dirty = __ceph_mark_dirty_caps(ci, CEPH_CAP_FILE_WR,
|
| 2588 | &prealloc_cf);
|
| 2589 | spin_unlock(&ci->i_ceph_lock);
|
| 2590 | if (dirty)
|
| 2591 | __mark_inode_dirty(inode, dirty);
|
| 2592 | if (ceph_quota_is_max_bytes_approaching(inode, iocb->ki_pos))
|
| 2593 | ceph_check_caps(ci, CHECK_CAPS_FLUSH);
|
| 2594 | }
|
| 2595 |
|
| 2596 | doutc(cl, "%p %llx.%llx %llu~%u dropping cap refs on %s\n",
|
| 2597 | inode, ceph_vinop(inode), pos, (unsigned)count,
|
| 2598 | ceph_cap_string(got));
|
| 2599 | ceph_put_cap_refs(ci, got);
|
| 2600 |
|
| 2601 | if (written == -EOLDSNAPC) {
|
| 2602 | doutc(cl, "%p %llx.%llx %llu~%u" "got EOLDSNAPC, retrying\n",
|
| 2603 | inode, ceph_vinop(inode), pos, (unsigned)count);
|
| 2604 | goto retry_snap;
|
| 2605 | }
|
| 2606 |
|
| 2607 | if (written >= 0) {
|
| 2608 | if (ceph_test_mount_opt(fsc, NEARFULL_SYNC) &&
|
| 2609 | ((map_flags & CEPH_OSDMAP_NEARFULL) ||
|
| 2610 | (pool_flags & CEPH_POOL_FLAG_NEARFULL)))
|
| 2611 | iocb->ki_flags |= IOCB_DSYNC;
|
| 2612 | written = generic_write_sync(iocb, written);
|
| 2613 | }
|
| 2614 |
|
| 2615 | goto out_unlocked;
|
| 2616 | out_caps:
|
| 2617 | ceph_put_cap_refs(ci, got);
|
| 2618 | out:
|
| 2619 | if (direct_lock)
|
| 2620 | ceph_end_io_direct(inode);
|
| 2621 | else
|
| 2622 | ceph_end_io_write(inode);
|
| 2623 | out_unlocked:
|
| 2624 | ceph_free_cap_flush(prealloc_cf);
|
| 2625 | return written ? written : err;
|
| 2626 | }
|
| 2627 |
|
| 2628 | |
| 2629 | * llseek. be sure to verify file size on SEEK_END.
|
| 2630 |
|
| 2631 | static loff_t ceph_llseek(struct file *file, loff_t offset, int whence)
|
| 2632 | {
|
| 2633 | if (whence == SEEK_END || whence == SEEK_DATA || whence == SEEK_HOLE) {
|
| 2634 | struct inode *inode = file_inode(file);
|
| 2635 | int ret;
|
| 2636 |
|
| 2637 | ret = ceph_do_getattr(inode, CEPH_STAT_CAP_SIZE, false);
|
| 2638 | if (ret < 0)
|
| 2639 | return ret;
|
| 2640 | }
|
| 2641 | return generic_file_llseek(file, offset, whence);
|
| 2642 | }
|
| 2643 |
|
| 2644 | static inline void ceph_zero_partial_page(struct inode *inode,
|
| 2645 | loff_t offset, size_t size)
|
| 2646 | {
|
| 2647 | struct folio *folio;
|
| 2648 |
|
| 2649 | folio = filemap_lock_folio(inode->i_mapping, offset >> PAGE_SHIFT);
|
| 2650 | if (IS_ERR(folio))
|
| 2651 | return;
|
| 2652 |
|
| 2653 | folio_wait_writeback(folio);
|
| 2654 | folio_zero_range(folio, offset_in_folio(folio, offset), size);
|
| 2655 | folio_unlock(folio);
|
| 2656 | folio_put(folio);
|
| 2657 | }
|
| 2658 |
|
| 2659 | static void ceph_zero_pagecache_range(struct inode *inode, loff_t offset,
|
| 2660 | loff_t length)
|
| 2661 | {
|
| 2662 | loff_t nearly = round_up(offset, PAGE_SIZE);
|
| 2663 | if (offset < nearly) {
|
| 2664 | loff_t size = nearly - offset;
|
| 2665 | if (length < size)
|
| 2666 | size = length;
|
| 2667 | ceph_zero_partial_page(inode, offset, size);
|
| 2668 | offset += size;
|
| 2669 | length -= size;
|
| 2670 | }
|
| 2671 | if (length >= PAGE_SIZE) {
|
| 2672 | loff_t size = round_down(length, PAGE_SIZE);
|
| 2673 | truncate_pagecache_range(inode, offset, offset + size - 1);
|
| 2674 | offset += size;
|
| 2675 | length -= size;
|
| 2676 | }
|
| 2677 | if (length)
|
| 2678 | ceph_zero_partial_page(inode, offset, length);
|
| 2679 | }
|
| 2680 |
|
| 2681 | static int ceph_zero_partial_object(struct inode *inode,
|
| 2682 | loff_t offset, loff_t *length)
|
| 2683 | {
|
| 2684 | struct ceph_inode_info *ci = ceph_inode(inode);
|
| 2685 | struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
|
| 2686 | struct ceph_osd_request *req;
|
| 2687 | struct ceph_snap_context *snapc;
|
| 2688 | int ret = 0;
|
| 2689 | loff_t zero = 0;
|
| 2690 | int op;
|
| 2691 |
|
| 2692 | if (ceph_inode_is_shutdown(inode))
|
| 2693 | return -EIO;
|
| 2694 |
|
| 2695 | if (!length) {
|
| 2696 | op = offset ? CEPH_OSD_OP_DELETE : CEPH_OSD_OP_TRUNCATE;
|
| 2697 | length = &zero;
|
| 2698 | } else {
|
| 2699 | op = CEPH_OSD_OP_ZERO;
|
| 2700 | }
|
| 2701 |
|
| 2702 | spin_lock(&ci->i_ceph_lock);
|
| 2703 | if (__ceph_have_pending_cap_snap(ci)) {
|
| 2704 | struct ceph_cap_snap *capsnap =
|
| 2705 | list_last_entry(&ci->i_cap_snaps,
|
| 2706 | struct ceph_cap_snap,
|
| 2707 | ci_item);
|
| 2708 | snapc = ceph_get_snap_context(capsnap->context);
|
| 2709 | } else {
|
| 2710 | BUG_ON(!ci->i_head_snapc);
|
| 2711 | snapc = ceph_get_snap_context(ci->i_head_snapc);
|
| 2712 | }
|
| 2713 | spin_unlock(&ci->i_ceph_lock);
|
| 2714 |
|
| 2715 | req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout,
|
| 2716 | ceph_vino(inode),
|
| 2717 | offset, length,
|
| 2718 | 0, 1, op,
|
| 2719 | CEPH_OSD_FLAG_WRITE,
|
| 2720 | snapc, 0, 0, false);
|
| 2721 | if (IS_ERR(req)) {
|
| 2722 | ret = PTR_ERR(req);
|
| 2723 | goto out;
|
| 2724 | }
|
| 2725 |
|
| 2726 | req->r_mtime = inode_get_mtime(inode);
|
| 2727 | ceph_osdc_start_request(&fsc->client->osdc, req);
|
| 2728 | ret = ceph_osdc_wait_request(&fsc->client->osdc, req);
|
| 2729 | if (ret == -ENOENT)
|
| 2730 | ret = 0;
|
| 2731 | ceph_osdc_put_request(req);
|
| 2732 |
|
| 2733 | out:
|
| 2734 | ceph_put_snap_context(snapc);
|
| 2735 | return ret;
|
| 2736 | }
|
| 2737 |
|
| 2738 | static int ceph_zero_objects(struct inode *inode, loff_t offset, loff_t length)
|
| 2739 | {
|
| 2740 | int ret = 0;
|
| 2741 | struct ceph_inode_info *ci = ceph_inode(inode);
|
| 2742 | s32 stripe_unit = ci->i_layout.stripe_unit;
|
| 2743 | s32 stripe_count = ci->i_layout.stripe_count;
|
| 2744 | s32 object_size = ci->i_layout.object_size;
|
| 2745 | u64 object_set_size = (u64) object_size * stripe_count;
|
| 2746 | u64 nearly, t;
|
| 2747 |
|
| 2748 |
|
| 2749 | nearly = offset + object_set_size - 1;
|
| 2750 | t = nearly;
|
| 2751 | nearly -= do_div(t, object_set_size);
|
| 2752 |
|
| 2753 | while (length && offset < nearly) {
|
| 2754 | loff_t size = length;
|
| 2755 | ret = ceph_zero_partial_object(inode, offset, &size);
|
| 2756 | if (ret < 0)
|
| 2757 | return ret;
|
| 2758 | offset += size;
|
| 2759 | length -= size;
|
| 2760 | }
|
| 2761 | while (length >= object_set_size) {
|
| 2762 | int i;
|
| 2763 | loff_t pos = offset;
|
| 2764 | for (i = 0; i < stripe_count; ++i) {
|
| 2765 | ret = ceph_zero_partial_object(inode, pos, NULL);
|
| 2766 | if (ret < 0)
|
| 2767 | return ret;
|
| 2768 | pos += stripe_unit;
|
| 2769 | }
|
| 2770 | offset += object_set_size;
|
| 2771 | length -= object_set_size;
|
| 2772 | }
|
| 2773 | while (length) {
|
| 2774 | loff_t size = length;
|
| 2775 | ret = ceph_zero_partial_object(inode, offset, &size);
|
| 2776 | if (ret < 0)
|
| 2777 | return ret;
|
| 2778 | offset += size;
|
| 2779 | length -= size;
|
| 2780 | }
|
| 2781 | return ret;
|
| 2782 | }
|
| 2783 |
|
| 2784 | static long ceph_fallocate(struct file *file, int mode,
|
| 2785 | loff_t offset, loff_t length)
|
| 2786 | {
|
| 2787 | struct ceph_file_info *fi = file->private_data;
|
| 2788 | struct inode *inode = file_inode(file);
|
| 2789 | struct ceph_inode_info *ci = ceph_inode(inode);
|
| 2790 | struct ceph_cap_flush *prealloc_cf;
|
| 2791 | struct ceph_client *cl = ceph_inode_to_client(inode);
|
| 2792 | int want, got = 0;
|
| 2793 | int dirty;
|
| 2794 | int ret = 0;
|
| 2795 | loff_t endoff = 0;
|
| 2796 | loff_t size;
|
| 2797 |
|
| 2798 | doutc(cl, "%p %llx.%llx mode %x, offset %llu length %llu\n",
|
| 2799 | inode, ceph_vinop(inode), mode, offset, length);
|
| 2800 |
|
| 2801 | if (mode != (FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
|
| 2802 | return -EOPNOTSUPP;
|
| 2803 |
|
| 2804 | if (!S_ISREG(inode->i_mode))
|
| 2805 | return -EOPNOTSUPP;
|
| 2806 |
|
| 2807 | if (IS_ENCRYPTED(inode))
|
| 2808 | return -EOPNOTSUPP;
|
| 2809 |
|
| 2810 | prealloc_cf = ceph_alloc_cap_flush();
|
| 2811 | if (!prealloc_cf)
|
| 2812 | return -ENOMEM;
|
| 2813 |
|
| 2814 | inode_lock(inode);
|
| 2815 |
|
| 2816 | if (ceph_snap(inode) != CEPH_NOSNAP) {
|
| 2817 | ret = -EROFS;
|
| 2818 | goto unlock;
|
| 2819 | }
|
| 2820 |
|
| 2821 | size = i_size_read(inode);
|
| 2822 |
|
| 2823 |
|
| 2824 | if (offset >= size)
|
| 2825 | goto unlock;
|
| 2826 | if ((offset + length) > size)
|
| 2827 | length = size - offset;
|
| 2828 |
|
| 2829 | if (fi->fmode & CEPH_FILE_MODE_LAZY)
|
| 2830 | want = CEPH_CAP_FILE_BUFFER | CEPH_CAP_FILE_LAZYIO;
|
| 2831 | else
|
| 2832 | want = CEPH_CAP_FILE_BUFFER;
|
| 2833 |
|
| 2834 | ret = ceph_get_caps(file, CEPH_CAP_FILE_WR, want, endoff, &got);
|
| 2835 | if (ret < 0)
|
| 2836 | goto unlock;
|
| 2837 |
|
| 2838 | ret = file_modified(file);
|
| 2839 | if (ret)
|
| 2840 | goto put_caps;
|
| 2841 |
|
| 2842 | filemap_invalidate_lock(inode->i_mapping);
|
| 2843 | ceph_fscache_invalidate(inode, false);
|
| 2844 | ceph_zero_pagecache_range(inode, offset, length);
|
| 2845 | ret = ceph_zero_objects(inode, offset, length);
|
| 2846 |
|
| 2847 | if (!ret) {
|
| 2848 | spin_lock(&ci->i_ceph_lock);
|
| 2849 | dirty = __ceph_mark_dirty_caps(ci, CEPH_CAP_FILE_WR,
|
| 2850 | &prealloc_cf);
|
| 2851 | spin_unlock(&ci->i_ceph_lock);
|
| 2852 | if (dirty)
|
| 2853 | __mark_inode_dirty(inode, dirty);
|
| 2854 | }
|
| 2855 | filemap_invalidate_unlock(inode->i_mapping);
|
| 2856 |
|
| 2857 | put_caps:
|
| 2858 | ceph_put_cap_refs(ci, got);
|
| 2859 | unlock:
|
| 2860 | inode_unlock(inode);
|
| 2861 | ceph_free_cap_flush(prealloc_cf);
|
| 2862 | return ret;
|
| 2863 | }
|
| 2864 |
|
| 2865 | |
| 2866 | * This function tries to get FILE_WR capabilities for dst_ci and FILE_RD for
|
| 2867 | * src_ci. Two attempts are made to obtain both caps, and an error is return if
|
| 2868 | * this fails; zero is returned on success.
|
| 2869 |
|
| 2870 | static int get_rd_wr_caps(struct file *src_filp, int *src_got,
|
| 2871 | struct file *dst_filp,
|
| 2872 | loff_t dst_endoff, int *dst_got)
|
| 2873 | {
|
| 2874 | int ret = 0;
|
| 2875 | bool retrying = false;
|
| 2876 |
|
| 2877 | retry_caps:
|
| 2878 | ret = ceph_get_caps(dst_filp, CEPH_CAP_FILE_WR, CEPH_CAP_FILE_BUFFER,
|
| 2879 | dst_endoff, dst_got);
|
| 2880 | if (ret < 0)
|
| 2881 | return ret;
|
| 2882 |
|
| 2883 | |
| 2884 | * Since we're already holding the FILE_WR capability for the dst file,
|
| 2885 | * we would risk a deadlock by using ceph_get_caps. Thus, we'll do some
|
| 2886 | * retry dance instead to try to get both capabilities.
|
| 2887 |
|
| 2888 | ret = ceph_try_get_caps(file_inode(src_filp),
|
| 2889 | CEPH_CAP_FILE_RD, CEPH_CAP_FILE_SHARED,
|
| 2890 | false, src_got);
|
| 2891 | if (ret <= 0) {
|
| 2892 |
|
| 2893 | ceph_put_cap_refs(ceph_inode(file_inode(dst_filp)), *dst_got);
|
| 2894 | if (retrying) {
|
| 2895 | if (!ret)
|
| 2896 |
|
| 2897 | ret = -EAGAIN;
|
| 2898 | return ret;
|
| 2899 | }
|
| 2900 | ret = ceph_get_caps(src_filp, CEPH_CAP_FILE_RD,
|
| 2901 | CEPH_CAP_FILE_SHARED, -1, src_got);
|
| 2902 | if (ret < 0)
|
| 2903 | return ret;
|
| 2904 |
|
| 2905 | ceph_put_cap_refs(ceph_inode(file_inode(src_filp)), *src_got);
|
| 2906 | retrying = true;
|
| 2907 | goto retry_caps;
|
| 2908 | }
|
| 2909 | return ret;
|
| 2910 | }
|
| 2911 |
|
| 2912 | static void put_rd_wr_caps(struct ceph_inode_info *src_ci, int src_got,
|
| 2913 | struct ceph_inode_info *dst_ci, int dst_got)
|
| 2914 | {
|
| 2915 | ceph_put_cap_refs(src_ci, src_got);
|
| 2916 | ceph_put_cap_refs(dst_ci, dst_got);
|
| 2917 | }
|
| 2918 |
|
| 2919 | |
| 2920 | * This function does several size-related checks, returning an error if:
|
| 2921 | * - source file is smaller than off+len
|
| 2922 | * - destination file size is not OK (inode_newsize_ok())
|
| 2923 | * - max bytes quotas is exceeded
|
| 2924 |
|
| 2925 | static int is_file_size_ok(struct inode *src_inode, struct inode *dst_inode,
|
| 2926 | loff_t src_off, loff_t dst_off, size_t len)
|
| 2927 | {
|
| 2928 | struct ceph_client *cl = ceph_inode_to_client(src_inode);
|
| 2929 | loff_t size, endoff;
|
| 2930 |
|
| 2931 | size = i_size_read(src_inode);
|
| 2932 | |
| 2933 | * Don't copy beyond source file EOF. Instead of simply setting length
|
| 2934 | * to (size - src_off), just drop to VFS default implementation, as the
|
| 2935 | * local i_size may be stale due to other clients writing to the source
|
| 2936 | * inode.
|
| 2937 |
|
| 2938 | if (src_off + len > size) {
|
| 2939 | doutc(cl, "Copy beyond EOF (%llu + %zu > %llu)\n", src_off,
|
| 2940 | len, size);
|
| 2941 | return -EOPNOTSUPP;
|
| 2942 | }
|
| 2943 | size = i_size_read(dst_inode);
|
| 2944 |
|
| 2945 | endoff = dst_off + len;
|
| 2946 | if (inode_newsize_ok(dst_inode, endoff))
|
| 2947 | return -EOPNOTSUPP;
|
| 2948 |
|
| 2949 | if (ceph_quota_is_max_bytes_exceeded(dst_inode, endoff))
|
| 2950 | return -EDQUOT;
|
| 2951 |
|
| 2952 | return 0;
|
| 2953 | }
|
| 2954 |
|
| 2955 | static struct ceph_osd_request *
|
| 2956 | ceph_alloc_copyfrom_request(struct ceph_osd_client *osdc,
|
| 2957 | u64 src_snapid,
|
| 2958 | struct ceph_object_id *src_oid,
|
| 2959 | struct ceph_object_locator *src_oloc,
|
| 2960 | struct ceph_object_id *dst_oid,
|
| 2961 | struct ceph_object_locator *dst_oloc,
|
| 2962 | u32 truncate_seq, u64 truncate_size)
|
| 2963 | {
|
| 2964 | struct ceph_osd_request *req;
|
| 2965 | int ret;
|
| 2966 | u32 src_fadvise_flags =
|
| 2967 | CEPH_OSD_OP_FLAG_FADVISE_SEQUENTIAL |
|
| 2968 | CEPH_OSD_OP_FLAG_FADVISE_NOCACHE;
|
| 2969 | u32 dst_fadvise_flags =
|
| 2970 | CEPH_OSD_OP_FLAG_FADVISE_SEQUENTIAL |
|
| 2971 | CEPH_OSD_OP_FLAG_FADVISE_DONTNEED;
|
| 2972 |
|
| 2973 | req = ceph_osdc_alloc_request(osdc, NULL, 1, false, GFP_KERNEL);
|
| 2974 | if (!req)
|
| 2975 | return ERR_PTR(-ENOMEM);
|
| 2976 |
|
| 2977 | req->r_flags = CEPH_OSD_FLAG_WRITE;
|
| 2978 |
|
| 2979 | ceph_oloc_copy(&req->r_t.base_oloc, dst_oloc);
|
| 2980 | ceph_oid_copy(&req->r_t.base_oid, dst_oid);
|
| 2981 |
|
| 2982 | ret = osd_req_op_copy_from_init(req, src_snapid, 0,
|
| 2983 | src_oid, src_oloc,
|
| 2984 | src_fadvise_flags,
|
| 2985 | dst_fadvise_flags,
|
| 2986 | truncate_seq,
|
| 2987 | truncate_size,
|
| 2988 | CEPH_OSD_COPY_FROM_FLAG_TRUNCATE_SEQ);
|
| 2989 | if (ret)
|
| 2990 | goto out;
|
| 2991 |
|
| 2992 | ret = ceph_osdc_alloc_messages(req, GFP_KERNEL);
|
| 2993 | if (ret)
|
| 2994 | goto out;
|
| 2995 |
|
| 2996 | return req;
|
| 2997 |
|
| 2998 | out:
|
| 2999 | ceph_osdc_put_request(req);
|
| 3000 | return ERR_PTR(ret);
|
| 3001 | }
|
| 3002 |
|
| 3003 | static ssize_t ceph_do_objects_copy(struct ceph_inode_info *src_ci, u64 *src_off,
|
| 3004 | struct ceph_inode_info *dst_ci, u64 *dst_off,
|
| 3005 | struct ceph_fs_client *fsc,
|
| 3006 | size_t len, unsigned int flags)
|
| 3007 | {
|
| 3008 | struct ceph_object_locator src_oloc, dst_oloc;
|
| 3009 | struct ceph_object_id src_oid, dst_oid;
|
| 3010 | struct ceph_osd_client *osdc;
|
| 3011 | struct ceph_osd_request *req;
|
| 3012 | ssize_t bytes = 0;
|
| 3013 | u64 src_objnum, src_objoff, dst_objnum, dst_objoff;
|
| 3014 | u32 src_objlen, dst_objlen;
|
| 3015 | u32 object_size = src_ci->i_layout.object_size;
|
| 3016 | struct ceph_client *cl = fsc->client;
|
| 3017 | int ret;
|
| 3018 |
|
| 3019 | src_oloc.pool = src_ci->i_layout.pool_id;
|
| 3020 | src_oloc.pool_ns = ceph_try_get_string(src_ci->i_layout.pool_ns);
|
| 3021 | dst_oloc.pool = dst_ci->i_layout.pool_id;
|
| 3022 | dst_oloc.pool_ns = ceph_try_get_string(dst_ci->i_layout.pool_ns);
|
| 3023 | osdc = &fsc->client->osdc;
|
| 3024 |
|
| 3025 | while (len >= object_size) {
|
| 3026 | ceph_calc_file_object_mapping(&src_ci->i_layout, *src_off,
|
| 3027 | object_size, &src_objnum,
|
| 3028 | &src_objoff, &src_objlen);
|
| 3029 | ceph_calc_file_object_mapping(&dst_ci->i_layout, *dst_off,
|
| 3030 | object_size, &dst_objnum,
|
| 3031 | &dst_objoff, &dst_objlen);
|
| 3032 | ceph_oid_init(&src_oid);
|
| 3033 | ceph_oid_printf(&src_oid, "%llx.%08llx",
|
| 3034 | src_ci->i_vino.ino, src_objnum);
|
| 3035 | ceph_oid_init(&dst_oid);
|
| 3036 | ceph_oid_printf(&dst_oid, "%llx.%08llx",
|
| 3037 | dst_ci->i_vino.ino, dst_objnum);
|
| 3038 |
|
| 3039 | req = ceph_alloc_copyfrom_request(osdc, src_ci->i_vino.snap,
|
| 3040 | &src_oid, &src_oloc,
|
| 3041 | &dst_oid, &dst_oloc,
|
| 3042 | dst_ci->i_truncate_seq,
|
| 3043 | dst_ci->i_truncate_size);
|
| 3044 | if (IS_ERR(req))
|
| 3045 | ret = PTR_ERR(req);
|
| 3046 | else {
|
| 3047 | ceph_osdc_start_request(osdc, req);
|
| 3048 | ret = ceph_osdc_wait_request(osdc, req);
|
| 3049 | ceph_update_copyfrom_metrics(&fsc->mdsc->metric,
|
| 3050 | req->r_start_latency,
|
| 3051 | req->r_end_latency,
|
| 3052 | object_size, ret);
|
| 3053 | ceph_osdc_put_request(req);
|
| 3054 | }
|
| 3055 | if (ret) {
|
| 3056 | if (ret == -EOPNOTSUPP) {
|
| 3057 | fsc->have_copy_from2 = false;
|
| 3058 | pr_notice_client(cl,
|
| 3059 | "OSDs don't support copy-from2; disabling copy offload\n");
|
| 3060 | }
|
| 3061 | doutc(cl, "returned %d\n", ret);
|
| 3062 | if (bytes <= 0)
|
| 3063 | bytes = ret;
|
| 3064 | goto out;
|
| 3065 | }
|
| 3066 | len -= object_size;
|
| 3067 | bytes += object_size;
|
| 3068 | *src_off += object_size;
|
| 3069 | *dst_off += object_size;
|
| 3070 | }
|
| 3071 |
|
| 3072 | out:
|
| 3073 | ceph_oloc_destroy(&src_oloc);
|
| 3074 | ceph_oloc_destroy(&dst_oloc);
|
| 3075 | return bytes;
|
| 3076 | }
|
| 3077 |
|
| 3078 | static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off,
|
| 3079 | struct file *dst_file, loff_t dst_off,
|
| 3080 | size_t len, unsigned int flags)
|
| 3081 | {
|
| 3082 | struct inode *src_inode = file_inode(src_file);
|
| 3083 | struct inode *dst_inode = file_inode(dst_file);
|
| 3084 | struct ceph_inode_info *src_ci = ceph_inode(src_inode);
|
| 3085 | struct ceph_inode_info *dst_ci = ceph_inode(dst_inode);
|
| 3086 | struct ceph_cap_flush *prealloc_cf;
|
| 3087 | struct ceph_fs_client *src_fsc = ceph_inode_to_fs_client(src_inode);
|
| 3088 | struct ceph_client *cl = src_fsc->client;
|
| 3089 | loff_t size;
|
| 3090 | ssize_t ret = -EIO, bytes;
|
| 3091 | u64 src_objnum, dst_objnum, src_objoff, dst_objoff;
|
| 3092 | u32 src_objlen, dst_objlen;
|
| 3093 | int src_got = 0, dst_got = 0, err, dirty;
|
| 3094 |
|
| 3095 | if (src_inode->i_sb != dst_inode->i_sb) {
|
| 3096 | struct ceph_fs_client *dst_fsc = ceph_inode_to_fs_client(dst_inode);
|
| 3097 |
|
| 3098 | if (ceph_fsid_compare(&src_fsc->client->fsid,
|
| 3099 | &dst_fsc->client->fsid)) {
|
| 3100 | dout("Copying files across clusters: src: %pU dst: %pU\n",
|
| 3101 | &src_fsc->client->fsid, &dst_fsc->client->fsid);
|
| 3102 | return -EXDEV;
|
| 3103 | }
|
| 3104 | }
|
| 3105 | if (ceph_snap(dst_inode) != CEPH_NOSNAP)
|
| 3106 | return -EROFS;
|
| 3107 |
|
| 3108 | |
| 3109 | * Some of the checks below will return -EOPNOTSUPP, which will force a
|
| 3110 | * fallback to the default VFS copy_file_range implementation. This is
|
| 3111 | * desirable in several cases (for ex, the 'len' is smaller than the
|
| 3112 | * size of the objects, or in cases where that would be more
|
| 3113 | * efficient).
|
| 3114 |
|
| 3115 |
|
| 3116 | if (ceph_test_mount_opt(src_fsc, NOCOPYFROM))
|
| 3117 | return -EOPNOTSUPP;
|
| 3118 |
|
| 3119 | if (!src_fsc->have_copy_from2)
|
| 3120 | return -EOPNOTSUPP;
|
| 3121 |
|
| 3122 | |
| 3123 | * Striped file layouts require that we copy partial objects, but the
|
| 3124 | * OSD copy-from operation only supports full-object copies. Limit
|
| 3125 | * this to non-striped file layouts for now.
|
| 3126 |
|
| 3127 | if ((src_ci->i_layout.stripe_unit != dst_ci->i_layout.stripe_unit) ||
|
| 3128 | (src_ci->i_layout.stripe_count != 1) ||
|
| 3129 | (dst_ci->i_layout.stripe_count != 1) ||
|
| 3130 | (src_ci->i_layout.object_size != dst_ci->i_layout.object_size)) {
|
| 3131 | doutc(cl, "Invalid src/dst files layout\n");
|
| 3132 | return -EOPNOTSUPP;
|
| 3133 | }
|
| 3134 |
|
| 3135 |
|
| 3136 | if (IS_ENCRYPTED(src_inode) || IS_ENCRYPTED(dst_inode))
|
| 3137 | return -EOPNOTSUPP;
|
| 3138 |
|
| 3139 | if (len < src_ci->i_layout.object_size)
|
| 3140 | return -EOPNOTSUPP;
|
| 3141 |
|
| 3142 | prealloc_cf = ceph_alloc_cap_flush();
|
| 3143 | if (!prealloc_cf)
|
| 3144 | return -ENOMEM;
|
| 3145 |
|
| 3146 |
|
| 3147 | ret = file_write_and_wait_range(src_file, src_off, (src_off + len));
|
| 3148 | if (ret < 0) {
|
| 3149 | doutc(cl, "failed to write src file (%zd)\n", ret);
|
| 3150 | goto out;
|
| 3151 | }
|
| 3152 | ret = file_write_and_wait_range(dst_file, dst_off, (dst_off + len));
|
| 3153 | if (ret < 0) {
|
| 3154 | doutc(cl, "failed to write dst file (%zd)\n", ret);
|
| 3155 | goto out;
|
| 3156 | }
|
| 3157 |
|
| 3158 | |
| 3159 | * We need FILE_WR caps for dst_ci and FILE_RD for src_ci as other
|
| 3160 | * clients may have dirty data in their caches. And OSDs know nothing
|
| 3161 | * about caps, so they can't safely do the remote object copies.
|
| 3162 |
|
| 3163 | err = get_rd_wr_caps(src_file, &src_got,
|
| 3164 | dst_file, (dst_off + len), &dst_got);
|
| 3165 | if (err < 0) {
|
| 3166 | doutc(cl, "get_rd_wr_caps returned %d\n", err);
|
| 3167 | ret = -EOPNOTSUPP;
|
| 3168 | goto out;
|
| 3169 | }
|
| 3170 |
|
| 3171 | ret = is_file_size_ok(src_inode, dst_inode, src_off, dst_off, len);
|
| 3172 | if (ret < 0)
|
| 3173 | goto out_caps;
|
| 3174 |
|
| 3175 |
|
| 3176 | ceph_fscache_invalidate(dst_inode, false);
|
| 3177 | ret = invalidate_inode_pages2_range(dst_inode->i_mapping,
|
| 3178 | dst_off >> PAGE_SHIFT,
|
| 3179 | (dst_off + len) >> PAGE_SHIFT);
|
| 3180 | if (ret < 0) {
|
| 3181 | doutc(cl, "Failed to invalidate inode pages (%zd)\n",
|
| 3182 | ret);
|
| 3183 | ret = 0;
|
| 3184 | }
|
| 3185 | ceph_calc_file_object_mapping(&src_ci->i_layout, src_off,
|
| 3186 | src_ci->i_layout.object_size,
|
| 3187 | &src_objnum, &src_objoff, &src_objlen);
|
| 3188 | ceph_calc_file_object_mapping(&dst_ci->i_layout, dst_off,
|
| 3189 | dst_ci->i_layout.object_size,
|
| 3190 | &dst_objnum, &dst_objoff, &dst_objlen);
|
| 3191 |
|
| 3192 | if (src_objoff != dst_objoff) {
|
| 3193 | ret = -EOPNOTSUPP;
|
| 3194 | goto out_caps;
|
| 3195 | }
|
| 3196 |
|
| 3197 | |
| 3198 | * Do a manual copy if the object offset isn't object aligned.
|
| 3199 | * 'src_objlen' contains the bytes left until the end of the object,
|
| 3200 | * starting at the src_off
|
| 3201 |
|
| 3202 | if (src_objoff) {
|
| 3203 | doutc(cl, "Initial partial copy of %u bytes\n", src_objlen);
|
| 3204 |
|
| 3205 | |
| 3206 | * we need to temporarily drop all caps as we'll be calling
|
| 3207 | * {read,write}_iter, which will get caps again.
|
| 3208 |
|
| 3209 | put_rd_wr_caps(src_ci, src_got, dst_ci, dst_got);
|
| 3210 | ret = splice_file_range(src_file, &src_off, dst_file, &dst_off,
|
| 3211 | src_objlen);
|
| 3212 |
|
| 3213 | if (ret < (long)src_objlen) {
|
| 3214 | doutc(cl, "Failed partial copy (%zd)\n", ret);
|
| 3215 | goto out;
|
| 3216 | }
|
| 3217 | len -= ret;
|
| 3218 | err = get_rd_wr_caps(src_file, &src_got,
|
| 3219 | dst_file, (dst_off + len), &dst_got);
|
| 3220 | if (err < 0)
|
| 3221 | goto out;
|
| 3222 | err = is_file_size_ok(src_inode, dst_inode,
|
| 3223 | src_off, dst_off, len);
|
| 3224 | if (err < 0)
|
| 3225 | goto out_caps;
|
| 3226 | }
|
| 3227 |
|
| 3228 | size = i_size_read(dst_inode);
|
| 3229 | bytes = ceph_do_objects_copy(src_ci, &src_off, dst_ci, &dst_off,
|
| 3230 | src_fsc, len, flags);
|
| 3231 | if (bytes <= 0) {
|
| 3232 | if (!ret)
|
| 3233 | ret = bytes;
|
| 3234 | goto out_caps;
|
| 3235 | }
|
| 3236 | doutc(cl, "Copied %zu bytes out of %zu\n", bytes, len);
|
| 3237 | len -= bytes;
|
| 3238 | ret += bytes;
|
| 3239 |
|
| 3240 | file_update_time(dst_file);
|
| 3241 | inode_inc_iversion_raw(dst_inode);
|
| 3242 |
|
| 3243 | if (dst_off > size) {
|
| 3244 |
|
| 3245 | if (ceph_inode_set_size(dst_inode, dst_off) ||
|
| 3246 | ceph_quota_is_max_bytes_approaching(dst_inode, dst_off))
|
| 3247 | ceph_check_caps(dst_ci, CHECK_CAPS_AUTHONLY | CHECK_CAPS_FLUSH);
|
| 3248 | }
|
| 3249 |
|
| 3250 | spin_lock(&dst_ci->i_ceph_lock);
|
| 3251 | dirty = __ceph_mark_dirty_caps(dst_ci, CEPH_CAP_FILE_WR, &prealloc_cf);
|
| 3252 | spin_unlock(&dst_ci->i_ceph_lock);
|
| 3253 | if (dirty)
|
| 3254 | __mark_inode_dirty(dst_inode, dirty);
|
| 3255 |
|
| 3256 | out_caps:
|
| 3257 | put_rd_wr_caps(src_ci, src_got, dst_ci, dst_got);
|
| 3258 |
|
| 3259 | |
| 3260 | * Do the final manual copy if we still have some bytes left, unless
|
| 3261 | * there were errors in remote object copies (len >= object_size).
|
| 3262 |
|
| 3263 | if (len && (len < src_ci->i_layout.object_size)) {
|
| 3264 | doutc(cl, "Final partial copy of %zu bytes\n", len);
|
| 3265 | bytes = splice_file_range(src_file, &src_off, dst_file,
|
| 3266 | &dst_off, len);
|
| 3267 | if (bytes > 0)
|
| 3268 | ret += bytes;
|
| 3269 | else
|
| 3270 | doutc(cl, "Failed partial copy (%zd)\n", bytes);
|
| 3271 | }
|
| 3272 |
|
| 3273 | out:
|
| 3274 | ceph_free_cap_flush(prealloc_cf);
|
| 3275 |
|
| 3276 | return ret;
|
| 3277 | }
|
| 3278 |
|
| 3279 | static ssize_t ceph_copy_file_range(struct file *src_file, loff_t src_off,
|
| 3280 | struct file *dst_file, loff_t dst_off,
|
| 3281 | size_t len, unsigned int flags)
|
| 3282 | {
|
| 3283 | ssize_t ret;
|
| 3284 |
|
| 3285 | ret = __ceph_copy_file_range(src_file, src_off, dst_file, dst_off,
|
| 3286 | len, flags);
|
| 3287 |
|
| 3288 | if (ret == -EOPNOTSUPP || ret == -EXDEV)
|
| 3289 | ret = splice_copy_file_range(src_file, src_off, dst_file,
|
| 3290 | dst_off, len);
|
| 3291 | return ret;
|
| 3292 | }
|
| 3293 |
|
| 3294 | const struct file_operations ceph_file_fops = {
|
| 3295 | .open = ceph_open,
|
| 3296 | .release = ceph_release,
|
| 3297 | .llseek = ceph_llseek,
|
| 3298 | .read_iter = ceph_read_iter,
|
| 3299 | .write_iter = ceph_write_iter,
|
| 3300 | .mmap_prepare = ceph_mmap_prepare,
|
| 3301 | .fsync = ceph_fsync,
|
| 3302 | .lock = ceph_lock,
|
| 3303 | .flock = ceph_flock,
|
| 3304 | .splice_read = ceph_splice_read,
|
| 3305 | .splice_write = iter_file_splice_write,
|
| 3306 | .unlocked_ioctl = ceph_ioctl,
|
| 3307 | .compat_ioctl = compat_ptr_ioctl,
|
| 3308 | .fallocate = ceph_fallocate,
|
| 3309 | .copy_file_range = ceph_copy_file_range,
|
| 3310 | };
|