| 1 | |
| 2 | |
| 3 | * Copyright (c) 2011-2014, Intel Corporation.
|
| 4 | * Copyright (c) 2017-2021 Christoph Hellwig.
|
| 5 |
|
| 6 | #include <linux/blk-integrity.h>
|
| 7 | #include <linux/ptrace.h>
|
| 8 | #include <linux/nvme_ioctl.h>
|
| 9 | #include <linux/io_uring/cmd.h>
|
| 10 | #include "nvme.h"
|
| 11 |
|
| 12 | enum {
|
| 13 | NVME_IOCTL_VEC = (1 << 0),
|
| 14 | NVME_IOCTL_PARTITION = (1 << 1),
|
| 15 | };
|
| 16 |
|
| 17 | static bool nvme_admin_cmd_allowed(struct nvme_ctrl *ctrl,
|
| 18 | struct nvme_command *c)
|
| 19 | {
|
| 20 | |
| 21 | * Do not allow unprivileged passthrough of admin commands except
|
| 22 | * for a subset of identify commands that contain information required
|
| 23 | * to form proper I/O commands in userspace and do not expose any
|
| 24 | * potentially sensitive information.
|
| 25 |
|
| 26 | switch (c->common.opcode) {
|
| 27 | case nvme_admin_identify:
|
| 28 | switch (c->identify.cns) {
|
| 29 | case NVME_ID_CNS_NS:
|
| 30 | case NVME_ID_CNS_CS_NS:
|
| 31 | case NVME_ID_CNS_NS_CS_INDEP:
|
| 32 | case NVME_ID_CNS_CS_CTRL:
|
| 33 | case NVME_ID_CNS_CTRL:
|
| 34 | return true;
|
| 35 | }
|
| 36 | break;
|
| 37 | case nvme_admin_set_features:
|
| 38 | |
| 39 | * Reject Set Features that change controller state the driver
|
| 40 | * manages itself; setting them behind the driver's back from
|
| 41 | * userspace leaves it unable to react correctly. Keep Alive is
|
| 42 | * only armed for fabrics - on other transports it has no
|
| 43 | * reserved tag and harms idle power states.
|
| 44 |
|
| 45 | switch (le32_to_cpu(c->features.fid) & 0xff) {
|
| 46 | case NVME_FEAT_KATO:
|
| 47 | if (ctrl->ops->flags & NVME_F_FABRICS)
|
| 48 | break;
|
| 49 | fallthrough;
|
| 50 | case NVME_FEAT_HOST_BEHAVIOR:
|
| 51 | case NVME_FEAT_HOST_MEM_BUF:
|
| 52 | case NVME_FEAT_NUM_QUEUES:
|
| 53 | case NVME_FEAT_AUTO_PST:
|
| 54 | return false;
|
| 55 | }
|
| 56 | break;
|
| 57 | }
|
| 58 | return capable(CAP_SYS_ADMIN);
|
| 59 | }
|
| 60 |
|
| 61 | static bool nvme_ns_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c,
|
| 62 | bool open_for_write)
|
| 63 | {
|
| 64 | u32 effects;
|
| 65 |
|
| 66 | |
| 67 | * Check if the controller provides a Commands Supported and Effects log
|
| 68 | * and marks this command as supported. If not reject unprivileged
|
| 69 | * passthrough.
|
| 70 |
|
| 71 | effects = nvme_command_effects(ns->ctrl, ns, c->common.opcode);
|
| 72 | if (!(effects & NVME_CMD_EFFECTS_CSUPP))
|
| 73 | return capable(CAP_SYS_ADMIN);
|
| 74 |
|
| 75 | |
| 76 | * Don't allow passthrough for command that have intrusive (or unknown)
|
| 77 | * effects.
|
| 78 |
|
| 79 | if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC |
|
| 80 | NVME_CMD_EFFECTS_UUID_SEL |
|
| 81 | NVME_CMD_EFFECTS_SCOPE_MASK))
|
| 82 | return capable(CAP_SYS_ADMIN);
|
| 83 |
|
| 84 | |
| 85 | * Only allow I/O commands that transfer data to the controller or that
|
| 86 | * change the logical block contents if the file descriptor is open for
|
| 87 | * writing.
|
| 88 |
|
| 89 | if ((nvme_is_write(c) || (effects & NVME_CMD_EFFECTS_LBCC)) &&
|
| 90 | !open_for_write)
|
| 91 | return capable(CAP_SYS_ADMIN);
|
| 92 |
|
| 93 | return true;
|
| 94 | }
|
| 95 |
|
| 96 | static bool nvme_cmd_allowed(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
|
| 97 | struct nvme_command *c, unsigned int flags,
|
| 98 | bool open_for_write)
|
| 99 | {
|
| 100 | |
| 101 | * Do not allow unprivileged passthrough on partitions, as that
|
| 102 | * allows an escape from the containment of the partition.
|
| 103 |
|
| 104 | if (flags & NVME_IOCTL_PARTITION)
|
| 105 | return capable(CAP_SYS_ADMIN);
|
| 106 |
|
| 107 | |
| 108 | * Do not allow unprivileged processes to send vendor specific or
|
| 109 | * fabrics commands as we can't be sure about their effects.
|
| 110 |
|
| 111 | if (c->common.opcode >= nvme_cmd_vendor_start ||
|
| 112 | c->common.opcode == nvme_fabrics_command)
|
| 113 | return capable(CAP_SYS_ADMIN);
|
| 114 |
|
| 115 | if (!ns)
|
| 116 | return nvme_admin_cmd_allowed(ctrl, c);
|
| 117 |
|
| 118 | return nvme_ns_cmd_allowed(ns, c, open_for_write);
|
| 119 | }
|
| 120 |
|
| 121 | |
| 122 | * Convert integer values from ioctl structures to user pointers, silently
|
| 123 | * ignoring the upper bits in the compat case to match behaviour of 32-bit
|
| 124 | * kernels.
|
| 125 |
|
| 126 | static void __user *nvme_to_user_ptr(uintptr_t ptrval)
|
| 127 | {
|
| 128 | if (in_compat_syscall())
|
| 129 | ptrval = (compat_uptr_t)ptrval;
|
| 130 | return (void __user *)ptrval;
|
| 131 | }
|
| 132 |
|
| 133 | static struct request *nvme_alloc_user_request(struct request_queue *q,
|
| 134 | struct nvme_command *cmd, blk_opf_t rq_flags,
|
| 135 | blk_mq_req_flags_t blk_flags)
|
| 136 | {
|
| 137 | struct nvme_ns *ns = q->queuedata;
|
| 138 | struct request *req;
|
| 139 |
|
| 140 | |
| 141 | * The NVME_MPATH flag is set only for IO commands sent to a namespace
|
| 142 | * with a multipath enabled head. The request is not eligible for
|
| 143 | * failover as passthrough requests also append REQ_FAILFAST_DRIVER.
|
| 144 |
|
| 145 | if (ns && nvme_ns_head_multipath(ns->head))
|
| 146 | rq_flags |= REQ_NVME_MPATH;
|
| 147 |
|
| 148 | req = blk_mq_alloc_request(q, nvme_req_op(cmd) | rq_flags, blk_flags);
|
| 149 | if (IS_ERR(req))
|
| 150 | return req;
|
| 151 | nvme_init_request(req, cmd);
|
| 152 | nvme_req(req)->flags |= NVME_REQ_USERCMD;
|
| 153 | return req;
|
| 154 | }
|
| 155 |
|
| 156 | static int nvme_map_user_request(struct request *req, u64 ubuffer,
|
| 157 | unsigned bufflen, void __user *meta_buffer, unsigned meta_len,
|
| 158 | struct iov_iter *iter, unsigned int flags)
|
| 159 | {
|
| 160 | struct request_queue *q = req->q;
|
| 161 | struct nvme_ns *ns = q->queuedata;
|
| 162 | struct block_device *bdev = ns ? ns->disk->part0 : NULL;
|
| 163 | bool supports_metadata = bdev && blk_get_integrity(bdev->bd_disk);
|
| 164 | bool has_metadata = meta_buffer && meta_len;
|
| 165 | int ret;
|
| 166 |
|
| 167 | if (has_metadata && !supports_metadata)
|
| 168 | return -EINVAL;
|
| 169 |
|
| 170 | if (iter)
|
| 171 | ret = blk_rq_map_user_iov(q, req, NULL, iter, GFP_KERNEL);
|
| 172 | else
|
| 173 | ret = blk_rq_map_user_io(req, NULL, nvme_to_user_ptr(ubuffer),
|
| 174 | bufflen, GFP_KERNEL, flags & NVME_IOCTL_VEC, 0,
|
| 175 | 0, rq_data_dir(req));
|
| 176 | if (ret)
|
| 177 | return ret;
|
| 178 |
|
| 179 | if (has_metadata) {
|
| 180 | ret = blk_rq_integrity_map_user(req, meta_buffer, meta_len);
|
| 181 | if (ret)
|
| 182 | goto out_unmap;
|
| 183 | }
|
| 184 |
|
| 185 | return ret;
|
| 186 |
|
| 187 | out_unmap:
|
| 188 | if (req->bio)
|
| 189 | blk_rq_unmap_user(req->bio);
|
| 190 | return ret;
|
| 191 | }
|
| 192 |
|
| 193 | static int nvme_submit_user_cmd(struct request_queue *q,
|
| 194 | struct nvme_command *cmd, u64 ubuffer, unsigned bufflen,
|
| 195 | void __user *meta_buffer, unsigned meta_len,
|
| 196 | u64 *result, unsigned timeout, unsigned int flags)
|
| 197 | {
|
| 198 | struct nvme_ns *ns = q->queuedata;
|
| 199 | struct nvme_ctrl *ctrl;
|
| 200 | struct request *req;
|
| 201 | struct bio *bio;
|
| 202 | u32 effects;
|
| 203 | int ret;
|
| 204 |
|
| 205 | req = nvme_alloc_user_request(q, cmd, 0, 0);
|
| 206 | if (IS_ERR(req))
|
| 207 | return PTR_ERR(req);
|
| 208 |
|
| 209 | req->timeout = timeout;
|
| 210 | if (ubuffer && bufflen) {
|
| 211 | ret = nvme_map_user_request(req, ubuffer, bufflen, meta_buffer,
|
| 212 | meta_len, NULL, flags);
|
| 213 | if (ret)
|
| 214 | goto out_free_req;
|
| 215 | }
|
| 216 |
|
| 217 | bio = req->bio;
|
| 218 | ctrl = nvme_req(req)->ctrl;
|
| 219 |
|
| 220 | effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode);
|
| 221 | ret = nvme_execute_rq(req, false);
|
| 222 | if (result)
|
| 223 | *result = le64_to_cpu(nvme_req(req)->result.u64);
|
| 224 | if (bio)
|
| 225 | blk_rq_unmap_user(bio);
|
| 226 | blk_mq_free_request(req);
|
| 227 |
|
| 228 | if (effects)
|
| 229 | nvme_passthru_end(ctrl, ns, effects, cmd, ret);
|
| 230 | return ret;
|
| 231 |
|
| 232 | out_free_req:
|
| 233 | blk_mq_free_request(req);
|
| 234 | return ret;
|
| 235 | }
|
| 236 |
|
| 237 | static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio,
|
| 238 | unsigned int flags, bool open_for_write)
|
| 239 | {
|
| 240 | struct nvme_user_io io;
|
| 241 | struct nvme_command c;
|
| 242 | unsigned length, meta_len;
|
| 243 | void __user *metadata;
|
| 244 |
|
| 245 | if (copy_from_user(&io, uio, sizeof(io)))
|
| 246 | return -EFAULT;
|
| 247 | if (io.flags)
|
| 248 | return -EINVAL;
|
| 249 |
|
| 250 | switch (io.opcode) {
|
| 251 | case nvme_cmd_write:
|
| 252 | case nvme_cmd_read:
|
| 253 | case nvme_cmd_compare:
|
| 254 | break;
|
| 255 | default:
|
| 256 | return -EINVAL;
|
| 257 | }
|
| 258 |
|
| 259 | length = (io.nblocks + 1) << ns->head->lba_shift;
|
| 260 |
|
| 261 | if ((io.control & NVME_RW_PRINFO_PRACT) &&
|
| 262 | (ns->head->ms == ns->head->pi_size)) {
|
| 263 | |
| 264 | * Protection information is stripped/inserted by the
|
| 265 | * controller.
|
| 266 |
|
| 267 | if (nvme_to_user_ptr(io.metadata))
|
| 268 | return -EINVAL;
|
| 269 | meta_len = 0;
|
| 270 | metadata = NULL;
|
| 271 | } else {
|
| 272 | meta_len = (io.nblocks + 1) * ns->head->ms;
|
| 273 | metadata = nvme_to_user_ptr(io.metadata);
|
| 274 | }
|
| 275 |
|
| 276 | if (ns->head->features & NVME_NS_EXT_LBAS) {
|
| 277 | length += meta_len;
|
| 278 | meta_len = 0;
|
| 279 | } else if (meta_len) {
|
| 280 | if ((io.metadata & 3) || !io.metadata)
|
| 281 | return -EINVAL;
|
| 282 | }
|
| 283 |
|
| 284 | memset(&c, 0, sizeof(c));
|
| 285 | c.rw.opcode = io.opcode;
|
| 286 | c.rw.flags = io.flags;
|
| 287 | c.rw.nsid = cpu_to_le32(ns->head->ns_id);
|
| 288 | c.rw.slba = cpu_to_le64(io.slba);
|
| 289 | c.rw.length = cpu_to_le16(io.nblocks);
|
| 290 | c.rw.control = cpu_to_le16(io.control);
|
| 291 | c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
|
| 292 | c.rw.reftag = cpu_to_le32(io.reftag);
|
| 293 | c.rw.lbat = cpu_to_le16(io.apptag);
|
| 294 | c.rw.lbatm = cpu_to_le16(io.appmask);
|
| 295 |
|
| 296 | if (!nvme_cmd_allowed(ns->ctrl, ns, &c, flags, open_for_write))
|
| 297 | return -EACCES;
|
| 298 |
|
| 299 | return nvme_submit_user_cmd(ns->queue, &c, io.addr, length, metadata,
|
| 300 | meta_len, NULL, 0, 0);
|
| 301 | }
|
| 302 |
|
| 303 | static bool nvme_validate_passthru_nsid(struct nvme_ctrl *ctrl,
|
| 304 | struct nvme_ns *ns, __u32 nsid)
|
| 305 | {
|
| 306 | if (ns && nsid != ns->head->ns_id) {
|
| 307 | dev_err(ctrl->device,
|
| 308 | "%s: nsid (%u) in cmd does not match nsid (%u) of namespace\n",
|
| 309 | current->comm, nsid, ns->head->ns_id);
|
| 310 | return false;
|
| 311 | }
|
| 312 |
|
| 313 | return true;
|
| 314 | }
|
| 315 |
|
| 316 | static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
|
| 317 | struct nvme_passthru_cmd __user *ucmd, unsigned int flags,
|
| 318 | bool open_for_write)
|
| 319 | {
|
| 320 | struct nvme_passthru_cmd cmd;
|
| 321 | struct nvme_command c;
|
| 322 | unsigned timeout = 0;
|
| 323 | u64 result;
|
| 324 | int status;
|
| 325 |
|
| 326 | if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
|
| 327 | return -EFAULT;
|
| 328 | if (cmd.flags)
|
| 329 | return -EINVAL;
|
| 330 | if (!nvme_validate_passthru_nsid(ctrl, ns, cmd.nsid))
|
| 331 | return -EINVAL;
|
| 332 |
|
| 333 | memset(&c, 0, sizeof(c));
|
| 334 | c.common.opcode = cmd.opcode;
|
| 335 | c.common.flags = cmd.flags;
|
| 336 | c.common.nsid = cpu_to_le32(cmd.nsid);
|
| 337 | c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
|
| 338 | c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
|
| 339 | c.common.cdw10 = cpu_to_le32(cmd.cdw10);
|
| 340 | c.common.cdw11 = cpu_to_le32(cmd.cdw11);
|
| 341 | c.common.cdw12 = cpu_to_le32(cmd.cdw12);
|
| 342 | c.common.cdw13 = cpu_to_le32(cmd.cdw13);
|
| 343 | c.common.cdw14 = cpu_to_le32(cmd.cdw14);
|
| 344 | c.common.cdw15 = cpu_to_le32(cmd.cdw15);
|
| 345 |
|
| 346 | if (!nvme_cmd_allowed(ctrl, ns, &c, 0, open_for_write))
|
| 347 | return -EACCES;
|
| 348 |
|
| 349 | if (cmd.timeout_ms)
|
| 350 | timeout = msecs_to_jiffies(cmd.timeout_ms);
|
| 351 |
|
| 352 | status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
|
| 353 | cmd.addr, cmd.data_len, nvme_to_user_ptr(cmd.metadata),
|
| 354 | cmd.metadata_len, &result, timeout, 0);
|
| 355 |
|
| 356 | if (status >= 0) {
|
| 357 | if (put_user(result, &ucmd->result))
|
| 358 | return -EFAULT;
|
| 359 | }
|
| 360 |
|
| 361 | return status;
|
| 362 | }
|
| 363 |
|
| 364 | static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
|
| 365 | struct nvme_passthru_cmd64 __user *ucmd, unsigned int flags,
|
| 366 | bool open_for_write)
|
| 367 | {
|
| 368 | struct nvme_passthru_cmd64 cmd;
|
| 369 | struct nvme_command c;
|
| 370 | unsigned timeout = 0;
|
| 371 | int status;
|
| 372 |
|
| 373 | if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
|
| 374 | return -EFAULT;
|
| 375 | if (cmd.flags)
|
| 376 | return -EINVAL;
|
| 377 | if (!nvme_validate_passthru_nsid(ctrl, ns, cmd.nsid))
|
| 378 | return -EINVAL;
|
| 379 |
|
| 380 | memset(&c, 0, sizeof(c));
|
| 381 | c.common.opcode = cmd.opcode;
|
| 382 | c.common.flags = cmd.flags;
|
| 383 | c.common.nsid = cpu_to_le32(cmd.nsid);
|
| 384 | c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
|
| 385 | c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
|
| 386 | c.common.cdw10 = cpu_to_le32(cmd.cdw10);
|
| 387 | c.common.cdw11 = cpu_to_le32(cmd.cdw11);
|
| 388 | c.common.cdw12 = cpu_to_le32(cmd.cdw12);
|
| 389 | c.common.cdw13 = cpu_to_le32(cmd.cdw13);
|
| 390 | c.common.cdw14 = cpu_to_le32(cmd.cdw14);
|
| 391 | c.common.cdw15 = cpu_to_le32(cmd.cdw15);
|
| 392 |
|
| 393 | if (!nvme_cmd_allowed(ctrl, ns, &c, flags, open_for_write))
|
| 394 | return -EACCES;
|
| 395 |
|
| 396 | if (cmd.timeout_ms)
|
| 397 | timeout = msecs_to_jiffies(cmd.timeout_ms);
|
| 398 |
|
| 399 | status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
|
| 400 | cmd.addr, cmd.data_len, nvme_to_user_ptr(cmd.metadata),
|
| 401 | cmd.metadata_len, &cmd.result, timeout, flags);
|
| 402 |
|
| 403 | if (status >= 0) {
|
| 404 | if (put_user(cmd.result, &ucmd->result))
|
| 405 | return -EFAULT;
|
| 406 | }
|
| 407 |
|
| 408 | return status;
|
| 409 | }
|
| 410 |
|
| 411 | struct nvme_uring_data {
|
| 412 | __u64 metadata;
|
| 413 | __u64 addr;
|
| 414 | __u32 data_len;
|
| 415 | __u32 metadata_len;
|
| 416 | __u32 timeout_ms;
|
| 417 | };
|
| 418 |
|
| 419 | |
| 420 | * This overlays struct io_uring_cmd pdu.
|
| 421 | * Expect build errors if this grows larger than that.
|
| 422 |
|
| 423 | struct nvme_uring_cmd_pdu {
|
| 424 | struct request *req;
|
| 425 | struct bio *bio;
|
| 426 | u64 result;
|
| 427 | int status;
|
| 428 | };
|
| 429 |
|
| 430 | static inline struct nvme_uring_cmd_pdu *nvme_uring_cmd_pdu(
|
| 431 | struct io_uring_cmd *ioucmd)
|
| 432 | {
|
| 433 | return io_uring_cmd_to_pdu(ioucmd, struct nvme_uring_cmd_pdu);
|
| 434 | }
|
| 435 |
|
| 436 | static void nvme_uring_task_cb(struct io_tw_req tw_req, io_tw_token_t tw)
|
| 437 | {
|
| 438 | struct io_uring_cmd *ioucmd = io_uring_cmd_from_tw(tw_req);
|
| 439 | struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
|
| 440 |
|
| 441 | if (pdu->bio)
|
| 442 | blk_rq_unmap_user(pdu->bio);
|
| 443 | io_uring_cmd_done32(ioucmd, pdu->status, pdu->result,
|
| 444 | IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
|
| 445 | }
|
| 446 |
|
| 447 | static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,
|
| 448 | blk_status_t err,
|
| 449 | const struct io_comp_batch *iob)
|
| 450 | {
|
| 451 | struct io_uring_cmd *ioucmd = req->end_io_data;
|
| 452 | struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
|
| 453 |
|
| 454 | if (nvme_req(req)->flags & NVME_REQ_CANCELLED) {
|
| 455 | pdu->status = -EINTR;
|
| 456 | } else {
|
| 457 | pdu->status = nvme_req(req)->status;
|
| 458 | if (!pdu->status)
|
| 459 | pdu->status = blk_status_to_errno(err);
|
| 460 | }
|
| 461 | pdu->result = le64_to_cpu(nvme_req(req)->result.u64);
|
| 462 |
|
| 463 | |
| 464 | * For IOPOLL, check if this completion is happening in the context
|
| 465 | * of the same io_ring that owns the request (local context). If so,
|
| 466 | * we can complete inline without task_work overhead. Otherwise, we
|
| 467 | * must punt to task_work to ensure completion happens in the correct
|
| 468 | * ring's context.
|
| 469 |
|
| 470 | if (blk_rq_is_poll(req) && iob &&
|
| 471 | iob->poll_ctx == io_uring_cmd_ctx_handle(ioucmd)) {
|
| 472 | if (pdu->bio)
|
| 473 | blk_rq_unmap_user(pdu->bio);
|
| 474 | io_uring_cmd_done32(ioucmd, pdu->status, pdu->result, 0);
|
| 475 | } else {
|
| 476 | io_uring_cmd_do_in_task_lazy(ioucmd, nvme_uring_task_cb);
|
| 477 | }
|
| 478 | return RQ_END_IO_FREE;
|
| 479 | }
|
| 480 |
|
| 481 | static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
|
| 482 | struct io_uring_cmd *ioucmd, unsigned int issue_flags, bool vec)
|
| 483 | {
|
| 484 | struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
|
| 485 | const struct nvme_uring_cmd *cmd = io_uring_sqe128_cmd(ioucmd->sqe,
|
| 486 | struct nvme_uring_cmd);
|
| 487 | struct request_queue *q = ns ? ns->queue : ctrl->admin_q;
|
| 488 | bool open_for_write = ioucmd->file->f_mode & FMODE_WRITE;
|
| 489 | struct nvme_uring_data d;
|
| 490 | struct nvme_command c;
|
| 491 | struct iov_iter iter;
|
| 492 | struct iov_iter *map_iter = NULL;
|
| 493 | struct request *req;
|
| 494 | blk_opf_t rq_flags = 0;
|
| 495 | blk_mq_req_flags_t blk_flags = 0;
|
| 496 | int ret;
|
| 497 |
|
| 498 | c.common.opcode = READ_ONCE(cmd->opcode);
|
| 499 | c.common.flags = READ_ONCE(cmd->flags);
|
| 500 | if (c.common.flags)
|
| 501 | return -EINVAL;
|
| 502 |
|
| 503 | c.common.command_id = 0;
|
| 504 | c.common.nsid = cpu_to_le32(cmd->nsid);
|
| 505 | if (!nvme_validate_passthru_nsid(ctrl, ns, le32_to_cpu(c.common.nsid)))
|
| 506 | return -EINVAL;
|
| 507 |
|
| 508 | c.common.cdw2[0] = cpu_to_le32(READ_ONCE(cmd->cdw2));
|
| 509 | c.common.cdw2[1] = cpu_to_le32(READ_ONCE(cmd->cdw3));
|
| 510 | c.common.metadata = 0;
|
| 511 | c.common.dptr.prp1 = c.common.dptr.prp2 = 0;
|
| 512 | c.common.cdw10 = cpu_to_le32(READ_ONCE(cmd->cdw10));
|
| 513 | c.common.cdw11 = cpu_to_le32(READ_ONCE(cmd->cdw11));
|
| 514 | c.common.cdw12 = cpu_to_le32(READ_ONCE(cmd->cdw12));
|
| 515 | c.common.cdw13 = cpu_to_le32(READ_ONCE(cmd->cdw13));
|
| 516 | c.common.cdw14 = cpu_to_le32(READ_ONCE(cmd->cdw14));
|
| 517 | c.common.cdw15 = cpu_to_le32(READ_ONCE(cmd->cdw15));
|
| 518 |
|
| 519 | if (!nvme_cmd_allowed(ctrl, ns, &c, 0, open_for_write))
|
| 520 | return -EACCES;
|
| 521 |
|
| 522 | d.metadata = READ_ONCE(cmd->metadata);
|
| 523 | d.addr = READ_ONCE(cmd->addr);
|
| 524 | d.data_len = READ_ONCE(cmd->data_len);
|
| 525 | d.metadata_len = READ_ONCE(cmd->metadata_len);
|
| 526 | d.timeout_ms = READ_ONCE(cmd->timeout_ms);
|
| 527 |
|
| 528 | if (d.data_len && (ioucmd->flags & IORING_URING_CMD_FIXED)) {
|
| 529 | int ddir = nvme_is_write(&c) ? WRITE : READ;
|
| 530 |
|
| 531 | if (vec)
|
| 532 | ret = io_uring_cmd_import_fixed_vec(ioucmd,
|
| 533 | u64_to_user_ptr(d.addr), d.data_len,
|
| 534 | ddir, &iter, issue_flags);
|
| 535 | else
|
| 536 | ret = io_uring_cmd_import_fixed(d.addr, d.data_len,
|
| 537 | ddir, &iter, ioucmd, issue_flags);
|
| 538 | if (ret < 0)
|
| 539 | return ret;
|
| 540 |
|
| 541 | map_iter = &iter;
|
| 542 | }
|
| 543 |
|
| 544 | if (issue_flags & IO_URING_F_NONBLOCK) {
|
| 545 | rq_flags |= REQ_NOWAIT;
|
| 546 | blk_flags = BLK_MQ_REQ_NOWAIT;
|
| 547 | }
|
| 548 | if (issue_flags & IO_URING_F_IOPOLL)
|
| 549 | rq_flags |= REQ_POLLED;
|
| 550 |
|
| 551 | req = nvme_alloc_user_request(q, &c, rq_flags, blk_flags);
|
| 552 | if (IS_ERR(req))
|
| 553 | return PTR_ERR(req);
|
| 554 | req->timeout = d.timeout_ms ? msecs_to_jiffies(d.timeout_ms) : 0;
|
| 555 |
|
| 556 | if (d.data_len) {
|
| 557 | ret = nvme_map_user_request(req, d.addr, d.data_len,
|
| 558 | nvme_to_user_ptr(d.metadata), d.metadata_len,
|
| 559 | map_iter, vec ? NVME_IOCTL_VEC : 0);
|
| 560 | if (ret)
|
| 561 | goto out_free_req;
|
| 562 | }
|
| 563 |
|
| 564 |
|
| 565 | pdu->bio = req->bio;
|
| 566 | pdu->req = req;
|
| 567 | req->end_io_data = ioucmd;
|
| 568 | req->end_io = nvme_uring_cmd_end_io;
|
| 569 | blk_execute_rq_nowait(req, false);
|
| 570 | return -EIOCBQUEUED;
|
| 571 |
|
| 572 | out_free_req:
|
| 573 | blk_mq_free_request(req);
|
| 574 | return ret;
|
| 575 | }
|
| 576 |
|
| 577 | static bool is_ctrl_ioctl(unsigned int cmd)
|
| 578 | {
|
| 579 | if (cmd == NVME_IOCTL_ADMIN_CMD || cmd == NVME_IOCTL_ADMIN64_CMD)
|
| 580 | return true;
|
| 581 | if (is_sed_ioctl(cmd))
|
| 582 | return true;
|
| 583 | return false;
|
| 584 | }
|
| 585 |
|
| 586 | static int nvme_ctrl_ioctl(struct nvme_ctrl *ctrl, unsigned int cmd,
|
| 587 | void __user *argp, bool open_for_write)
|
| 588 | {
|
| 589 | switch (cmd) {
|
| 590 | case NVME_IOCTL_ADMIN_CMD:
|
| 591 | return nvme_user_cmd(ctrl, NULL, argp, 0, open_for_write);
|
| 592 | case NVME_IOCTL_ADMIN64_CMD:
|
| 593 | return nvme_user_cmd64(ctrl, NULL, argp, 0, open_for_write);
|
| 594 | default:
|
| 595 | return sed_ioctl(ctrl->opal_dev, cmd, argp);
|
| 596 | }
|
| 597 | }
|
| 598 |
|
| 599 | #ifdef COMPAT_FOR_U64_ALIGNMENT
|
| 600 | struct nvme_user_io32 {
|
| 601 | __u8 opcode;
|
| 602 | __u8 flags;
|
| 603 | __u16 control;
|
| 604 | __u16 nblocks;
|
| 605 | __u16 rsvd;
|
| 606 | __u64 metadata;
|
| 607 | __u64 addr;
|
| 608 | __u64 slba;
|
| 609 | __u32 dsmgmt;
|
| 610 | __u32 reftag;
|
| 611 | __u16 apptag;
|
| 612 | __u16 appmask;
|
| 613 | } __attribute__((__packed__));
|
| 614 | #define NVME_IOCTL_SUBMIT_IO32 _IOW('N', 0x42, struct nvme_user_io32)
|
| 615 | #endif
|
| 616 |
|
| 617 | static int nvme_ns_ioctl(struct nvme_ns *ns, unsigned int cmd,
|
| 618 | void __user *argp, unsigned int flags, bool open_for_write)
|
| 619 | {
|
| 620 | switch (cmd) {
|
| 621 | case NVME_IOCTL_ID:
|
| 622 | force_successful_syscall_return();
|
| 623 | return ns->head->ns_id;
|
| 624 | case NVME_IOCTL_IO_CMD:
|
| 625 | return nvme_user_cmd(ns->ctrl, ns, argp, flags, open_for_write);
|
| 626 | |
| 627 | * struct nvme_user_io can have different padding on some 32-bit ABIs.
|
| 628 | * Just accept the compat version as all fields that are used are the
|
| 629 | * same size and at the same offset.
|
| 630 |
|
| 631 | #ifdef COMPAT_FOR_U64_ALIGNMENT
|
| 632 | case NVME_IOCTL_SUBMIT_IO32:
|
| 633 | #endif
|
| 634 | case NVME_IOCTL_SUBMIT_IO:
|
| 635 | return nvme_submit_io(ns, argp, flags, open_for_write);
|
| 636 | case NVME_IOCTL_IO64_CMD_VEC:
|
| 637 | flags |= NVME_IOCTL_VEC;
|
| 638 | fallthrough;
|
| 639 | case NVME_IOCTL_IO64_CMD:
|
| 640 | return nvme_user_cmd64(ns->ctrl, ns, argp, flags,
|
| 641 | open_for_write);
|
| 642 | default:
|
| 643 | return -ENOTTY;
|
| 644 | }
|
| 645 | }
|
| 646 |
|
| 647 | int nvme_ioctl(struct block_device *bdev, blk_mode_t mode,
|
| 648 | unsigned int cmd, unsigned long arg)
|
| 649 | {
|
| 650 | struct nvme_ns *ns = bdev->bd_disk->private_data;
|
| 651 | bool open_for_write = mode & BLK_OPEN_WRITE;
|
| 652 | void __user *argp = (void __user *)arg;
|
| 653 | unsigned int flags = 0;
|
| 654 |
|
| 655 | if (bdev_is_partition(bdev))
|
| 656 | flags |= NVME_IOCTL_PARTITION;
|
| 657 |
|
| 658 | if (is_ctrl_ioctl(cmd))
|
| 659 | return nvme_ctrl_ioctl(ns->ctrl, cmd, argp, open_for_write);
|
| 660 | return nvme_ns_ioctl(ns, cmd, argp, flags, open_for_write);
|
| 661 | }
|
| 662 |
|
| 663 | long nvme_ns_chr_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
|
| 664 | {
|
| 665 | struct nvme_ns *ns =
|
| 666 | container_of(file_inode(file)->i_cdev, struct nvme_ns, cdev);
|
| 667 | bool open_for_write = file->f_mode & FMODE_WRITE;
|
| 668 | void __user *argp = (void __user *)arg;
|
| 669 |
|
| 670 | if (is_ctrl_ioctl(cmd))
|
| 671 | return nvme_ctrl_ioctl(ns->ctrl, cmd, argp, open_for_write);
|
| 672 | return nvme_ns_ioctl(ns, cmd, argp, 0, open_for_write);
|
| 673 | }
|
| 674 |
|
| 675 | static int nvme_uring_cmd_checks(unsigned int issue_flags)
|
| 676 | {
|
| 677 |
|
| 678 |
|
| 679 | if ((issue_flags & (IO_URING_F_SQE128|IO_URING_F_CQE32)) !=
|
| 680 | (IO_URING_F_SQE128|IO_URING_F_CQE32))
|
| 681 | return -EOPNOTSUPP;
|
| 682 | return 0;
|
| 683 | }
|
| 684 |
|
| 685 | static int nvme_ns_uring_cmd(struct nvme_ns *ns, struct io_uring_cmd *ioucmd,
|
| 686 | unsigned int issue_flags)
|
| 687 | {
|
| 688 | struct nvme_ctrl *ctrl = ns->ctrl;
|
| 689 | int ret;
|
| 690 |
|
| 691 | ret = nvme_uring_cmd_checks(issue_flags);
|
| 692 | if (ret)
|
| 693 | return ret;
|
| 694 |
|
| 695 | switch (ioucmd->cmd_op) {
|
| 696 | case NVME_URING_CMD_IO:
|
| 697 | ret = nvme_uring_cmd_io(ctrl, ns, ioucmd, issue_flags, false);
|
| 698 | break;
|
| 699 | case NVME_URING_CMD_IO_VEC:
|
| 700 | ret = nvme_uring_cmd_io(ctrl, ns, ioucmd, issue_flags, true);
|
| 701 | break;
|
| 702 | default:
|
| 703 | ret = -ENOTTY;
|
| 704 | }
|
| 705 |
|
| 706 | return ret;
|
| 707 | }
|
| 708 |
|
| 709 | int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags)
|
| 710 | {
|
| 711 | struct nvme_ns *ns = container_of(file_inode(ioucmd->file)->i_cdev,
|
| 712 | struct nvme_ns, cdev);
|
| 713 |
|
| 714 | return nvme_ns_uring_cmd(ns, ioucmd, issue_flags);
|
| 715 | }
|
| 716 |
|
| 717 | int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,
|
| 718 | struct io_comp_batch *iob,
|
| 719 | unsigned int poll_flags)
|
| 720 | {
|
| 721 | struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
|
| 722 | struct request *req = pdu->req;
|
| 723 |
|
| 724 | if (req && blk_rq_is_poll(req))
|
| 725 | return blk_rq_poll(req, iob, poll_flags);
|
| 726 | return 0;
|
| 727 | }
|
| 728 | #ifdef CONFIG_NVME_MULTIPATH
|
| 729 | static int nvme_ns_head_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,
|
| 730 | void __user *argp, struct nvme_ns_head *head, int srcu_idx,
|
| 731 | bool open_for_write)
|
| 732 | __releases_shared(&head->srcu)
|
| 733 | {
|
| 734 | struct nvme_ctrl *ctrl = ns->ctrl;
|
| 735 | int ret;
|
| 736 |
|
| 737 | nvme_get_ctrl(ns->ctrl);
|
| 738 | srcu_read_unlock(&head->srcu, srcu_idx);
|
| 739 | ret = nvme_ctrl_ioctl(ctrl, cmd, argp, open_for_write);
|
| 740 |
|
| 741 | nvme_put_ctrl(ctrl);
|
| 742 | return ret;
|
| 743 | }
|
| 744 |
|
| 745 | int nvme_ns_head_ioctl(struct block_device *bdev, blk_mode_t mode,
|
| 746 | unsigned int cmd, unsigned long arg)
|
| 747 | {
|
| 748 | struct nvme_ns_head *head = bdev->bd_disk->private_data;
|
| 749 | bool open_for_write = mode & BLK_OPEN_WRITE;
|
| 750 | void __user *argp = (void __user *)arg;
|
| 751 | struct nvme_ns *ns;
|
| 752 | int srcu_idx, ret = -EWOULDBLOCK;
|
| 753 | unsigned int flags = 0;
|
| 754 |
|
| 755 | if (bdev_is_partition(bdev))
|
| 756 | flags |= NVME_IOCTL_PARTITION;
|
| 757 |
|
| 758 | srcu_idx = srcu_read_lock(&head->srcu);
|
| 759 | ns = nvme_find_path(head);
|
| 760 | if (!ns)
|
| 761 | goto out_unlock;
|
| 762 |
|
| 763 | |
| 764 | * Handle ioctls that apply to the controller instead of the namespace
|
| 765 | * separately and drop the ns SRCU reference early. This avoids a
|
| 766 | * deadlock when deleting namespaces using the passthrough interface.
|
| 767 |
|
| 768 | if (is_ctrl_ioctl(cmd))
|
| 769 | return nvme_ns_head_ctrl_ioctl(ns, cmd, argp, head, srcu_idx,
|
| 770 | open_for_write);
|
| 771 |
|
| 772 | ret = nvme_ns_ioctl(ns, cmd, argp, flags, open_for_write);
|
| 773 | out_unlock:
|
| 774 | srcu_read_unlock(&head->srcu, srcu_idx);
|
| 775 | return ret;
|
| 776 | }
|
| 777 |
|
| 778 | long nvme_ns_head_chr_ioctl(struct file *file, unsigned int cmd,
|
| 779 | unsigned long arg)
|
| 780 | {
|
| 781 | bool open_for_write = file->f_mode & FMODE_WRITE;
|
| 782 | struct cdev *cdev = file_inode(file)->i_cdev;
|
| 783 | struct nvme_ns_head *head =
|
| 784 | container_of(cdev, struct nvme_ns_head, cdev);
|
| 785 | void __user *argp = (void __user *)arg;
|
| 786 | struct nvme_ns *ns;
|
| 787 | int srcu_idx, ret = -EWOULDBLOCK;
|
| 788 |
|
| 789 | srcu_idx = srcu_read_lock(&head->srcu);
|
| 790 | ns = nvme_find_path(head);
|
| 791 | if (!ns)
|
| 792 | goto out_unlock;
|
| 793 |
|
| 794 | if (is_ctrl_ioctl(cmd))
|
| 795 | return nvme_ns_head_ctrl_ioctl(ns, cmd, argp, head, srcu_idx,
|
| 796 | open_for_write);
|
| 797 |
|
| 798 | ret = nvme_ns_ioctl(ns, cmd, argp, 0, open_for_write);
|
| 799 | out_unlock:
|
| 800 | srcu_read_unlock(&head->srcu, srcu_idx);
|
| 801 | return ret;
|
| 802 | }
|
| 803 |
|
| 804 | int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd,
|
| 805 | unsigned int issue_flags)
|
| 806 | {
|
| 807 | struct cdev *cdev = file_inode(ioucmd->file)->i_cdev;
|
| 808 | struct nvme_ns_head *head = container_of(cdev, struct nvme_ns_head, cdev);
|
| 809 | int srcu_idx = srcu_read_lock(&head->srcu);
|
| 810 | struct nvme_ns *ns = nvme_find_path(head);
|
| 811 | int ret = -EINVAL;
|
| 812 |
|
| 813 | if (ns)
|
| 814 | ret = nvme_ns_uring_cmd(ns, ioucmd, issue_flags);
|
| 815 | srcu_read_unlock(&head->srcu, srcu_idx);
|
| 816 | return ret;
|
| 817 | }
|
| 818 | #endif
|
| 819 |
|
| 820 | int nvme_dev_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags)
|
| 821 | {
|
| 822 | struct nvme_ctrl *ctrl = ioucmd->file->private_data;
|
| 823 | int ret;
|
| 824 |
|
| 825 | ret = nvme_uring_cmd_checks(issue_flags);
|
| 826 | if (ret)
|
| 827 | return ret;
|
| 828 |
|
| 829 | switch (ioucmd->cmd_op) {
|
| 830 | case NVME_URING_CMD_ADMIN:
|
| 831 | ret = nvme_uring_cmd_io(ctrl, NULL, ioucmd, issue_flags, false);
|
| 832 | break;
|
| 833 | case NVME_URING_CMD_ADMIN_VEC:
|
| 834 | ret = nvme_uring_cmd_io(ctrl, NULL, ioucmd, issue_flags, true);
|
| 835 | break;
|
| 836 | default:
|
| 837 | ret = -ENOTTY;
|
| 838 | }
|
| 839 |
|
| 840 | return ret;
|
| 841 | }
|
| 842 |
|
| 843 | static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp,
|
| 844 | bool open_for_write)
|
| 845 | {
|
| 846 | struct nvme_ns *ns;
|
| 847 | int ret, srcu_idx;
|
| 848 |
|
| 849 | srcu_idx = srcu_read_lock(&ctrl->srcu);
|
| 850 | if (list_empty(&ctrl->namespaces)) {
|
| 851 | ret = -ENOTTY;
|
| 852 | goto out_unlock;
|
| 853 | }
|
| 854 |
|
| 855 | ns = list_first_or_null_rcu(&ctrl->namespaces, struct nvme_ns, list);
|
| 856 | if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) {
|
| 857 | dev_warn(ctrl->device,
|
| 858 | "NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n");
|
| 859 | ret = -EINVAL;
|
| 860 | goto out_unlock;
|
| 861 | }
|
| 862 |
|
| 863 | dev_warn(ctrl->device,
|
| 864 | "using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n");
|
| 865 | if (!nvme_get_ns(ns)) {
|
| 866 | ret = -ENXIO;
|
| 867 | goto out_unlock;
|
| 868 | }
|
| 869 | srcu_read_unlock(&ctrl->srcu, srcu_idx);
|
| 870 |
|
| 871 | ret = nvme_user_cmd(ctrl, ns, argp, 0, open_for_write);
|
| 872 | nvme_put_ns(ns);
|
| 873 | return ret;
|
| 874 |
|
| 875 | out_unlock:
|
| 876 | srcu_read_unlock(&ctrl->srcu, srcu_idx);
|
| 877 | return ret;
|
| 878 | }
|
| 879 |
|
| 880 | long nvme_dev_ioctl(struct file *file, unsigned int cmd,
|
| 881 | unsigned long arg)
|
| 882 | {
|
| 883 | bool open_for_write = file->f_mode & FMODE_WRITE;
|
| 884 | struct nvme_ctrl *ctrl = file->private_data;
|
| 885 | void __user *argp = (void __user *)arg;
|
| 886 |
|
| 887 | switch (cmd) {
|
| 888 | case NVME_IOCTL_ADMIN_CMD:
|
| 889 | return nvme_user_cmd(ctrl, NULL, argp, 0, open_for_write);
|
| 890 | case NVME_IOCTL_ADMIN64_CMD:
|
| 891 | return nvme_user_cmd64(ctrl, NULL, argp, 0, open_for_write);
|
| 892 | case NVME_IOCTL_IO_CMD:
|
| 893 | return nvme_dev_user_cmd(ctrl, argp, open_for_write);
|
| 894 | case NVME_IOCTL_RESET:
|
| 895 | if (!capable(CAP_SYS_ADMIN))
|
| 896 | return -EACCES;
|
| 897 | dev_warn(ctrl->device, "resetting controller\n");
|
| 898 | return nvme_reset_ctrl_sync(ctrl);
|
| 899 | case NVME_IOCTL_SUBSYS_RESET:
|
| 900 | if (!capable(CAP_SYS_ADMIN))
|
| 901 | return -EACCES;
|
| 902 | return nvme_reset_subsystem(ctrl);
|
| 903 | case NVME_IOCTL_RESCAN:
|
| 904 | if (!capable(CAP_SYS_ADMIN))
|
| 905 | return -EACCES;
|
| 906 | nvme_queue_scan(ctrl);
|
| 907 | return 0;
|
| 908 | default:
|
| 909 | return -ENOTTY;
|
| 910 | }
|
| 911 | }
|