projects/linux·blob master · refs

785 lines · 18.4 KiB

1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3#include <linux/init.h>
4#include <linux/kernel.h>
5#include <linux/module.h>
6#include <linux/pci.h>
7#include <linux/device.h>
8#include <linux/sched/task.h>
9#include <linux/io-64-nonatomic-lo-hi.h>
10#include <linux/cdev.h>
11#include <linux/fs.h>
12#include <linux/poll.h>
13#include <linux/iommu.h>
14#include <linux/highmem.h>
15#include <uapi/linux/idxd.h>
16#include <linux/xarray.h>
17#include "registers.h"
18#include "idxd.h"
19
20struct idxd_cdev_context {
21 const char *name;
22 dev_t devt;
23 struct ida minor_ida;
24};
25
26/*
27 * Since user file names are global in DSA devices, define their ida's as
28 * global to avoid conflict file names.
29 */
30static DEFINE_IDA(file_ida);
31
32/*
33 * ictx is an array based off of accelerator types. enum idxd_type
34 * is used as index
35 */
36static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {
37 { .name = "dsa" },
38 { .name = "iax" }
39};
40
41struct idxd_user_context {
42 struct idxd_wq *wq;
43 struct task_struct *task;
44 unsigned int pasid;
45 struct mm_struct *mm;
46 unsigned int flags;
47 struct iommu_sva *sva;
48 struct idxd_dev idxd_dev;
49 u64 counters[COUNTER_MAX];
50 int id;
51 pid_t pid;
52};
53
54static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid);
55static void idxd_xa_pasid_remove(struct idxd_user_context *ctx);
56
57static inline struct idxd_user_context *dev_to_uctx(struct device *dev)
58{
59 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
60
61 return container_of(idxd_dev, struct idxd_user_context, idxd_dev);
62}
63
64static ssize_t cr_faults_show(struct device *dev, struct device_attribute *attr, char *buf)
65{
66 struct idxd_user_context *ctx = dev_to_uctx(dev);
67
68 return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULTS]);
69}
70static DEVICE_ATTR_RO(cr_faults);
71
72static ssize_t cr_fault_failures_show(struct device *dev,
73 struct device_attribute *attr, char *buf)
74{
75 struct idxd_user_context *ctx = dev_to_uctx(dev);
76
77 return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULT_FAILS]);
78}
79static DEVICE_ATTR_RO(cr_fault_failures);
80
81static ssize_t pid_show(struct device *dev, struct device_attribute *attr, char *buf)
82{
83 struct idxd_user_context *ctx = dev_to_uctx(dev);
84
85 return sysfs_emit(buf, "%u\n", ctx->pid);
86}
87static DEVICE_ATTR_RO(pid);
88
89static struct attribute *cdev_file_attributes[] = {
90 &dev_attr_cr_faults.attr,
91 &dev_attr_cr_fault_failures.attr,
92 &dev_attr_pid.attr,
93 NULL
94};
95
96static umode_t cdev_file_attr_visible(struct kobject *kobj, struct attribute *a, int n)
97{
98 struct device *dev = container_of(kobj, typeof(*dev), kobj);
99 struct idxd_user_context *ctx = dev_to_uctx(dev);
100 struct idxd_wq *wq = ctx->wq;
101
102 if (!wq_pasid_enabled(wq))
103 return 0;
104
105 return a->mode;
106}
107
108static const struct attribute_group cdev_file_attribute_group = {
109 .attrs = cdev_file_attributes,
110 .is_visible = cdev_file_attr_visible,
111};
112
113static const struct attribute_group *cdev_file_attribute_groups[] = {
114 &cdev_file_attribute_group,
115 NULL
116};
117
118static void idxd_file_dev_release(struct device *dev)
119{
120 struct idxd_user_context *ctx = dev_to_uctx(dev);
121 struct idxd_wq *wq = ctx->wq;
122 struct idxd_device *idxd = wq->idxd;
123 int rc;
124
125 ida_free(&file_ida, ctx->id);
126
127 /* Wait for in-flight operations to complete. */
128 if (wq_shared(wq)) {
129 idxd_device_drain_pasid(idxd, ctx->pasid);
130 } else {
131 if (device_user_pasid_enabled(idxd)) {
132 /* The wq disable in the disable pasid function will drain the wq */
133 rc = idxd_wq_disable_pasid(wq);
134 if (rc < 0)
135 dev_err(dev, "wq disable pasid failed.\n");
136 } else {
137 idxd_wq_drain(wq);
138 }
139 }
140
141 if (ctx->sva) {
142 idxd_cdev_evl_drain_pasid(wq, ctx->pasid);
143 iommu_sva_unbind_device(ctx->sva);
144 idxd_xa_pasid_remove(ctx);
145 }
146 kfree(ctx);
147 mutex_lock(&wq->wq_lock);
148 idxd_wq_put(wq);
149 mutex_unlock(&wq->wq_lock);
150}
151
152static const struct device_type idxd_cdev_file_type = {
153 .name = "idxd_file",
154 .release = idxd_file_dev_release,
155 .groups = cdev_file_attribute_groups,
156};
157
158static void idxd_cdev_dev_release(struct device *dev)
159{
160 struct idxd_cdev *idxd_cdev = dev_to_cdev(dev);
161
162 kfree(idxd_cdev);
163}
164
165static const struct device_type idxd_cdev_device_type = {
166 .name = "idxd_cdev",
167 .release = idxd_cdev_dev_release,
168};
169
170static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode)
171{
172 struct cdev *cdev = inode->i_cdev;
173
174 return container_of(cdev, struct idxd_cdev, cdev);
175}
176
177static inline struct idxd_wq *inode_wq(struct inode *inode)
178{
179 struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode);
180
181 return idxd_cdev->wq;
182}
183
184static void idxd_xa_pasid_remove(struct idxd_user_context *ctx)
185{
186 struct idxd_wq *wq = ctx->wq;
187 void *ptr;
188
189 mutex_lock(&wq->uc_lock);
190 ptr = xa_cmpxchg(&wq->upasid_xa, ctx->pasid, ctx, NULL, GFP_KERNEL);
191 if (ptr != (void *)ctx)
192 dev_warn(&wq->idxd->pdev->dev, "xarray cmpxchg failed for pasid %u\n",
193 ctx->pasid);
194 mutex_unlock(&wq->uc_lock);
195}
196
197void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index)
198{
199 struct idxd_user_context *ctx;
200
201 if (index >= COUNTER_MAX)
202 return;
203
204 mutex_lock(&wq->uc_lock);
205 ctx = xa_load(&wq->upasid_xa, pasid);
206 if (!ctx) {
207 mutex_unlock(&wq->uc_lock);
208 return;
209 }
210 ctx->counters[index]++;
211 mutex_unlock(&wq->uc_lock);
212}
213
214static int idxd_cdev_open(struct inode *inode, struct file *filp)
215{
216 struct idxd_user_context *ctx;
217 struct idxd_device *idxd;
218 struct idxd_wq *wq;
219 struct device *dev, *fdev;
220 int rc = 0;
221 struct iommu_sva *sva = NULL;
222 unsigned int pasid;
223 struct idxd_cdev *idxd_cdev;
224
225 wq = inode_wq(inode);
226 idxd = wq->idxd;
227 dev = &idxd->pdev->dev;
228
229 dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq));
230
231 ctx = kzalloc_obj(*ctx);
232 if (!ctx)
233 return -ENOMEM;
234
235 mutex_lock(&wq->wq_lock);
236
237 if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) {
238 rc = -EBUSY;
239 goto failed;
240 }
241
242 ctx->wq = wq;
243 filp->private_data = ctx;
244 ctx->pid = current->pid;
245
246 if (device_user_pasid_enabled(idxd)) {
247 sva = iommu_sva_bind_device(dev, current->mm);
248 if (IS_ERR(sva)) {
249 rc = PTR_ERR(sva);
250 dev_err(dev, "pasid allocation failed: %d\n", rc);
251 goto failed;
252 }
253
254 pasid = iommu_sva_get_pasid(sva);
255 if (pasid == IOMMU_PASID_INVALID) {
256 rc = -EINVAL;
257 goto failed_get_pasid;
258 }
259
260 ctx->sva = sva;
261 ctx->pasid = pasid;
262 ctx->mm = current->mm;
263
264 mutex_lock(&wq->uc_lock);
265 rc = xa_insert(&wq->upasid_xa, pasid, ctx, GFP_KERNEL);
266 mutex_unlock(&wq->uc_lock);
267 if (rc < 0)
268 dev_warn(dev, "PASID entry already exist in xarray.\n");
269
270 if (wq_dedicated(wq)) {
271 rc = idxd_wq_set_pasid(wq, pasid);
272 if (rc < 0) {
273 dev_err(dev, "wq set pasid failed: %d\n", rc);
274 goto failed_set_pasid;
275 }
276 }
277 }
278
279 idxd_cdev = wq->idxd_cdev;
280 ctx->id = ida_alloc(&file_ida, GFP_KERNEL);
281 if (ctx->id < 0) {
282 dev_warn(dev, "ida alloc failure\n");
283 goto failed_ida;
284 }
285 ctx->idxd_dev.type = IDXD_DEV_CDEV_FILE;
286 fdev = user_ctx_dev(ctx);
287 device_initialize(fdev);
288 fdev->parent = cdev_dev(idxd_cdev);
289 fdev->bus = &dsa_bus_type;
290 fdev->type = &idxd_cdev_file_type;
291 idxd_wq_get(wq);
292
293 rc = dev_set_name(fdev, "file%d", ctx->id);
294 if (rc < 0) {
295 dev_warn(dev, "set name failure\n");
296 goto failed_dev_name;
297 }
298
299 rc = device_add(fdev);
300 if (rc < 0) {
301 dev_warn(dev, "file device add failure\n");
302 goto failed_dev_add;
303 }
304
305 mutex_unlock(&wq->wq_lock);
306 return 0;
307
308failed_dev_add:
309failed_dev_name:
310 mutex_unlock(&wq->wq_lock);
311 put_device(fdev);
312 return rc;
313failed_ida:
314failed_set_pasid:
315 if (device_user_pasid_enabled(idxd))
316 idxd_xa_pasid_remove(ctx);
317failed_get_pasid:
318 if (device_user_pasid_enabled(idxd) && !IS_ERR_OR_NULL(sva))
319 iommu_sva_unbind_device(sva);
320failed:
321 mutex_unlock(&wq->wq_lock);
322 kfree(ctx);
323 return rc;
324}
325
326static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid)
327{
328 struct idxd_device *idxd = wq->idxd;
329 struct idxd_evl *evl = idxd->evl;
330 union evl_status_reg status;
331 u16 h, t, size;
332 int ent_size = evl_ent_size(idxd);
333 struct __evl_entry *entry_head;
334
335 if (!evl)
336 return;
337
338 mutex_lock(&evl->lock);
339 status.bits = ioread64(idxd->reg_base + IDXD_EVLSTATUS_OFFSET);
340 t = status.tail;
341 h = status.head;
342 size = evl->size;
343
344 while (h != t) {
345 entry_head = (struct __evl_entry *)(evl->log + (h * ent_size));
346 if (entry_head->pasid == pasid && entry_head->wq_idx == wq->id)
347 set_bit(h, evl->bmap);
348 h = (h + 1) % size;
349 }
350 if (wq->wq)
351 drain_workqueue(wq->wq);
352
353 mutex_unlock(&evl->lock);
354}
355
356static int idxd_cdev_release(struct inode *node, struct file *filep)
357{
358 struct idxd_user_context *ctx = filep->private_data;
359 struct idxd_wq *wq = ctx->wq;
360 struct idxd_device *idxd = wq->idxd;
361 struct device *dev = &idxd->pdev->dev;
362
363 dev_dbg(dev, "%s called\n", __func__);
364 filep->private_data = NULL;
365
366 device_unregister(user_ctx_dev(ctx));
367
368 return 0;
369}
370
371static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma,
372 const char *func)
373{
374 struct device *dev = &wq->idxd->pdev->dev;
375
376 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
377 dev_info_ratelimited(dev,
378 "%s: %s: mapping too large: %lu\n",
379 current->comm, func,
380 vma->vm_end - vma->vm_start);
381 return -EINVAL;
382 }
383
384 return 0;
385}
386
387static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma)
388{
389 struct idxd_user_context *ctx = filp->private_data;
390 struct idxd_wq *wq = ctx->wq;
391 struct idxd_device *idxd = wq->idxd;
392 struct pci_dev *pdev = idxd->pdev;
393 phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR);
394 unsigned long pfn;
395 int rc;
396
397 dev_dbg(&pdev->dev, "%s called\n", __func__);
398
399 /*
400 * Due to an erratum in some of the devices supported by the driver,
401 * direct user submission to the device can be unsafe.
402 * (See the INTEL-SA-01084 security advisory)
403 *
404 * For the devices that exhibit this behavior, require that the user
405 * has CAP_SYS_RAWIO capabilities.
406 */
407 if (!idxd->user_submission_safe && !capable(CAP_SYS_RAWIO))
408 return -EPERM;
409
410 if (current->mm != ctx->mm)
411 return -EPERM;
412
413 rc = check_vma(wq, vma, __func__);
414 if (rc < 0)
415 return rc;
416
417 vm_flags_set(vma, VM_DONTCOPY);
418 pfn = (base + idxd_get_wq_portal_full_offset(wq->id,
419 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT;
420 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
421 vma->vm_private_data = ctx;
422
423 return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,
424 vma->vm_page_prot);
425}
426
427static int idxd_submit_user_descriptor(struct idxd_user_context *ctx,
428 struct dsa_hw_desc __user *udesc)
429{
430 struct idxd_wq *wq = ctx->wq;
431 struct idxd_dev *idxd_dev = &wq->idxd->idxd_dev;
432 const uint64_t comp_addr_align = is_dsa_dev(idxd_dev) ? 0x20 : 0x40;
433 void __iomem *portal = idxd_wq_portal_addr(wq);
434 struct dsa_hw_desc descriptor __aligned(64);
435 int rc;
436
437 rc = copy_from_user(&descriptor, udesc, sizeof(descriptor));
438 if (rc)
439 return -EFAULT;
440
441 /*
442 * DSA devices are capable of indirect ("batch") command submission.
443 * On devices where direct user submissions are not safe, we cannot
444 * allow this since there is no good way for us to verify these
445 * indirect commands. Narrow the restriction of operations with the
446 * BATCH opcode to only DSA version 1 devices.
447 */
448 if (is_dsa_dev(idxd_dev) && descriptor.opcode == DSA_OPCODE_BATCH &&
449 wq->idxd->hw.version == DEVICE_VERSION_1 &&
450 !wq->idxd->user_submission_safe)
451 return -EINVAL;
452 /*
453 * As per the programming specification, the completion address must be
454 * aligned to 32 or 64 bytes. If this is violated the hardware
455 * engine can get very confused (security issue).
456 */
457 if (!IS_ALIGNED(descriptor.completion_addr, comp_addr_align))
458 return -EINVAL;
459
460 if (wq_dedicated(wq))
461 iosubmit_cmds512(portal, &descriptor, 1);
462 else {
463 descriptor.priv = 0;
464 descriptor.pasid = ctx->pasid;
465 rc = idxd_enqcmds(wq, portal, &descriptor);
466 if (rc < 0)
467 return rc;
468 }
469
470 return 0;
471}
472
473static ssize_t idxd_cdev_write(struct file *filp, const char __user *buf, size_t len,
474 loff_t *unused)
475{
476 struct dsa_hw_desc __user *udesc = (struct dsa_hw_desc __user *)buf;
477 struct idxd_user_context *ctx = filp->private_data;
478 ssize_t written = 0;
479 int i;
480
481 if (current->mm != ctx->mm)
482 return -EPERM;
483
484 for (i = 0; i < len/sizeof(struct dsa_hw_desc); i++) {
485 int rc = idxd_submit_user_descriptor(ctx, udesc + i);
486
487 if (rc)
488 return written ? written : rc;
489
490 written += sizeof(struct dsa_hw_desc);
491 }
492
493 return written;
494}
495
496static __poll_t idxd_cdev_poll(struct file *filp,
497 struct poll_table_struct *wait)
498{
499 struct idxd_user_context *ctx = filp->private_data;
500 struct idxd_wq *wq = ctx->wq;
501 struct idxd_device *idxd = wq->idxd;
502 __poll_t out = 0;
503
504 if (current->mm != ctx->mm)
505 return POLLNVAL;
506
507 poll_wait(filp, &wq->err_queue, wait);
508 spin_lock(&idxd->dev_lock);
509 if (idxd->sw_err.valid)
510 out = EPOLLIN | EPOLLRDNORM;
511 spin_unlock(&idxd->dev_lock);
512
513 return out;
514}
515
516static const struct file_operations idxd_cdev_fops = {
517 .owner = THIS_MODULE,
518 .open = idxd_cdev_open,
519 .release = idxd_cdev_release,
520 .mmap = idxd_cdev_mmap,
521 .write = idxd_cdev_write,
522 .poll = idxd_cdev_poll,
523};
524
525int idxd_cdev_get_major(struct idxd_device *idxd)
526{
527 return MAJOR(ictx[idxd->data->type].devt);
528}
529
530int idxd_wq_add_cdev(struct idxd_wq *wq)
531{
532 struct idxd_device *idxd = wq->idxd;
533 struct idxd_cdev *idxd_cdev;
534 struct cdev *cdev;
535 struct device *dev;
536 struct idxd_cdev_context *cdev_ctx;
537 int rc, minor;
538
539 idxd_cdev = kzalloc_obj(*idxd_cdev);
540 if (!idxd_cdev)
541 return -ENOMEM;
542
543 idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV;
544 idxd_cdev->wq = wq;
545 cdev = &idxd_cdev->cdev;
546 dev = cdev_dev(idxd_cdev);
547 cdev_ctx = &ictx[wq->idxd->data->type];
548 minor = ida_alloc_max(&cdev_ctx->minor_ida, MINORMASK, GFP_KERNEL);
549 if (minor < 0) {
550 kfree(idxd_cdev);
551 return minor;
552 }
553 idxd_cdev->minor = minor;
554
555 device_initialize(dev);
556 dev->parent = wq_confdev(wq);
557 dev->bus = &dsa_bus_type;
558 dev->type = &idxd_cdev_device_type;
559 dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
560
561 rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id);
562 if (rc < 0)
563 goto err;
564
565 wq->idxd_cdev = idxd_cdev;
566 cdev_init(cdev, &idxd_cdev_fops);
567 rc = cdev_device_add(cdev, dev);
568 if (rc) {
569 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);
570 goto err;
571 }
572
573 return 0;
574
575 err:
576 put_device(dev);
577 wq->idxd_cdev = NULL;
578 return rc;
579}
580
581void idxd_wq_del_cdev(struct idxd_wq *wq)
582{
583 struct idxd_cdev_context *cdev_ctx;
584 struct idxd_cdev *idxd_cdev;
585
586 idxd_cdev = wq->idxd_cdev;
587 wq->idxd_cdev = NULL;
588 cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev));
589
590 cdev_ctx = &ictx[wq->idxd->data->type];
591 ida_free(&cdev_ctx->minor_ida, idxd_cdev->minor);
592 put_device(cdev_dev(idxd_cdev));
593}
594
595static int idxd_user_drv_probe(struct idxd_dev *idxd_dev)
596{
597 struct device *dev = &idxd_dev->conf_dev;
598 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
599 struct idxd_device *idxd = wq->idxd;
600 int rc;
601
602 if (idxd->state != IDXD_DEV_ENABLED)
603 return -ENXIO;
604
605 mutex_lock(&wq->wq_lock);
606
607 if (!idxd_wq_driver_name_match(wq, dev)) {
608 idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME;
609 rc = -ENODEV;
610 goto wq_err;
611 }
612
613 /*
614 * User type WQ is enabled only when SVA is enabled for two reasons:
615 * - If no IOMMU or IOMMU Passthrough without SVA, userspace
616 * can directly access physical address through the WQ.
617 * - The IDXD cdev driver does not provide any ways to pin
618 * user pages and translate the address from user VA to IOVA or
619 * PA without IOMMU SVA. Therefore the application has no way
620 * to instruct the device to perform DMA function. This makes
621 * the cdev not usable for normal application usage.
622 */
623 if (!device_user_pasid_enabled(idxd)) {
624 idxd->cmd_status = IDXD_SCMD_WQ_USER_NO_IOMMU;
625 dev_dbg(&idxd->pdev->dev,
626 "User type WQ cannot be enabled without SVA.\n");
627
628 rc = -EOPNOTSUPP;
629 goto wq_err;
630 }
631
632 wq->wq = create_workqueue(dev_name(wq_confdev(wq)));
633 if (!wq->wq) {
634 rc = -ENOMEM;
635 goto wq_err;
636 }
637
638 wq->type = IDXD_WQT_USER;
639 rc = idxd_drv_enable_wq(wq);
640 if (rc < 0)
641 goto err;
642
643 rc = idxd_wq_add_cdev(wq);
644 if (rc < 0) {
645 idxd->cmd_status = IDXD_SCMD_CDEV_ERR;
646 goto err_cdev;
647 }
648
649 idxd->cmd_status = 0;
650 mutex_unlock(&wq->wq_lock);
651 return 0;
652
653err_cdev:
654 idxd_drv_disable_wq(wq);
655err:
656 destroy_workqueue(wq->wq);
657 wq->type = IDXD_WQT_NONE;
658wq_err:
659 mutex_unlock(&wq->wq_lock);
660 return rc;
661}
662
663static void idxd_user_drv_remove(struct idxd_dev *idxd_dev)
664{
665 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
666
667 mutex_lock(&wq->wq_lock);
668 idxd_wq_del_cdev(wq);
669 idxd_drv_disable_wq(wq);
670 wq->type = IDXD_WQT_NONE;
671 destroy_workqueue(wq->wq);
672 wq->wq = NULL;
673 mutex_unlock(&wq->wq_lock);
674}
675
676static enum idxd_dev_type dev_types[] = {
677 IDXD_DEV_WQ,
678 IDXD_DEV_NONE,
679};
680
681struct idxd_device_driver idxd_user_drv = {
682 .probe = idxd_user_drv_probe,
683 .remove = idxd_user_drv_remove,
684 .name = "user",
685 .type = dev_types,
686};
687EXPORT_SYMBOL_GPL(idxd_user_drv);
688
689int idxd_cdev_register(void)
690{
691 int rc, i;
692
693 for (i = 0; i < IDXD_TYPE_MAX; i++) {
694 ida_init(&ictx[i].minor_ida);
695 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK,
696 ictx[i].name);
697 if (rc)
698 goto err_free_chrdev_region;
699 }
700
701 return 0;
702
703err_free_chrdev_region:
704 for (i--; i >= 0; i--)
705 unregister_chrdev_region(ictx[i].devt, MINORMASK);
706
707 return rc;
708}
709
710void idxd_cdev_remove(void)
711{
712 int i;
713
714 for (i = 0; i < IDXD_TYPE_MAX; i++) {
715 unregister_chrdev_region(ictx[i].devt, MINORMASK);
716 ida_destroy(&ictx[i].minor_ida);
717 }
718}
719
720/**
721 * idxd_copy_cr - copy completion record to user address space found by wq and
722 * PASID
723 * @wq: work queue
724 * @pasid: PASID
725 * @addr: user fault address to write
726 * @cr: completion record
727 * @len: number of bytes to copy
728 *
729 * This is called by a work that handles completion record fault.
730 *
731 * Return: number of bytes copied.
732 */
733int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr,
734 void *cr, int len)
735{
736 struct device *dev = &wq->idxd->pdev->dev;
737 int left = len, status_size = 1;
738 struct idxd_user_context *ctx;
739 struct mm_struct *mm;
740
741 mutex_lock(&wq->uc_lock);
742
743 ctx = xa_load(&wq->upasid_xa, pasid);
744 if (!ctx) {
745 dev_warn(dev, "No user context\n");
746 goto out;
747 }
748
749 mm = ctx->mm;
750 /*
751 * The completion record fault handling work is running in kernel
752 * thread context. It temporarily switches to the mm to copy cr
753 * to addr in the mm.
754 */
755 kthread_use_mm(mm);
756 left = copy_to_user((void __user *)addr + status_size, cr + status_size,
757 len - status_size);
758 /*
759 * Copy status only after the rest of completion record is copied
760 * successfully so that the user gets the complete completion record
761 * when a non-zero status is polled.
762 */
763 if (!left) {
764 u8 status;
765
766 /*
767 * Ensure that the completion record's status field is written
768 * after the rest of the completion record has been written.
769 * This ensures that the user receives the correct completion
770 * record information once polling for a non-zero status.
771 */
772 wmb();
773 status = *(u8 *)cr;
774 if (put_user(status, (u8 __user *)addr))
775 left += status_size;
776 } else {
777 left += status_size;
778 }
779 kthread_unuse_mm(mm);
780
781out:
782 mutex_unlock(&wq->uc_lock);
783
784 return len - left;
785}