projects/linux·blob master · refs

1317 lines · 33.6 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright(c) 2020 Intel Corporation. */
3
4#include <linux/io-64-nonatomic-lo-hi.h>
5#include <linux/firmware.h>
6#include <linux/device.h>
7#include <linux/slab.h>
8#include <linux/idr.h>
9#include <linux/pci.h>
10#include <cxlmem.h>
11#include "trace.h"
12#include "core.h"
13
14static DECLARE_RWSEM(cxl_memdev_rwsem);
15
16/*
17 * An entire PCI topology full of devices should be enough for any
18 * config
19 */
20#define CXL_MEM_MAX_DEVS 65536
21
22static int cxl_mem_major;
23static DEFINE_IDA(cxl_memdev_ida);
24
25static void cxl_memdev_release(struct device *dev)
26{
27 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
28 struct device *parent = dev->parent;
29
30 ida_free(&cxl_memdev_ida, cxlmd->id);
31 kfree(cxlmd);
32 put_device(parent);
33}
34
35static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid,
36 kgid_t *gid)
37{
38 return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev));
39}
40
41static ssize_t firmware_version_show(struct device *dev,
42 struct device_attribute *attr, char *buf)
43{
44 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
45 struct cxl_dev_state *cxlds = cxlmd->cxlds;
46 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
47
48 if (!mds)
49 return sysfs_emit(buf, "\n");
50 return sysfs_emit(buf, "%.16s\n", mds->firmware_version);
51}
52static DEVICE_ATTR_RO(firmware_version);
53
54static ssize_t payload_max_show(struct device *dev,
55 struct device_attribute *attr, char *buf)
56{
57 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
58 struct cxl_dev_state *cxlds = cxlmd->cxlds;
59 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
60
61 if (!mds)
62 return sysfs_emit(buf, "\n");
63 return sysfs_emit(buf, "%zu\n", cxlds->cxl_mbox.payload_size);
64}
65static DEVICE_ATTR_RO(payload_max);
66
67static ssize_t label_storage_size_show(struct device *dev,
68 struct device_attribute *attr, char *buf)
69{
70 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
71 struct cxl_dev_state *cxlds = cxlmd->cxlds;
72 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
73
74 if (!mds)
75 return sysfs_emit(buf, "\n");
76 return sysfs_emit(buf, "%zu\n", mds->lsa_size);
77}
78static DEVICE_ATTR_RO(label_storage_size);
79
80static resource_size_t cxl_ram_size(struct cxl_dev_state *cxlds)
81{
82 /* Static RAM is only expected at partition 0. */
83 if (cxlds->part[0].mode != CXL_PARTMODE_RAM)
84 return 0;
85 return resource_size(&cxlds->part[0].res);
86}
87
88static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr,
89 char *buf)
90{
91 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
92 struct cxl_dev_state *cxlds = cxlmd->cxlds;
93 unsigned long long len = cxl_ram_size(cxlds);
94
95 return sysfs_emit(buf, "%#llx\n", len);
96}
97
98static struct device_attribute dev_attr_ram_size =
99 __ATTR(size, 0444, ram_size_show, NULL);
100
101static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr,
102 char *buf)
103{
104 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
105 struct cxl_dev_state *cxlds = cxlmd->cxlds;
106 unsigned long long len = cxl_pmem_size(cxlds);
107
108 return sysfs_emit(buf, "%#llx\n", len);
109}
110
111static struct device_attribute dev_attr_pmem_size =
112 __ATTR(size, 0444, pmem_size_show, NULL);
113
114static ssize_t serial_show(struct device *dev, struct device_attribute *attr,
115 char *buf)
116{
117 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
118 struct cxl_dev_state *cxlds = cxlmd->cxlds;
119
120 return sysfs_emit(buf, "%#llx\n", cxlds->serial);
121}
122static DEVICE_ATTR_RO(serial);
123
124static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr,
125 char *buf)
126{
127 return sysfs_emit(buf, "%d\n", dev_to_node(dev));
128}
129static DEVICE_ATTR_RO(numa_node);
130
131static ssize_t security_state_show(struct device *dev,
132 struct device_attribute *attr,
133 char *buf)
134{
135 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
136 struct cxl_dev_state *cxlds = cxlmd->cxlds;
137 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
138 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
139 unsigned long state = mds->security.state;
140 int rc = 0;
141
142 /* sync with latest submission state */
143 mutex_lock(&cxl_mbox->mbox_mutex);
144 if (mds->security.sanitize_active)
145 rc = sysfs_emit(buf, "sanitize\n");
146 mutex_unlock(&cxl_mbox->mbox_mutex);
147 if (rc)
148 return rc;
149
150 if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET))
151 return sysfs_emit(buf, "disabled\n");
152 if (state & CXL_PMEM_SEC_STATE_FROZEN ||
153 state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT ||
154 state & CXL_PMEM_SEC_STATE_USER_PLIMIT)
155 return sysfs_emit(buf, "frozen\n");
156 if (state & CXL_PMEM_SEC_STATE_LOCKED)
157 return sysfs_emit(buf, "locked\n");
158
159 return sysfs_emit(buf, "unlocked\n");
160}
161static struct device_attribute dev_attr_security_state =
162 __ATTR(state, 0444, security_state_show, NULL);
163
164static ssize_t security_sanitize_store(struct device *dev,
165 struct device_attribute *attr,
166 const char *buf, size_t len)
167{
168 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
169 bool sanitize;
170 ssize_t rc;
171
172 if (kstrtobool(buf, &sanitize) || !sanitize)
173 return -EINVAL;
174
175 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SANITIZE);
176 if (rc)
177 return rc;
178
179 return len;
180}
181static struct device_attribute dev_attr_security_sanitize =
182 __ATTR(sanitize, 0200, NULL, security_sanitize_store);
183
184static ssize_t security_erase_store(struct device *dev,
185 struct device_attribute *attr,
186 const char *buf, size_t len)
187{
188 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
189 ssize_t rc;
190 bool erase;
191
192 if (kstrtobool(buf, &erase) || !erase)
193 return -EINVAL;
194
195 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SECURE_ERASE);
196 if (rc)
197 return rc;
198
199 return len;
200}
201static struct device_attribute dev_attr_security_erase =
202 __ATTR(erase, 0200, NULL, security_erase_store);
203
204bool cxl_memdev_has_poison_cmd(struct cxl_memdev *cxlmd,
205 enum poison_cmd_enabled_bits cmd)
206{
207 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
208
209 if (!mds)
210 return 0;
211
212 return test_bit(cmd, mds->poison.enabled_cmds);
213}
214
215static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd)
216{
217 struct cxl_dev_state *cxlds = cxlmd->cxlds;
218 u64 offset, length;
219 int rc = 0;
220
221 /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */
222 for (int i = 0; i < cxlds->nr_partitions; i++) {
223 const struct resource *res = &cxlds->part[i].res;
224
225 offset = res->start;
226 length = resource_size(res);
227 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
228 /*
229 * Invalid Physical Address is not an error for
230 * volatile addresses. Device support is optional.
231 */
232 if (rc == -EFAULT && cxlds->part[i].mode == CXL_PARTMODE_RAM)
233 rc = 0;
234 }
235 return rc;
236}
237
238int cxl_trigger_poison_list(struct cxl_memdev *cxlmd)
239{
240 struct cxl_port *port;
241 int rc;
242
243 port = cxlmd->endpoint;
244 if (!port || !is_cxl_endpoint(port))
245 return -EINVAL;
246
247 ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);
248 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))
249 return rc;
250
251 ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa);
252 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
253 return rc;
254
255 if (cxl_num_decoders_committed(port) == 0) {
256 /* No regions mapped to this memdev */
257 rc = cxl_get_poison_by_memdev(cxlmd);
258 } else {
259 /* Regions mapped, collect poison by endpoint */
260 rc = cxl_get_poison_by_endpoint(port);
261 }
262
263 return rc;
264}
265EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, "CXL");
266
267static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa)
268{
269 struct cxl_dev_state *cxlds = cxlmd->cxlds;
270
271 if (!IS_ENABLED(CONFIG_DEBUG_FS))
272 return 0;
273
274 if (!resource_size(&cxlds->dpa_res)) {
275 dev_dbg(cxlds->dev, "device has no dpa resource\n");
276 return -EINVAL;
277 }
278 if (!cxl_resource_contains_addr(&cxlds->dpa_res, dpa)) {
279 dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n",
280 dpa, &cxlds->dpa_res);
281 return -EINVAL;
282 }
283 if (!IS_ALIGNED(dpa, 64)) {
284 dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa);
285 return -EINVAL;
286 }
287
288 return 0;
289}
290
291int cxl_inject_poison_locked(struct cxl_memdev *cxlmd, u64 dpa)
292{
293 struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
294 struct cxl_mbox_inject_poison inject;
295 struct cxl_poison_record record;
296 struct cxl_mbox_cmd mbox_cmd;
297 struct cxl_region *cxlr;
298 int rc;
299
300 if (!IS_ENABLED(CONFIG_DEBUG_FS))
301 return 0;
302
303 lockdep_assert_held(&cxl_rwsem.dpa);
304 lockdep_assert_held(&cxl_rwsem.region);
305
306 rc = cxl_validate_poison_dpa(cxlmd, dpa);
307 if (rc)
308 return rc;
309
310 inject.address = cpu_to_le64(dpa);
311 mbox_cmd = (struct cxl_mbox_cmd) {
312 .opcode = CXL_MBOX_OP_INJECT_POISON,
313 .size_in = sizeof(inject),
314 .payload_in = &inject,
315 };
316 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
317 if (rc)
318 return rc;
319
320 cxlr = cxl_dpa_to_region(cxlmd, dpa);
321 if (cxlr)
322 dev_warn_once(cxl_mbox->host,
323 "poison inject dpa:%#llx region: %s\n", dpa,
324 dev_name(&cxlr->dev));
325
326 record = (struct cxl_poison_record) {
327 .address = cpu_to_le64(dpa),
328 .length = cpu_to_le32(1),
329 };
330 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT);
331
332 return 0;
333}
334
335int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa)
336{
337 int rc;
338
339 ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);
340 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))
341 return rc;
342
343 ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa);
344 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
345 return rc;
346
347 return cxl_inject_poison_locked(cxlmd, dpa);
348}
349EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, "CXL");
350
351int cxl_clear_poison_locked(struct cxl_memdev *cxlmd, u64 dpa)
352{
353 struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
354 struct cxl_mbox_clear_poison clear;
355 struct cxl_poison_record record;
356 struct cxl_mbox_cmd mbox_cmd;
357 struct cxl_region *cxlr;
358 int rc;
359
360 if (!IS_ENABLED(CONFIG_DEBUG_FS))
361 return 0;
362
363 lockdep_assert_held(&cxl_rwsem.dpa);
364 lockdep_assert_held(&cxl_rwsem.region);
365
366 rc = cxl_validate_poison_dpa(cxlmd, dpa);
367 if (rc)
368 return rc;
369
370 /*
371 * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command
372 * is defined to accept 64 bytes of write-data, along with the
373 * address to clear. This driver uses zeroes as write-data.
374 */
375 clear = (struct cxl_mbox_clear_poison) {
376 .address = cpu_to_le64(dpa)
377 };
378
379 mbox_cmd = (struct cxl_mbox_cmd) {
380 .opcode = CXL_MBOX_OP_CLEAR_POISON,
381 .size_in = sizeof(clear),
382 .payload_in = &clear,
383 };
384
385 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
386 if (rc)
387 return rc;
388
389 cxlr = cxl_dpa_to_region(cxlmd, dpa);
390 if (cxlr)
391 dev_warn_once(cxl_mbox->host,
392 "poison clear dpa:%#llx region: %s\n", dpa,
393 dev_name(&cxlr->dev));
394
395 record = (struct cxl_poison_record) {
396 .address = cpu_to_le64(dpa),
397 .length = cpu_to_le32(1),
398 };
399 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR);
400
401 return 0;
402}
403
404int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa)
405{
406 int rc;
407
408 ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);
409 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))
410 return rc;
411
412 ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa);
413 if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
414 return rc;
415
416 return cxl_clear_poison_locked(cxlmd, dpa);
417}
418EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, "CXL");
419
420static struct attribute *cxl_memdev_attributes[] = {
421 &dev_attr_serial.attr,
422 &dev_attr_firmware_version.attr,
423 &dev_attr_payload_max.attr,
424 &dev_attr_label_storage_size.attr,
425 &dev_attr_numa_node.attr,
426 NULL,
427};
428
429static struct cxl_dpa_perf *to_pmem_perf(struct cxl_dev_state *cxlds)
430{
431 for (int i = 0; i < cxlds->nr_partitions; i++)
432 if (cxlds->part[i].mode == CXL_PARTMODE_PMEM)
433 return &cxlds->part[i].perf;
434 return NULL;
435}
436
437static ssize_t pmem_qos_class_show(struct device *dev,
438 struct device_attribute *attr, char *buf)
439{
440 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
441 struct cxl_dev_state *cxlds = cxlmd->cxlds;
442
443 return sysfs_emit(buf, "%d\n", to_pmem_perf(cxlds)->qos_class);
444}
445
446static struct device_attribute dev_attr_pmem_qos_class =
447 __ATTR(qos_class, 0444, pmem_qos_class_show, NULL);
448
449static struct attribute *cxl_memdev_pmem_attributes[] = {
450 &dev_attr_pmem_size.attr,
451 &dev_attr_pmem_qos_class.attr,
452 NULL,
453};
454
455static struct cxl_dpa_perf *to_ram_perf(struct cxl_dev_state *cxlds)
456{
457 if (cxlds->part[0].mode != CXL_PARTMODE_RAM)
458 return NULL;
459 return &cxlds->part[0].perf;
460}
461
462static ssize_t ram_qos_class_show(struct device *dev,
463 struct device_attribute *attr, char *buf)
464{
465 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
466 struct cxl_dev_state *cxlds = cxlmd->cxlds;
467
468 return sysfs_emit(buf, "%d\n", to_ram_perf(cxlds)->qos_class);
469}
470
471static struct device_attribute dev_attr_ram_qos_class =
472 __ATTR(qos_class, 0444, ram_qos_class_show, NULL);
473
474static struct attribute *cxl_memdev_ram_attributes[] = {
475 &dev_attr_ram_size.attr,
476 &dev_attr_ram_qos_class.attr,
477 NULL,
478};
479
480static struct attribute *cxl_memdev_security_attributes[] = {
481 &dev_attr_security_state.attr,
482 &dev_attr_security_sanitize.attr,
483 &dev_attr_security_erase.attr,
484 NULL,
485};
486
487static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a,
488 int n)
489{
490 if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr)
491 return 0;
492 return a->mode;
493}
494
495static struct attribute_group cxl_memdev_attribute_group = {
496 .attrs = cxl_memdev_attributes,
497 .is_visible = cxl_memdev_visible,
498};
499
500static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n)
501{
502 struct device *dev = kobj_to_dev(kobj);
503 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
504 struct cxl_dpa_perf *perf = to_ram_perf(cxlmd->cxlds);
505
506 if (a == &dev_attr_ram_qos_class.attr &&
507 (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID))
508 return 0;
509
510 return a->mode;
511}
512
513static struct attribute_group cxl_memdev_ram_attribute_group = {
514 .name = "ram",
515 .attrs = cxl_memdev_ram_attributes,
516 .is_visible = cxl_ram_visible,
517};
518
519static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n)
520{
521 struct device *dev = kobj_to_dev(kobj);
522 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
523 struct cxl_dpa_perf *perf = to_pmem_perf(cxlmd->cxlds);
524
525 if (a == &dev_attr_pmem_qos_class.attr &&
526 (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID))
527 return 0;
528
529 return a->mode;
530}
531
532static struct attribute_group cxl_memdev_pmem_attribute_group = {
533 .name = "pmem",
534 .attrs = cxl_memdev_pmem_attributes,
535 .is_visible = cxl_pmem_visible,
536};
537
538static umode_t cxl_memdev_security_visible(struct kobject *kobj,
539 struct attribute *a, int n)
540{
541 struct device *dev = kobj_to_dev(kobj);
542 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
543 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
544
545 if (a == &dev_attr_security_sanitize.attr &&
546 !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))
547 return 0;
548
549 if (a == &dev_attr_security_erase.attr &&
550 !test_bit(CXL_SEC_ENABLED_SECURE_ERASE, mds->security.enabled_cmds))
551 return 0;
552
553 return a->mode;
554}
555
556static struct attribute_group cxl_memdev_security_attribute_group = {
557 .name = "security",
558 .attrs = cxl_memdev_security_attributes,
559 .is_visible = cxl_memdev_security_visible,
560};
561
562static const struct attribute_group *cxl_memdev_attribute_groups[] = {
563 &cxl_memdev_attribute_group,
564 &cxl_memdev_ram_attribute_group,
565 &cxl_memdev_pmem_attribute_group,
566 &cxl_memdev_security_attribute_group,
567 NULL,
568};
569
570void cxl_memdev_update_perf(struct cxl_memdev *cxlmd)
571{
572 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_ram_attribute_group);
573 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_pmem_attribute_group);
574}
575EXPORT_SYMBOL_NS_GPL(cxl_memdev_update_perf, "CXL");
576
577static const struct device_type cxl_class_memdev_type = {
578 .name = "cxl_memdev",
579 .release = cxl_memdev_release,
580 .devnode = cxl_memdev_devnode,
581 .groups = cxl_memdev_attribute_groups,
582};
583
584static const struct device_type cxl_memdev_type = {
585 .name = "cxl_memdev",
586 .release = cxl_memdev_release,
587 .devnode = cxl_memdev_devnode,
588};
589
590bool is_cxl_memdev(const struct device *dev)
591{
592 return (dev->type == &cxl_class_memdev_type ||
593 dev->type == &cxl_memdev_type);
594}
595EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, "CXL");
596
597static void add_part(struct cxl_dpa_info *info, u64 start, u64 size, enum cxl_partition_mode mode)
598{
599 int i = info->nr_partitions;
600
601 if (size == 0)
602 return;
603
604 info->part[i].range = (struct range) {
605 .start = start,
606 .end = start + size - 1,
607 };
608 info->part[i].mode = mode;
609 info->nr_partitions++;
610}
611
612int cxl_mem_dpa_fetch(struct cxl_memdev_state *mds, struct cxl_dpa_info *info)
613{
614 struct cxl_dev_state *cxlds = &mds->cxlds;
615 struct device *dev = cxlds->dev;
616 int rc;
617
618 if (!cxlds->media_ready) {
619 info->size = 0;
620 return 0;
621 }
622
623 info->size = mds->total_bytes;
624
625 if (mds->partition_align_bytes == 0) {
626 add_part(info, 0, mds->volatile_only_bytes, CXL_PARTMODE_RAM);
627 add_part(info, mds->volatile_only_bytes,
628 mds->persistent_only_bytes, CXL_PARTMODE_PMEM);
629 return 0;
630 }
631
632 rc = cxl_mem_get_partition_info(mds);
633 if (rc) {
634 dev_err(dev, "Failed to query partition information\n");
635 return rc;
636 }
637
638 add_part(info, 0, mds->active_volatile_bytes, CXL_PARTMODE_RAM);
639 add_part(info, mds->active_volatile_bytes, mds->active_persistent_bytes,
640 CXL_PARTMODE_PMEM);
641
642 return 0;
643}
644EXPORT_SYMBOL_NS_GPL(cxl_mem_dpa_fetch, "CXL");
645
646
647/**
648 * cxl_set_capacity: initialize dpa by a driver without a mailbox.
649 *
650 * @cxlds: pointer to cxl_dev_state
651 * @capacity: device volatile memory size
652 */
653int cxl_set_capacity(struct cxl_dev_state *cxlds, u64 capacity)
654{
655 struct cxl_dpa_info range_info = {
656 .size = capacity,
657 };
658
659 add_part(&range_info, 0, capacity, CXL_PARTMODE_RAM);
660 return cxl_dpa_setup(cxlds, &range_info);
661}
662EXPORT_SYMBOL_NS_GPL(cxl_set_capacity, "CXL");
663
664/**
665 * set_exclusive_cxl_commands() - atomically disable user cxl commands
666 * @mds: The device state to operate on
667 * @cmds: bitmap of commands to mark exclusive
668 *
669 * Grab the cxl_memdev_rwsem in write mode to flush in-flight
670 * invocations of the ioctl path and then disable future execution of
671 * commands with the command ids set in @cmds.
672 */
673void set_exclusive_cxl_commands(struct cxl_memdev_state *mds,
674 unsigned long *cmds)
675{
676 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
677
678 guard(rwsem_write)(&cxl_memdev_rwsem);
679 bitmap_or(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds,
680 cmds, CXL_MEM_COMMAND_ID_MAX);
681}
682EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, "CXL");
683
684/**
685 * clear_exclusive_cxl_commands() - atomically enable user cxl commands
686 * @mds: The device state to modify
687 * @cmds: bitmap of commands to mark available for userspace
688 */
689void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds,
690 unsigned long *cmds)
691{
692 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
693
694 guard(rwsem_write)(&cxl_memdev_rwsem);
695 bitmap_andnot(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds,
696 cmds, CXL_MEM_COMMAND_ID_MAX);
697}
698EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, "CXL");
699
700static void cxl_memdev_shutdown(struct device *dev)
701{
702 struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
703
704 guard(rwsem_write)(&cxl_memdev_rwsem);
705 cxlmd->cxlds = NULL;
706}
707
708static void cxl_memdev_unregister(void *_cxlmd)
709{
710 struct cxl_memdev *cxlmd = _cxlmd;
711 struct device *dev = &cxlmd->dev;
712
713 cdev_device_del(&cxlmd->cdev, dev);
714 cxl_memdev_shutdown(dev);
715 put_device(dev);
716}
717
718static void detach_memdev(struct work_struct *work)
719{
720 struct cxl_memdev *cxlmd;
721
722 cxlmd = container_of(work, typeof(*cxlmd), detach_work);
723
724 /*
725 * When the creator of @cxlmd sets ->attach it indicates CXL operation
726 * is required. In that case, @cxlmd detach escalates to parent device
727 * detach.
728 */
729 if (cxlmd->attach)
730 device_release_driver(cxlmd->dev.parent);
731 else
732 device_release_driver(&cxlmd->dev);
733 put_device(&cxlmd->dev);
734}
735
736static struct lock_class_key cxl_memdev_key;
737
738struct cxl_dev_state *_devm_cxl_dev_state_create(struct device *dev,
739 enum cxl_devtype type,
740 u64 serial, u16 dvsec,
741 size_t size, bool has_mbox)
742{
743 struct cxl_dev_state *cxlds = devm_kzalloc(dev, size, GFP_KERNEL);
744
745 if (!cxlds)
746 return NULL;
747
748 cxlds->dev = dev;
749 cxlds->type = type;
750 cxlds->serial = serial;
751 cxlds->cxl_dvsec = dvsec;
752 cxlds->reg_map.host = dev;
753 cxlds->reg_map.resource = CXL_RESOURCE_NONE;
754
755 if (has_mbox)
756 cxlds->cxl_mbox.host = dev;
757
758 return cxlds;
759}
760EXPORT_SYMBOL_NS_GPL(_devm_cxl_dev_state_create, "CXL");
761
762static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
763 const struct file_operations *fops,
764 const struct cxl_memdev_attach *attach)
765{
766 struct cxl_memdev *cxlmd;
767 struct device *dev;
768 struct cdev *cdev;
769 int rc;
770
771 cxlmd = kzalloc_obj(*cxlmd);
772 if (!cxlmd)
773 return ERR_PTR(-ENOMEM);
774
775 rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL);
776 if (rc < 0)
777 goto err;
778 cxlmd->id = rc;
779 cxlmd->depth = -1;
780 cxlmd->attach = attach;
781 cxlmd->endpoint = ERR_PTR(-ENXIO);
782
783 dev = &cxlmd->dev;
784 device_initialize(dev);
785 lockdep_set_class(&dev->mutex, &cxl_memdev_key);
786 dev->parent = get_device(cxlds->dev);
787 dev->bus = &cxl_bus_type;
788 dev->devt = MKDEV(cxl_mem_major, cxlmd->id);
789 if (cxlds->type == CXL_DEVTYPE_DEVMEM)
790 dev->type = &cxl_memdev_type;
791 else
792 dev->type = &cxl_class_memdev_type;
793 device_set_pm_not_required(dev);
794 INIT_WORK(&cxlmd->detach_work, detach_memdev);
795
796 cdev = &cxlmd->cdev;
797 cdev_init(cdev, fops);
798 return cxlmd;
799
800err:
801 kfree(cxlmd);
802 return ERR_PTR(rc);
803}
804
805static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd,
806 unsigned long arg)
807{
808 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
809 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
810
811 switch (cmd) {
812 case CXL_MEM_QUERY_COMMANDS:
813 return cxl_query_cmd(cxl_mbox, (void __user *)arg);
814 case CXL_MEM_SEND_COMMAND:
815 return cxl_send_cmd(cxl_mbox, (void __user *)arg);
816 default:
817 return -ENOTTY;
818 }
819}
820
821static long cxl_memdev_ioctl(struct file *file, unsigned int cmd,
822 unsigned long arg)
823{
824 struct cxl_memdev *cxlmd = file->private_data;
825 struct cxl_dev_state *cxlds;
826
827 guard(rwsem_read)(&cxl_memdev_rwsem);
828 cxlds = cxlmd->cxlds;
829 if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM)
830 return __cxl_memdev_ioctl(cxlmd, cmd, arg);
831
832 return -ENXIO;
833}
834
835static int cxl_memdev_open(struct inode *inode, struct file *file)
836{
837 struct cxl_memdev *cxlmd =
838 container_of(inode->i_cdev, typeof(*cxlmd), cdev);
839
840 get_device(&cxlmd->dev);
841 file->private_data = cxlmd;
842
843 return 0;
844}
845
846static int cxl_memdev_release_file(struct inode *inode, struct file *file)
847{
848 struct cxl_memdev *cxlmd =
849 container_of(inode->i_cdev, typeof(*cxlmd), cdev);
850
851 put_device(&cxlmd->dev);
852
853 return 0;
854}
855
856/**
857 * cxl_mem_get_fw_info - Get Firmware info
858 * @mds: The device data for the operation
859 *
860 * Retrieve firmware info for the device specified.
861 *
862 * Return: 0 if no error: or the result of the mailbox command.
863 *
864 * See CXL-3.0 8.2.9.3.1 Get FW Info
865 */
866static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds)
867{
868 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
869 struct cxl_mbox_get_fw_info info;
870 struct cxl_mbox_cmd mbox_cmd;
871 int rc;
872
873 mbox_cmd = (struct cxl_mbox_cmd) {
874 .opcode = CXL_MBOX_OP_GET_FW_INFO,
875 .size_out = sizeof(info),
876 .payload_out = &info,
877 };
878
879 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
880 if (rc < 0)
881 return rc;
882
883 mds->fw.num_slots = info.num_slots;
884 mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK,
885 info.slot_info);
886
887 return 0;
888}
889
890/**
891 * cxl_mem_activate_fw - Activate Firmware
892 * @mds: The device data for the operation
893 * @slot: slot number to activate
894 *
895 * Activate firmware in a given slot for the device specified.
896 *
897 * Return: 0 if no error: or the result of the mailbox command.
898 *
899 * See CXL-3.0 8.2.9.3.3 Activate FW
900 */
901static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot)
902{
903 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
904 struct cxl_mbox_activate_fw activate;
905 struct cxl_mbox_cmd mbox_cmd;
906
907 if (slot == 0 || slot > mds->fw.num_slots)
908 return -EINVAL;
909
910 mbox_cmd = (struct cxl_mbox_cmd) {
911 .opcode = CXL_MBOX_OP_ACTIVATE_FW,
912 .size_in = sizeof(activate),
913 .payload_in = &activate,
914 };
915
916 /* Only offline activation supported for now */
917 activate.action = CXL_FW_ACTIVATE_OFFLINE;
918 activate.slot = slot;
919
920 return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
921}
922
923/**
924 * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer
925 * @mds: The device data for the operation
926 *
927 * Abort an in-progress firmware transfer for the device specified.
928 *
929 * Return: 0 if no error: or the result of the mailbox command.
930 *
931 * See CXL-3.0 8.2.9.3.2 Transfer FW
932 */
933static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds)
934{
935 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
936 struct cxl_mbox_transfer_fw *transfer;
937 struct cxl_mbox_cmd mbox_cmd;
938 int rc;
939
940 transfer = kzalloc_flex(*transfer, data, 0);
941 if (!transfer)
942 return -ENOMEM;
943
944 /* Set a 1s poll interval and a total wait time of 30s */
945 mbox_cmd = (struct cxl_mbox_cmd) {
946 .opcode = CXL_MBOX_OP_TRANSFER_FW,
947 .size_in = sizeof(*transfer),
948 .payload_in = transfer,
949 .poll_interval_ms = 1000,
950 .poll_count = 30,
951 };
952
953 transfer->action = CXL_FW_TRANSFER_ACTION_ABORT;
954
955 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
956 kfree(transfer);
957 return rc;
958}
959
960static void cxl_fw_cleanup(struct fw_upload *fwl)
961{
962 struct cxl_memdev_state *mds = fwl->dd_handle;
963
964 mds->fw.next_slot = 0;
965}
966
967static int cxl_fw_do_cancel(struct fw_upload *fwl)
968{
969 struct cxl_memdev_state *mds = fwl->dd_handle;
970 struct cxl_dev_state *cxlds = &mds->cxlds;
971 struct cxl_memdev *cxlmd = cxlds->cxlmd;
972 int rc;
973
974 rc = cxl_mem_abort_fw_xfer(mds);
975 if (rc < 0)
976 dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc);
977
978 return FW_UPLOAD_ERR_CANCELED;
979}
980
981static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data,
982 u32 size)
983{
984 struct cxl_memdev_state *mds = fwl->dd_handle;
985 struct cxl_mbox_transfer_fw *transfer;
986 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
987
988 if (!size)
989 return FW_UPLOAD_ERR_INVALID_SIZE;
990
991 mds->fw.oneshot = struct_size(transfer, data, size) <=
992 cxl_mbox->payload_size;
993
994 if (cxl_mem_get_fw_info(mds))
995 return FW_UPLOAD_ERR_HW_ERROR;
996
997 /*
998 * So far no state has been changed, hence no other cleanup is
999 * necessary. Simply return the cancelled status.
1000 */
1001 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
1002 return FW_UPLOAD_ERR_CANCELED;
1003
1004 return FW_UPLOAD_ERR_NONE;
1005}
1006
1007static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data,
1008 u32 offset, u32 size, u32 *written)
1009{
1010 struct cxl_memdev_state *mds = fwl->dd_handle;
1011 struct cxl_dev_state *cxlds = &mds->cxlds;
1012 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
1013 struct cxl_memdev *cxlmd = cxlds->cxlmd;
1014 struct cxl_mbox_transfer_fw *transfer;
1015 struct cxl_mbox_cmd mbox_cmd;
1016 u32 cur_size, remaining;
1017 size_t size_in;
1018 int rc;
1019
1020 *written = 0;
1021
1022 /* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */
1023 if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) {
1024 dev_err(&cxlmd->dev,
1025 "misaligned offset for FW transfer slice (%u)\n",
1026 offset);
1027 return FW_UPLOAD_ERR_RW_ERROR;
1028 }
1029
1030 /*
1031 * Pick transfer size based on mds->payload_size @size must bw 128-byte
1032 * aligned, ->payload_size is a power of 2 starting at 256 bytes, and
1033 * sizeof(*transfer) is 128. These constraints imply that @cur_size
1034 * will always be 128b aligned.
1035 */
1036 cur_size = min_t(size_t, size, cxl_mbox->payload_size - sizeof(*transfer));
1037
1038 remaining = size - cur_size;
1039 size_in = struct_size(transfer, data, cur_size);
1040
1041 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
1042 return cxl_fw_do_cancel(fwl);
1043
1044 /*
1045 * Slot numbers are 1-indexed
1046 * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1')
1047 * Check for rollover using modulo, and 1-index it by adding 1
1048 */
1049 mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1;
1050
1051 /* Do the transfer via mailbox cmd */
1052 transfer = kzalloc(size_in, GFP_KERNEL);
1053 if (!transfer)
1054 return FW_UPLOAD_ERR_RW_ERROR;
1055
1056 transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT);
1057 memcpy(transfer->data, data + offset, cur_size);
1058 if (mds->fw.oneshot) {
1059 transfer->action = CXL_FW_TRANSFER_ACTION_FULL;
1060 transfer->slot = mds->fw.next_slot;
1061 } else {
1062 if (offset == 0) {
1063 transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE;
1064 } else if (remaining == 0) {
1065 transfer->action = CXL_FW_TRANSFER_ACTION_END;
1066 transfer->slot = mds->fw.next_slot;
1067 } else {
1068 transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE;
1069 }
1070 }
1071
1072 mbox_cmd = (struct cxl_mbox_cmd) {
1073 .opcode = CXL_MBOX_OP_TRANSFER_FW,
1074 .size_in = size_in,
1075 .payload_in = transfer,
1076 .poll_interval_ms = 1000,
1077 .poll_count = 30,
1078 };
1079
1080 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
1081 if (rc < 0) {
1082 rc = FW_UPLOAD_ERR_RW_ERROR;
1083 goto out_free;
1084 }
1085
1086 *written = cur_size;
1087
1088 /* Activate FW if oneshot or if the last slice was written */
1089 if (mds->fw.oneshot || remaining == 0) {
1090 dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n",
1091 mds->fw.next_slot);
1092 rc = cxl_mem_activate_fw(mds, mds->fw.next_slot);
1093 if (rc < 0) {
1094 dev_err(&cxlmd->dev, "Error activating firmware: %d\n",
1095 rc);
1096 rc = FW_UPLOAD_ERR_HW_ERROR;
1097 goto out_free;
1098 }
1099 }
1100
1101 rc = FW_UPLOAD_ERR_NONE;
1102
1103out_free:
1104 kfree(transfer);
1105 return rc;
1106}
1107
1108static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl)
1109{
1110 struct cxl_memdev_state *mds = fwl->dd_handle;
1111
1112 /*
1113 * cxl_internal_send_cmd() handles background operations synchronously.
1114 * No need to wait for completions here - any errors would've been
1115 * reported and handled during the ->write() call(s).
1116 * Just check if a cancel request was received, and return success.
1117 */
1118 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
1119 return cxl_fw_do_cancel(fwl);
1120
1121 return FW_UPLOAD_ERR_NONE;
1122}
1123
1124static void cxl_fw_cancel(struct fw_upload *fwl)
1125{
1126 struct cxl_memdev_state *mds = fwl->dd_handle;
1127
1128 set_bit(CXL_FW_CANCEL, mds->fw.state);
1129}
1130
1131static const struct fw_upload_ops cxl_memdev_fw_ops = {
1132 .prepare = cxl_fw_prepare,
1133 .write = cxl_fw_write,
1134 .poll_complete = cxl_fw_poll_complete,
1135 .cancel = cxl_fw_cancel,
1136 .cleanup = cxl_fw_cleanup,
1137};
1138
1139static void cxl_remove_fw_upload(void *fwl)
1140{
1141 firmware_upload_unregister(fwl);
1142}
1143
1144int devm_cxl_setup_fw_upload(struct device *host, struct cxl_memdev_state *mds)
1145{
1146 struct cxl_dev_state *cxlds = &mds->cxlds;
1147 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
1148 struct device *dev = &cxlds->cxlmd->dev;
1149 struct fw_upload *fwl;
1150
1151 if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, cxl_mbox->enabled_cmds))
1152 return 0;
1153
1154 fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev),
1155 &cxl_memdev_fw_ops, mds);
1156 if (IS_ERR(fwl))
1157 return PTR_ERR(fwl);
1158 return devm_add_action_or_reset(host, cxl_remove_fw_upload, fwl);
1159}
1160EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_fw_upload, "CXL");
1161
1162static const struct file_operations cxl_memdev_fops = {
1163 .owner = THIS_MODULE,
1164 .unlocked_ioctl = cxl_memdev_ioctl,
1165 .open = cxl_memdev_open,
1166 .release = cxl_memdev_release_file,
1167 .compat_ioctl = compat_ptr_ioctl,
1168 .llseek = noop_llseek,
1169};
1170
1171/*
1172 * Activate ioctl operations, no cxl_memdev_rwsem manipulation needed as this is
1173 * ordered with cdev_add() publishing the device.
1174 */
1175static int cxlmd_add(struct cxl_memdev *cxlmd, struct cxl_dev_state *cxlds)
1176{
1177 int rc;
1178
1179 cxlmd->cxlds = cxlds;
1180 cxlds->cxlmd = cxlmd;
1181
1182 rc = cdev_device_add(&cxlmd->cdev, &cxlmd->dev);
1183 if (rc) {
1184 /*
1185 * The cdev was briefly live, shutdown any ioctl operations that
1186 * saw that state.
1187 */
1188 cxl_memdev_shutdown(&cxlmd->dev);
1189 return rc;
1190 }
1191
1192 return 0;
1193}
1194
1195DEFINE_FREE(put_cxlmd, struct cxl_memdev *,
1196 if (!IS_ERR_OR_NULL(_T)) put_device(&_T->dev))
1197
1198static bool cxl_memdev_attach_failed(struct cxl_memdev *cxlmd)
1199{
1200 /*
1201 * If @attach is provided fail if the driver is not attached upon
1202 * return. Note that failure here could be the result of a race to
1203 * teardown the CXL port topology. I.e. cxl_mem_probe() could have
1204 * succeeded and then cxl_mem unbound before the lock is acquired.
1205 */
1206 guard(device)(&cxlmd->dev);
1207 return (cxlmd->attach && !cxlmd->dev.driver);
1208}
1209
1210static struct cxl_memdev *cxl_memdev_autoremove(struct cxl_memdev *cxlmd)
1211{
1212 int rc;
1213
1214 if (cxl_memdev_attach_failed(cxlmd)) {
1215 cxl_memdev_unregister(cxlmd);
1216 return ERR_PTR(-ENXIO);
1217 }
1218
1219 rc = devm_add_action_or_reset(cxlmd->cxlds->dev, cxl_memdev_unregister,
1220 cxlmd);
1221 if (rc)
1222 return ERR_PTR(rc);
1223
1224 return cxlmd;
1225}
1226
1227/*
1228 * Core helper for devm_cxl_add_memdev() that wants to both create a device and
1229 * assert to the caller that upon return cxl_mem::probe() has been invoked.
1230 */
1231struct cxl_memdev *__devm_cxl_add_memdev(struct cxl_dev_state *cxlds,
1232 const struct cxl_memdev_attach *attach)
1233{
1234 struct device *dev;
1235 int rc;
1236
1237 struct cxl_memdev *cxlmd __free(put_cxlmd) =
1238 cxl_memdev_alloc(cxlds, &cxl_memdev_fops, attach);
1239 if (IS_ERR(cxlmd))
1240 return cxlmd;
1241
1242 dev = &cxlmd->dev;
1243 rc = dev_set_name(dev, "mem%d", cxlmd->id);
1244 if (rc)
1245 return ERR_PTR(rc);
1246
1247 rc = cxlmd_add(cxlmd, cxlds);
1248 if (rc)
1249 return ERR_PTR(rc);
1250
1251 return cxl_memdev_autoremove(no_free_ptr(cxlmd));
1252}
1253EXPORT_SYMBOL_FOR_MODULES(__devm_cxl_add_memdev, "cxl_mem");
1254
1255static void sanitize_teardown_notifier(void *data)
1256{
1257 struct cxl_memdev_state *mds = data;
1258 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
1259 struct kernfs_node *state;
1260
1261 /*
1262 * Prevent new irq triggered invocations of the workqueue and
1263 * flush inflight invocations.
1264 */
1265 mutex_lock(&cxl_mbox->mbox_mutex);
1266 state = mds->security.sanitize_node;
1267 mds->security.sanitize_node = NULL;
1268 mutex_unlock(&cxl_mbox->mbox_mutex);
1269
1270 cancel_delayed_work_sync(&mds->security.poll_dwork);
1271 sysfs_put(state);
1272}
1273
1274int devm_cxl_sanitize_setup_notifier(struct device *host,
1275 struct cxl_memdev *cxlmd)
1276{
1277 struct cxl_dev_state *cxlds = cxlmd->cxlds;
1278 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
1279 struct kernfs_node *sec;
1280
1281 if (!test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))
1282 return 0;
1283
1284 /*
1285 * Note, the expectation is that @cxlmd would have failed to be
1286 * created if these sysfs_get_dirent calls fail.
1287 */
1288 sec = sysfs_get_dirent(cxlmd->dev.kobj.sd, "security");
1289 if (!sec)
1290 return -ENOENT;
1291 mds->security.sanitize_node = sysfs_get_dirent(sec, "state");
1292 sysfs_put(sec);
1293 if (!mds->security.sanitize_node)
1294 return -ENOENT;
1295
1296 return devm_add_action_or_reset(host, sanitize_teardown_notifier, mds);
1297}
1298EXPORT_SYMBOL_NS_GPL(devm_cxl_sanitize_setup_notifier, "CXL");
1299
1300__init int cxl_memdev_init(void)
1301{
1302 dev_t devt;
1303 int rc;
1304
1305 rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl");
1306 if (rc)
1307 return rc;
1308
1309 cxl_mem_major = MAJOR(devt);
1310
1311 return 0;
1312}
1313
1314void cxl_memdev_exit(void)
1315{
1316 unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS);
1317}