| 1 | |
| 2 | |
| 3 | * Data Object Exchange
|
| 4 | * PCIe r6.0, sec 6.30 DOE
|
| 5 | *
|
| 6 | * Copyright (C) 2021 Huawei
|
| 7 | * Jonathan Cameron <[email protected]>
|
| 8 | *
|
| 9 | * Copyright (C) 2022 Intel Corporation
|
| 10 | * Ira Weiny <[email protected]>
|
| 11 |
|
| 12 |
|
| 13 | #define dev_fmt(fmt) "DOE: " fmt
|
| 14 |
|
| 15 | #include <linux/bitfield.h>
|
| 16 | #include <linux/delay.h>
|
| 17 | #include <linux/device.h>
|
| 18 | #include <linux/jiffies.h>
|
| 19 | #include <linux/mutex.h>
|
| 20 | #include <linux/pci.h>
|
| 21 | #include <linux/pci-doe.h>
|
| 22 | #include <linux/sysfs.h>
|
| 23 | #include <linux/workqueue.h>
|
| 24 |
|
| 25 | #include "pci.h"
|
| 26 |
|
| 27 |
|
| 28 | #define PCI_DOE_TIMEOUT HZ
|
| 29 | #define PCI_DOE_POLL_INTERVAL (PCI_DOE_TIMEOUT / 128)
|
| 30 |
|
| 31 | #define PCI_DOE_FLAG_CANCEL 0
|
| 32 | #define PCI_DOE_FLAG_DEAD 1
|
| 33 |
|
| 34 |
|
| 35 | #define PCI_DOE_MAX_LENGTH (1 << 18)
|
| 36 |
|
| 37 | |
| 38 | * struct pci_doe_mb - State for a single DOE mailbox
|
| 39 | *
|
| 40 | * This state is used to manage a single DOE mailbox capability. All fields
|
| 41 | * should be considered opaque to the consumers and the structure passed into
|
| 42 | * the helpers below after being created by pci_doe_create_mb().
|
| 43 | *
|
| 44 | * @pdev: PCI device this mailbox belongs to
|
| 45 | * @cap_offset: Capability offset
|
| 46 | * @feats: Array of features supported (encoded as long values)
|
| 47 | * @wq: Wait queue for work item
|
| 48 | * @work_queue: Queue of pci_doe_work items
|
| 49 | * @flags: Bit array of PCI_DOE_FLAG_* flags
|
| 50 | * @sysfs_attrs: Array of sysfs device attributes
|
| 51 |
|
| 52 | struct pci_doe_mb {
|
| 53 | struct pci_dev *pdev;
|
| 54 | u16 cap_offset;
|
| 55 | struct xarray feats;
|
| 56 |
|
| 57 | wait_queue_head_t wq;
|
| 58 | struct workqueue_struct *work_queue;
|
| 59 | unsigned long flags;
|
| 60 |
|
| 61 | #ifdef CONFIG_SYSFS
|
| 62 | struct device_attribute *sysfs_attrs;
|
| 63 | #endif
|
| 64 | };
|
| 65 |
|
| 66 | struct pci_doe_feature {
|
| 67 | u16 vid;
|
| 68 | u8 type;
|
| 69 | };
|
| 70 |
|
| 71 | |
| 72 | * struct pci_doe_task - represents a single query/response
|
| 73 | *
|
| 74 | * @feat: DOE Feature
|
| 75 | * @request_pl: The request payload
|
| 76 | * @request_pl_sz: Size of the request payload (bytes)
|
| 77 | * @response_pl: The response payload
|
| 78 | * @response_pl_sz: Size of the response payload (bytes)
|
| 79 | * @rv: Return value. Length of received response or error (bytes)
|
| 80 | * @complete: Called when task is complete
|
| 81 | * @private: Private data for the consumer
|
| 82 | * @work: Used internally by the mailbox
|
| 83 | * @doe_mb: Used internally by the mailbox
|
| 84 |
|
| 85 | struct pci_doe_task {
|
| 86 | struct pci_doe_feature feat;
|
| 87 | const __le32 *request_pl;
|
| 88 | size_t request_pl_sz;
|
| 89 | __le32 *response_pl;
|
| 90 | size_t response_pl_sz;
|
| 91 | int rv;
|
| 92 | void (*complete)(struct pci_doe_task *task);
|
| 93 | void *private;
|
| 94 |
|
| 95 |
|
| 96 | struct work_struct work;
|
| 97 | struct pci_doe_mb *doe_mb;
|
| 98 | };
|
| 99 |
|
| 100 | #ifdef CONFIG_SYSFS
|
| 101 | static ssize_t doe_discovery_show(struct device *dev,
|
| 102 | struct device_attribute *attr,
|
| 103 | char *buf)
|
| 104 | {
|
| 105 | return sysfs_emit(buf, "0001:00\n");
|
| 106 | }
|
| 107 | static DEVICE_ATTR_RO(doe_discovery);
|
| 108 |
|
| 109 | static struct attribute *pci_doe_sysfs_feature_attrs[] = {
|
| 110 | &dev_attr_doe_discovery.attr,
|
| 111 | NULL
|
| 112 | };
|
| 113 |
|
| 114 | static bool pci_doe_features_sysfs_group_visible(struct kobject *kobj)
|
| 115 | {
|
| 116 | struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
|
| 117 |
|
| 118 | return !xa_empty(&pdev->doe_mbs);
|
| 119 | }
|
| 120 | DEFINE_SIMPLE_SYSFS_GROUP_VISIBLE(pci_doe_features_sysfs)
|
| 121 |
|
| 122 | const struct attribute_group pci_doe_sysfs_group = {
|
| 123 | .name = "doe_features",
|
| 124 | .attrs = pci_doe_sysfs_feature_attrs,
|
| 125 | .is_visible = SYSFS_GROUP_VISIBLE(pci_doe_features_sysfs),
|
| 126 | };
|
| 127 |
|
| 128 | static ssize_t pci_doe_sysfs_feature_show(struct device *dev,
|
| 129 | struct device_attribute *attr,
|
| 130 | char *buf)
|
| 131 | {
|
| 132 | return sysfs_emit(buf, "%s\n", attr->attr.name);
|
| 133 | }
|
| 134 |
|
| 135 | static void pci_doe_sysfs_feature_remove(struct pci_dev *pdev,
|
| 136 | struct pci_doe_mb *doe_mb)
|
| 137 | {
|
| 138 | struct device_attribute *attrs = doe_mb->sysfs_attrs;
|
| 139 | struct device *dev = &pdev->dev;
|
| 140 | unsigned long i;
|
| 141 | void *entry;
|
| 142 |
|
| 143 | if (!attrs)
|
| 144 | return;
|
| 145 |
|
| 146 | doe_mb->sysfs_attrs = NULL;
|
| 147 | xa_for_each(&doe_mb->feats, i, entry) {
|
| 148 | if (attrs[i].show)
|
| 149 | sysfs_remove_file_from_group(&dev->kobj, &attrs[i].attr,
|
| 150 | pci_doe_sysfs_group.name);
|
| 151 | kfree(attrs[i].attr.name);
|
| 152 | }
|
| 153 | kfree(attrs);
|
| 154 | }
|
| 155 |
|
| 156 | static int pci_doe_sysfs_feature_populate(struct pci_dev *pdev,
|
| 157 | struct pci_doe_mb *doe_mb)
|
| 158 | {
|
| 159 | struct device *dev = &pdev->dev;
|
| 160 | struct device_attribute *attrs;
|
| 161 | unsigned long num_features = 0;
|
| 162 | unsigned long vid, type;
|
| 163 | unsigned long i;
|
| 164 | void *entry;
|
| 165 | int ret;
|
| 166 |
|
| 167 | xa_for_each(&doe_mb->feats, i, entry)
|
| 168 | num_features++;
|
| 169 |
|
| 170 | attrs = kzalloc_objs(*attrs, num_features);
|
| 171 | if (!attrs) {
|
| 172 | pci_warn(pdev, "Failed allocating the device_attribute array\n");
|
| 173 | return -ENOMEM;
|
| 174 | }
|
| 175 |
|
| 176 | doe_mb->sysfs_attrs = attrs;
|
| 177 | xa_for_each(&doe_mb->feats, i, entry) {
|
| 178 | sysfs_attr_init(&attrs[i].attr);
|
| 179 | vid = xa_to_value(entry) >> 8;
|
| 180 | type = xa_to_value(entry) & 0xFF;
|
| 181 |
|
| 182 | if (vid == PCI_VENDOR_ID_PCI_SIG &&
|
| 183 | type == PCI_DOE_FEATURE_DISCOVERY) {
|
| 184 |
|
| 185 | |
| 186 | * DOE Discovery, manually displayed by
|
| 187 | * `dev_attr_doe_discovery`
|
| 188 |
|
| 189 | continue;
|
| 190 | }
|
| 191 |
|
| 192 | attrs[i].attr.name = kasprintf(GFP_KERNEL,
|
| 193 | "%04lx:%02lx", vid, type);
|
| 194 | if (!attrs[i].attr.name) {
|
| 195 | ret = -ENOMEM;
|
| 196 | pci_warn(pdev, "Failed allocating the attribute name\n");
|
| 197 | goto fail;
|
| 198 | }
|
| 199 |
|
| 200 | attrs[i].attr.mode = 0444;
|
| 201 | attrs[i].show = pci_doe_sysfs_feature_show;
|
| 202 |
|
| 203 | ret = sysfs_add_file_to_group(&dev->kobj, &attrs[i].attr,
|
| 204 | pci_doe_sysfs_group.name);
|
| 205 | if (ret) {
|
| 206 | attrs[i].show = NULL;
|
| 207 | if (ret != -EEXIST) {
|
| 208 | pci_warn(pdev, "Failed adding %s to sysfs group\n",
|
| 209 | attrs[i].attr.name);
|
| 210 | goto fail;
|
| 211 | } else
|
| 212 | kfree(attrs[i].attr.name);
|
| 213 | }
|
| 214 | }
|
| 215 |
|
| 216 | return 0;
|
| 217 |
|
| 218 | fail:
|
| 219 | pci_doe_sysfs_feature_remove(pdev, doe_mb);
|
| 220 | return ret;
|
| 221 | }
|
| 222 |
|
| 223 | void pci_doe_sysfs_teardown(struct pci_dev *pdev)
|
| 224 | {
|
| 225 | struct pci_doe_mb *doe_mb;
|
| 226 | unsigned long index;
|
| 227 |
|
| 228 | xa_for_each(&pdev->doe_mbs, index, doe_mb)
|
| 229 | pci_doe_sysfs_feature_remove(pdev, doe_mb);
|
| 230 | }
|
| 231 |
|
| 232 | void pci_doe_sysfs_init(struct pci_dev *pdev)
|
| 233 | {
|
| 234 | struct pci_doe_mb *doe_mb;
|
| 235 | unsigned long index;
|
| 236 | int ret;
|
| 237 |
|
| 238 | xa_for_each(&pdev->doe_mbs, index, doe_mb) {
|
| 239 | ret = pci_doe_sysfs_feature_populate(pdev, doe_mb);
|
| 240 | if (ret)
|
| 241 | return;
|
| 242 | }
|
| 243 | }
|
| 244 | #endif
|
| 245 |
|
| 246 | static int pci_doe_wait(struct pci_doe_mb *doe_mb, unsigned long timeout)
|
| 247 | {
|
| 248 | if (wait_event_timeout(doe_mb->wq,
|
| 249 | test_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags),
|
| 250 | timeout))
|
| 251 | return -EIO;
|
| 252 | return 0;
|
| 253 | }
|
| 254 |
|
| 255 | static void pci_doe_write_ctrl(struct pci_doe_mb *doe_mb, u32 val)
|
| 256 | {
|
| 257 | struct pci_dev *pdev = doe_mb->pdev;
|
| 258 | int offset = doe_mb->cap_offset;
|
| 259 |
|
| 260 | pci_write_config_dword(pdev, offset + PCI_DOE_CTRL, val);
|
| 261 | }
|
| 262 |
|
| 263 | static int pci_doe_abort(struct pci_doe_mb *doe_mb)
|
| 264 | {
|
| 265 | struct pci_dev *pdev = doe_mb->pdev;
|
| 266 | int offset = doe_mb->cap_offset;
|
| 267 | unsigned long timeout_jiffies;
|
| 268 |
|
| 269 | pci_dbg(pdev, "[%x] Issuing Abort\n", offset);
|
| 270 |
|
| 271 | timeout_jiffies = jiffies + PCI_DOE_TIMEOUT;
|
| 272 | pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_ABORT);
|
| 273 |
|
| 274 | do {
|
| 275 | int rc;
|
| 276 | u32 val;
|
| 277 |
|
| 278 | rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
|
| 279 | if (rc)
|
| 280 | return rc;
|
| 281 | pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
|
| 282 |
|
| 283 |
|
| 284 | if (!FIELD_GET(PCI_DOE_STATUS_ERROR, val) &&
|
| 285 | !FIELD_GET(PCI_DOE_STATUS_BUSY, val))
|
| 286 | return 0;
|
| 287 |
|
| 288 | } while (!time_after(jiffies, timeout_jiffies));
|
| 289 |
|
| 290 |
|
| 291 | pci_err(pdev, "[%x] ABORT timed out\n", offset);
|
| 292 | return -EIO;
|
| 293 | }
|
| 294 |
|
| 295 | static int pci_doe_send_req(struct pci_doe_mb *doe_mb,
|
| 296 | struct pci_doe_task *task)
|
| 297 | {
|
| 298 | struct pci_dev *pdev = doe_mb->pdev;
|
| 299 | int offset = doe_mb->cap_offset;
|
| 300 | unsigned long timeout_jiffies;
|
| 301 | size_t length, remainder;
|
| 302 | u32 val;
|
| 303 | int i;
|
| 304 |
|
| 305 | |
| 306 | * Check the DOE busy bit is not set. If it is set, this could indicate
|
| 307 | * someone other than Linux (e.g. firmware) is using the mailbox. Note
|
| 308 | * it is expected that firmware and OS will negotiate access rights via
|
| 309 | * an, as yet to be defined, method.
|
| 310 | *
|
| 311 | * Wait up to one PCI_DOE_TIMEOUT period to allow the prior command to
|
| 312 | * finish. Otherwise, simply error out as unable to field the request.
|
| 313 | *
|
| 314 | * PCIe r6.2 sec 6.30.3 states no interrupt is raised when the DOE Busy
|
| 315 | * bit is cleared, so polling here is our best option for the moment.
|
| 316 |
|
| 317 | timeout_jiffies = jiffies + PCI_DOE_TIMEOUT;
|
| 318 | do {
|
| 319 | pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
|
| 320 | } while (FIELD_GET(PCI_DOE_STATUS_BUSY, val) &&
|
| 321 | !time_after(jiffies, timeout_jiffies));
|
| 322 |
|
| 323 | if (FIELD_GET(PCI_DOE_STATUS_BUSY, val))
|
| 324 | return -EBUSY;
|
| 325 |
|
| 326 | if (FIELD_GET(PCI_DOE_STATUS_ERROR, val))
|
| 327 | return -EIO;
|
| 328 |
|
| 329 |
|
| 330 | length = 2 + DIV_ROUND_UP(task->request_pl_sz, sizeof(__le32));
|
| 331 | if (length > PCI_DOE_MAX_LENGTH)
|
| 332 | return -EIO;
|
| 333 | if (length == PCI_DOE_MAX_LENGTH)
|
| 334 | length = 0;
|
| 335 |
|
| 336 |
|
| 337 | val = FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_VID, task->feat.vid) |
|
| 338 | FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, task->feat.type);
|
| 339 | pci_write_config_dword(pdev, offset + PCI_DOE_WRITE, val);
|
| 340 | pci_write_config_dword(pdev, offset + PCI_DOE_WRITE,
|
| 341 | FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH,
|
| 342 | length));
|
| 343 |
|
| 344 |
|
| 345 | for (i = 0; i < task->request_pl_sz / sizeof(__le32); i++)
|
| 346 | pci_write_config_dword(pdev, offset + PCI_DOE_WRITE,
|
| 347 | le32_to_cpu(task->request_pl[i]));
|
| 348 |
|
| 349 |
|
| 350 | remainder = task->request_pl_sz % sizeof(__le32);
|
| 351 | if (remainder) {
|
| 352 | val = 0;
|
| 353 | memcpy(&val, &task->request_pl[i], remainder);
|
| 354 | le32_to_cpus(&val);
|
| 355 | pci_write_config_dword(pdev, offset + PCI_DOE_WRITE, val);
|
| 356 | }
|
| 357 |
|
| 358 | pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_GO);
|
| 359 |
|
| 360 | return 0;
|
| 361 | }
|
| 362 |
|
| 363 | static bool pci_doe_data_obj_ready(struct pci_doe_mb *doe_mb)
|
| 364 | {
|
| 365 | struct pci_dev *pdev = doe_mb->pdev;
|
| 366 | int offset = doe_mb->cap_offset;
|
| 367 | u32 val;
|
| 368 |
|
| 369 | pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
|
| 370 | if (FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val))
|
| 371 | return true;
|
| 372 | return false;
|
| 373 | }
|
| 374 |
|
| 375 | static int pci_doe_recv_resp(struct pci_doe_mb *doe_mb, struct pci_doe_task *task)
|
| 376 | {
|
| 377 | size_t length, payload_length, remainder, received;
|
| 378 | struct pci_dev *pdev = doe_mb->pdev;
|
| 379 | int offset = doe_mb->cap_offset;
|
| 380 | int i = 0;
|
| 381 | u32 val;
|
| 382 |
|
| 383 |
|
| 384 | pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
|
| 385 | if ((FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val) != task->feat.vid) ||
|
| 386 | (FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val) != task->feat.type)) {
|
| 387 | dev_err_ratelimited(&pdev->dev, "[%x] expected [VID, Feature] = [%04x, %02x], got [%04x, %02x]\n",
|
| 388 | doe_mb->cap_offset, task->feat.vid, task->feat.type,
|
| 389 | FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val),
|
| 390 | FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val));
|
| 391 | return -EIO;
|
| 392 | }
|
| 393 |
|
| 394 | pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
|
| 395 |
|
| 396 | pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
|
| 397 | pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
|
| 398 |
|
| 399 | length = FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH, val);
|
| 400 |
|
| 401 | if (!length)
|
| 402 | length = PCI_DOE_MAX_LENGTH;
|
| 403 | if (length < 2)
|
| 404 | return -EIO;
|
| 405 |
|
| 406 |
|
| 407 | length -= 2;
|
| 408 | received = task->response_pl_sz;
|
| 409 | payload_length = DIV_ROUND_UP(task->response_pl_sz, sizeof(__le32));
|
| 410 | remainder = task->response_pl_sz % sizeof(__le32);
|
| 411 |
|
| 412 |
|
| 413 | if (!remainder)
|
| 414 | remainder = sizeof(__le32);
|
| 415 |
|
| 416 | if (length < payload_length) {
|
| 417 | received = length * sizeof(__le32);
|
| 418 | payload_length = length;
|
| 419 | remainder = sizeof(__le32);
|
| 420 | }
|
| 421 |
|
| 422 | if (payload_length) {
|
| 423 |
|
| 424 | for (; i < payload_length - 1; i++) {
|
| 425 | pci_read_config_dword(pdev, offset + PCI_DOE_READ,
|
| 426 | &val);
|
| 427 | task->response_pl[i] = cpu_to_le32(val);
|
| 428 | pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
|
| 429 | }
|
| 430 |
|
| 431 |
|
| 432 | pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
|
| 433 | cpu_to_le32s(&val);
|
| 434 | memcpy(&task->response_pl[i], &val, remainder);
|
| 435 |
|
| 436 | if (!pci_doe_data_obj_ready(doe_mb))
|
| 437 | return -EIO;
|
| 438 | pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
|
| 439 | i++;
|
| 440 | }
|
| 441 |
|
| 442 |
|
| 443 | for (; i < length; i++) {
|
| 444 | pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
|
| 445 | pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
|
| 446 | }
|
| 447 |
|
| 448 |
|
| 449 | pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
|
| 450 | if (FIELD_GET(PCI_DOE_STATUS_ERROR, val))
|
| 451 | return -EIO;
|
| 452 |
|
| 453 | return received;
|
| 454 | }
|
| 455 |
|
| 456 | static void signal_task_complete(struct pci_doe_task *task, int rv)
|
| 457 | {
|
| 458 | task->rv = rv;
|
| 459 | destroy_work_on_stack(&task->work);
|
| 460 | task->complete(task);
|
| 461 | }
|
| 462 |
|
| 463 | static void signal_task_abort(struct pci_doe_task *task, int rv)
|
| 464 | {
|
| 465 | struct pci_doe_mb *doe_mb = task->doe_mb;
|
| 466 | struct pci_dev *pdev = doe_mb->pdev;
|
| 467 |
|
| 468 | if (pci_doe_abort(doe_mb)) {
|
| 469 | |
| 470 | * If the device can't process an abort; set the mailbox dead
|
| 471 | * - no more submissions
|
| 472 |
|
| 473 | pci_err(pdev, "[%x] Abort failed marking mailbox dead\n",
|
| 474 | doe_mb->cap_offset);
|
| 475 | set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags);
|
| 476 | }
|
| 477 | signal_task_complete(task, rv);
|
| 478 | }
|
| 479 |
|
| 480 | static void doe_statemachine_work(struct work_struct *work)
|
| 481 | {
|
| 482 | struct pci_doe_task *task = container_of(work, struct pci_doe_task,
|
| 483 | work);
|
| 484 | struct pci_doe_mb *doe_mb = task->doe_mb;
|
| 485 | struct pci_dev *pdev = doe_mb->pdev;
|
| 486 | int offset = doe_mb->cap_offset;
|
| 487 | unsigned long timeout_jiffies;
|
| 488 | u32 val;
|
| 489 | int rc;
|
| 490 |
|
| 491 | if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags)) {
|
| 492 | signal_task_complete(task, -EIO);
|
| 493 | return;
|
| 494 | }
|
| 495 |
|
| 496 |
|
| 497 | rc = pci_doe_send_req(doe_mb, task);
|
| 498 | if (rc) {
|
| 499 | |
| 500 | * The specification does not provide any guidance on how to
|
| 501 | * resolve conflicting requests from other entities.
|
| 502 | * Furthermore, it is likely that busy will not be detected
|
| 503 | * most of the time. Flag any detection of status busy with an
|
| 504 | * error.
|
| 505 |
|
| 506 | if (rc == -EBUSY)
|
| 507 | dev_err_ratelimited(&pdev->dev, "[%x] busy detected; another entity is sending conflicting requests\n",
|
| 508 | offset);
|
| 509 | signal_task_abort(task, rc);
|
| 510 | return;
|
| 511 | }
|
| 512 |
|
| 513 | timeout_jiffies = jiffies + PCI_DOE_TIMEOUT;
|
| 514 |
|
| 515 | retry_resp:
|
| 516 | pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
|
| 517 | if (FIELD_GET(PCI_DOE_STATUS_ERROR, val)) {
|
| 518 | signal_task_abort(task, -EIO);
|
| 519 | return;
|
| 520 | }
|
| 521 |
|
| 522 | if (!FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val)) {
|
| 523 | if (time_after(jiffies, timeout_jiffies)) {
|
| 524 | signal_task_abort(task, -EIO);
|
| 525 | return;
|
| 526 | }
|
| 527 | rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
|
| 528 | if (rc) {
|
| 529 | signal_task_abort(task, rc);
|
| 530 | return;
|
| 531 | }
|
| 532 | goto retry_resp;
|
| 533 | }
|
| 534 |
|
| 535 | rc = pci_doe_recv_resp(doe_mb, task);
|
| 536 | if (rc < 0) {
|
| 537 | signal_task_abort(task, rc);
|
| 538 | return;
|
| 539 | }
|
| 540 |
|
| 541 | signal_task_complete(task, rc);
|
| 542 | }
|
| 543 |
|
| 544 | static void pci_doe_task_complete(struct pci_doe_task *task)
|
| 545 | {
|
| 546 | complete(task->private);
|
| 547 | }
|
| 548 |
|
| 549 | static int pci_doe_discovery(struct pci_doe_mb *doe_mb, u8 capver, u8 *index, u16 *vid,
|
| 550 | u8 *feature)
|
| 551 | {
|
| 552 | u32 request_pl = FIELD_PREP(PCI_DOE_DATA_OBJECT_DISC_REQ_3_INDEX,
|
| 553 | *index) |
|
| 554 | FIELD_PREP(PCI_DOE_DATA_OBJECT_DISC_REQ_3_VER,
|
| 555 | (capver >= 2) ? 2 : 0);
|
| 556 | __le32 request_pl_le = cpu_to_le32(request_pl);
|
| 557 | __le32 response_pl_le;
|
| 558 | u32 response_pl;
|
| 559 | int rc;
|
| 560 |
|
| 561 | rc = pci_doe(doe_mb, PCI_VENDOR_ID_PCI_SIG, PCI_DOE_FEATURE_DISCOVERY,
|
| 562 | &request_pl_le, sizeof(request_pl_le),
|
| 563 | &response_pl_le, sizeof(response_pl_le));
|
| 564 | if (rc < 0)
|
| 565 | return rc;
|
| 566 |
|
| 567 | if (rc != sizeof(response_pl_le))
|
| 568 | return -EIO;
|
| 569 |
|
| 570 | response_pl = le32_to_cpu(response_pl_le);
|
| 571 | *vid = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_VID, response_pl);
|
| 572 | *feature = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_TYPE,
|
| 573 | response_pl);
|
| 574 | *index = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_NEXT_INDEX,
|
| 575 | response_pl);
|
| 576 |
|
| 577 | return 0;
|
| 578 | }
|
| 579 |
|
| 580 | static void *pci_doe_xa_feat_entry(u16 vid, u8 type)
|
| 581 | {
|
| 582 | return xa_mk_value((vid << 8) | type);
|
| 583 | }
|
| 584 |
|
| 585 | static int pci_doe_cache_features(struct pci_doe_mb *doe_mb)
|
| 586 | {
|
| 587 | u8 index = 0;
|
| 588 | u8 xa_idx = 0;
|
| 589 | u32 hdr = 0;
|
| 590 |
|
| 591 | pci_read_config_dword(doe_mb->pdev, doe_mb->cap_offset, &hdr);
|
| 592 |
|
| 593 | do {
|
| 594 | int rc;
|
| 595 | u16 vid;
|
| 596 | u8 type;
|
| 597 |
|
| 598 | rc = pci_doe_discovery(doe_mb, PCI_EXT_CAP_VER(hdr), &index,
|
| 599 | &vid, &type);
|
| 600 | if (rc)
|
| 601 | return rc;
|
| 602 |
|
| 603 | pci_dbg(doe_mb->pdev,
|
| 604 | "[%x] Found feature %d vid: %x type: %x\n",
|
| 605 | doe_mb->cap_offset, xa_idx, vid, type);
|
| 606 |
|
| 607 | rc = xa_insert(&doe_mb->feats, xa_idx++,
|
| 608 | pci_doe_xa_feat_entry(vid, type), GFP_KERNEL);
|
| 609 | if (rc)
|
| 610 | return rc;
|
| 611 | } while (index);
|
| 612 |
|
| 613 | return 0;
|
| 614 | }
|
| 615 |
|
| 616 | static void pci_doe_cancel_tasks(struct pci_doe_mb *doe_mb)
|
| 617 | {
|
| 618 |
|
| 619 | set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags);
|
| 620 |
|
| 621 |
|
| 622 | set_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags);
|
| 623 | wake_up(&doe_mb->wq);
|
| 624 | }
|
| 625 |
|
| 626 | |
| 627 | * pci_doe_create_mb() - Create a DOE mailbox object
|
| 628 | *
|
| 629 | * @pdev: PCI device to create the DOE mailbox for
|
| 630 | * @cap_offset: Offset of the DOE mailbox
|
| 631 | *
|
| 632 | * Create a single mailbox object to manage the mailbox feature at the
|
| 633 | * cap_offset specified.
|
| 634 | *
|
| 635 | * RETURNS: created mailbox object on success
|
| 636 | * ERR_PTR(-errno) on failure
|
| 637 |
|
| 638 | static struct pci_doe_mb *pci_doe_create_mb(struct pci_dev *pdev,
|
| 639 | u16 cap_offset)
|
| 640 | {
|
| 641 | struct pci_doe_mb *doe_mb;
|
| 642 | int rc;
|
| 643 |
|
| 644 | doe_mb = kzalloc_obj(*doe_mb);
|
| 645 | if (!doe_mb)
|
| 646 | return ERR_PTR(-ENOMEM);
|
| 647 |
|
| 648 | doe_mb->pdev = pdev;
|
| 649 | doe_mb->cap_offset = cap_offset;
|
| 650 | init_waitqueue_head(&doe_mb->wq);
|
| 651 | xa_init(&doe_mb->feats);
|
| 652 |
|
| 653 | doe_mb->work_queue = alloc_ordered_workqueue("%s %s DOE [%x]", 0,
|
| 654 | dev_bus_name(&pdev->dev),
|
| 655 | pci_name(pdev),
|
| 656 | doe_mb->cap_offset);
|
| 657 | if (!doe_mb->work_queue) {
|
| 658 | pci_err(pdev, "[%x] failed to allocate work queue\n",
|
| 659 | doe_mb->cap_offset);
|
| 660 | rc = -ENOMEM;
|
| 661 | goto err_free;
|
| 662 | }
|
| 663 |
|
| 664 |
|
| 665 | rc = pci_doe_abort(doe_mb);
|
| 666 | if (rc) {
|
| 667 | pci_err(pdev, "[%x] failed to reset mailbox with abort command : %d\n",
|
| 668 | doe_mb->cap_offset, rc);
|
| 669 | goto err_destroy_wq;
|
| 670 | }
|
| 671 |
|
| 672 | |
| 673 | * The state machine and the mailbox should be in sync now;
|
| 674 | * Use the mailbox to query features.
|
| 675 |
|
| 676 | rc = pci_doe_cache_features(doe_mb);
|
| 677 | if (rc) {
|
| 678 | pci_err(pdev, "[%x] failed to cache features : %d\n",
|
| 679 | doe_mb->cap_offset, rc);
|
| 680 | goto err_cancel;
|
| 681 | }
|
| 682 |
|
| 683 | return doe_mb;
|
| 684 |
|
| 685 | err_cancel:
|
| 686 | pci_doe_cancel_tasks(doe_mb);
|
| 687 | xa_destroy(&doe_mb->feats);
|
| 688 | err_destroy_wq:
|
| 689 | destroy_workqueue(doe_mb->work_queue);
|
| 690 | err_free:
|
| 691 | kfree(doe_mb);
|
| 692 | return ERR_PTR(rc);
|
| 693 | }
|
| 694 |
|
| 695 | |
| 696 | * pci_doe_destroy_mb() - Destroy a DOE mailbox object
|
| 697 | *
|
| 698 | * @doe_mb: DOE mailbox
|
| 699 | *
|
| 700 | * Destroy all internal data structures created for the DOE mailbox.
|
| 701 |
|
| 702 | static void pci_doe_destroy_mb(struct pci_doe_mb *doe_mb)
|
| 703 | {
|
| 704 | pci_doe_cancel_tasks(doe_mb);
|
| 705 | xa_destroy(&doe_mb->feats);
|
| 706 | destroy_workqueue(doe_mb->work_queue);
|
| 707 | kfree(doe_mb);
|
| 708 | }
|
| 709 |
|
| 710 | |
| 711 | * pci_doe_supports_feat() - Return if the DOE instance supports the given
|
| 712 | * feature
|
| 713 | * @doe_mb: DOE mailbox capability to query
|
| 714 | * @vid: Feature Vendor ID
|
| 715 | * @type: Feature type
|
| 716 | *
|
| 717 | * RETURNS: True if the DOE mailbox supports the feature specified
|
| 718 |
|
| 719 | static bool pci_doe_supports_feat(struct pci_doe_mb *doe_mb, u16 vid, u8 type)
|
| 720 | {
|
| 721 | unsigned long index;
|
| 722 | void *entry;
|
| 723 |
|
| 724 |
|
| 725 | if (vid == PCI_VENDOR_ID_PCI_SIG && type == PCI_DOE_FEATURE_DISCOVERY)
|
| 726 | return true;
|
| 727 |
|
| 728 | xa_for_each(&doe_mb->feats, index, entry)
|
| 729 | if (entry == pci_doe_xa_feat_entry(vid, type))
|
| 730 | return true;
|
| 731 |
|
| 732 | return false;
|
| 733 | }
|
| 734 |
|
| 735 | |
| 736 | * pci_doe_submit_task() - Submit a task to be processed by the state machine
|
| 737 | *
|
| 738 | * @doe_mb: DOE mailbox capability to submit to
|
| 739 | * @task: task to be queued
|
| 740 | *
|
| 741 | * Submit a DOE task (request/response) to the DOE mailbox to be processed.
|
| 742 | * Returns upon queueing the task object. If the queue is full this function
|
| 743 | * will sleep until there is room in the queue.
|
| 744 | *
|
| 745 | * task->complete will be called when the state machine is done processing this
|
| 746 | * task.
|
| 747 | *
|
| 748 | * @task must be allocated on the stack.
|
| 749 | *
|
| 750 | * Excess data will be discarded.
|
| 751 | *
|
| 752 | * RETURNS: 0 when task has been successfully queued, -ERRNO on error
|
| 753 |
|
| 754 | static int pci_doe_submit_task(struct pci_doe_mb *doe_mb,
|
| 755 | struct pci_doe_task *task)
|
| 756 | {
|
| 757 | if (!pci_doe_supports_feat(doe_mb, task->feat.vid, task->feat.type))
|
| 758 | return -EINVAL;
|
| 759 |
|
| 760 | if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags))
|
| 761 | return -EIO;
|
| 762 |
|
| 763 | task->doe_mb = doe_mb;
|
| 764 | INIT_WORK_ONSTACK(&task->work, doe_statemachine_work);
|
| 765 | queue_work(doe_mb->work_queue, &task->work);
|
| 766 | return 0;
|
| 767 | }
|
| 768 |
|
| 769 | |
| 770 | * pci_doe() - Perform Data Object Exchange
|
| 771 | *
|
| 772 | * @doe_mb: DOE Mailbox
|
| 773 | * @vendor: Vendor ID
|
| 774 | * @type: Data Object Type
|
| 775 | * @request: Request payload
|
| 776 | * @request_sz: Size of request payload (bytes)
|
| 777 | * @response: Response payload
|
| 778 | * @response_sz: Size of response payload (bytes)
|
| 779 | *
|
| 780 | * Submit @request to @doe_mb and store the @response.
|
| 781 | * The DOE exchange is performed synchronously and may therefore sleep.
|
| 782 | *
|
| 783 | * Payloads are treated as opaque byte streams which are transmitted verbatim,
|
| 784 | * without byte-swapping. If payloads contain little-endian register values,
|
| 785 | * the caller is responsible for conversion with cpu_to_le32() / le32_to_cpu().
|
| 786 | *
|
| 787 | * For convenience, arbitrary payload sizes are allowed even though PCIe r6.0
|
| 788 | * sec 6.30.1 specifies the Data Object Header 2 "Length" in dwords. The last
|
| 789 | * (partial) dword is copied with byte granularity and padded with zeroes if
|
| 790 | * necessary. Callers are thus relieved of using dword-sized bounce buffers.
|
| 791 | *
|
| 792 | * RETURNS: Length of received response or negative errno.
|
| 793 | * Received data in excess of @response_sz is discarded.
|
| 794 | * The length may be smaller than @response_sz and the caller
|
| 795 | * is responsible for checking that.
|
| 796 |
|
| 797 | int pci_doe(struct pci_doe_mb *doe_mb, u16 vendor, u8 type,
|
| 798 | const void *request, size_t request_sz,
|
| 799 | void *response, size_t response_sz)
|
| 800 | {
|
| 801 | DECLARE_COMPLETION_ONSTACK(c);
|
| 802 | struct pci_doe_task task = {
|
| 803 | .feat.vid = vendor,
|
| 804 | .feat.type = type,
|
| 805 | .request_pl = request,
|
| 806 | .request_pl_sz = request_sz,
|
| 807 | .response_pl = response,
|
| 808 | .response_pl_sz = response_sz,
|
| 809 | .complete = pci_doe_task_complete,
|
| 810 | .private = &c,
|
| 811 | };
|
| 812 | int rc;
|
| 813 |
|
| 814 | rc = pci_doe_submit_task(doe_mb, &task);
|
| 815 | if (rc)
|
| 816 | return rc;
|
| 817 |
|
| 818 | wait_for_completion(&c);
|
| 819 |
|
| 820 | return task.rv;
|
| 821 | }
|
| 822 | EXPORT_SYMBOL_GPL(pci_doe);
|
| 823 |
|
| 824 | |
| 825 | * pci_find_doe_mailbox() - Find Data Object Exchange mailbox
|
| 826 | *
|
| 827 | * @pdev: PCI device
|
| 828 | * @vendor: Vendor ID
|
| 829 | * @type: Data Object Type
|
| 830 | *
|
| 831 | * Find first DOE mailbox of a PCI device which supports the given feature.
|
| 832 | *
|
| 833 | * RETURNS: Pointer to the DOE mailbox or NULL if none was found.
|
| 834 |
|
| 835 | struct pci_doe_mb *pci_find_doe_mailbox(struct pci_dev *pdev, u16 vendor,
|
| 836 | u8 type)
|
| 837 | {
|
| 838 | struct pci_doe_mb *doe_mb;
|
| 839 | unsigned long index;
|
| 840 |
|
| 841 | xa_for_each(&pdev->doe_mbs, index, doe_mb)
|
| 842 | if (pci_doe_supports_feat(doe_mb, vendor, type))
|
| 843 | return doe_mb;
|
| 844 |
|
| 845 | return NULL;
|
| 846 | }
|
| 847 | EXPORT_SYMBOL_GPL(pci_find_doe_mailbox);
|
| 848 |
|
| 849 | void pci_doe_init(struct pci_dev *pdev)
|
| 850 | {
|
| 851 | struct pci_doe_mb *doe_mb;
|
| 852 | u16 offset = 0;
|
| 853 | int rc;
|
| 854 |
|
| 855 | xa_init(&pdev->doe_mbs);
|
| 856 |
|
| 857 | while ((offset = pci_find_next_ext_capability(pdev, offset,
|
| 858 | PCI_EXT_CAP_ID_DOE))) {
|
| 859 | doe_mb = pci_doe_create_mb(pdev, offset);
|
| 860 | if (IS_ERR(doe_mb)) {
|
| 861 | pci_err(pdev, "[%x] failed to create mailbox: %pe\n",
|
| 862 | offset, doe_mb);
|
| 863 | continue;
|
| 864 | }
|
| 865 |
|
| 866 | rc = xa_insert(&pdev->doe_mbs, offset, doe_mb, GFP_KERNEL);
|
| 867 | if (rc) {
|
| 868 | pci_err(pdev, "[%x] failed to insert mailbox: %d\n",
|
| 869 | offset, rc);
|
| 870 | pci_doe_destroy_mb(doe_mb);
|
| 871 | }
|
| 872 | }
|
| 873 | }
|
| 874 |
|
| 875 | void pci_doe_destroy(struct pci_dev *pdev)
|
| 876 | {
|
| 877 | struct pci_doe_mb *doe_mb;
|
| 878 | unsigned long index;
|
| 879 |
|
| 880 | xa_for_each(&pdev->doe_mbs, index, doe_mb)
|
| 881 | pci_doe_destroy_mb(doe_mb);
|
| 882 |
|
| 883 | xa_destroy(&pdev->doe_mbs);
|
| 884 | }
|
| 885 |
|
| 886 | void pci_doe_disconnected(struct pci_dev *pdev)
|
| 887 | {
|
| 888 | struct pci_doe_mb *doe_mb;
|
| 889 | unsigned long index;
|
| 890 |
|
| 891 | xa_for_each(&pdev->doe_mbs, index, doe_mb)
|
| 892 | pci_doe_cancel_tasks(doe_mb);
|
| 893 | }
|