| 1 | |
| 2 | |
| 3 | * Copyright (c) 2020 Synopsys, Inc. and/or its affiliates.
|
| 4 | * Synopsys DesignWare xData driver
|
| 5 | *
|
| 6 | * Author: Gustavo Pimentel <[email protected]>
|
| 7 |
|
| 8 |
|
| 9 | #include <linux/miscdevice.h>
|
| 10 | #include <linux/bitfield.h>
|
| 11 | #include <linux/pci-epf.h>
|
| 12 | #include <linux/kernel.h>
|
| 13 | #include <linux/module.h>
|
| 14 | #include <linux/device.h>
|
| 15 | #include <linux/bitops.h>
|
| 16 | #include <linux/mutex.h>
|
| 17 | #include <linux/delay.h>
|
| 18 | #include <linux/pci.h>
|
| 19 | #include <linux/string_choices.h>
|
| 20 |
|
| 21 | #define DW_XDATA_DRIVER_NAME "dw-xdata-pcie"
|
| 22 |
|
| 23 | #define DW_XDATA_EP_MEM_OFFSET 0x8000000
|
| 24 |
|
| 25 | static DEFINE_IDA(xdata_ida);
|
| 26 |
|
| 27 | #define STATUS_DONE BIT(0)
|
| 28 |
|
| 29 | #define CONTROL_DOORBELL BIT(0)
|
| 30 | #define CONTROL_IS_WRITE BIT(1)
|
| 31 | #define CONTROL_LENGTH(a) FIELD_PREP(GENMASK(13, 2), a)
|
| 32 | #define CONTROL_PATTERN_INC BIT(16)
|
| 33 | #define CONTROL_NO_ADDR_INC BIT(18)
|
| 34 |
|
| 35 | #define XPERF_CONTROL_ENABLE BIT(5)
|
| 36 |
|
| 37 | #define BURST_REPEAT BIT(31)
|
| 38 | #define BURST_VALUE 0x1001
|
| 39 |
|
| 40 | #define PATTERN_VALUE 0x0
|
| 41 |
|
| 42 | struct dw_xdata_regs {
|
| 43 | u32 addr_lsb;
|
| 44 | u32 addr_msb;
|
| 45 | u32 burst_cnt;
|
| 46 | u32 control;
|
| 47 | u32 pattern;
|
| 48 | u32 status;
|
| 49 | u32 RAM_addr;
|
| 50 | u32 RAM_port;
|
| 51 | u32 _reserved0[14];
|
| 52 | u32 perf_control;
|
| 53 | u32 _reserved1[41];
|
| 54 | u32 wr_cnt_lsb;
|
| 55 | u32 wr_cnt_msb;
|
| 56 | u32 rd_cnt_lsb;
|
| 57 | u32 rd_cnt_msb;
|
| 58 | } __packed;
|
| 59 |
|
| 60 | struct dw_xdata_region {
|
| 61 | phys_addr_t paddr;
|
| 62 | void __iomem *vaddr;
|
| 63 | };
|
| 64 |
|
| 65 | struct dw_xdata {
|
| 66 | struct dw_xdata_region rg_region;
|
| 67 | size_t max_wr_len;
|
| 68 | size_t max_rd_len;
|
| 69 | struct mutex mutex;
|
| 70 | struct pci_dev *pdev;
|
| 71 | struct miscdevice misc_dev;
|
| 72 | };
|
| 73 |
|
| 74 | static inline struct dw_xdata_regs __iomem *__dw_regs(struct dw_xdata *dw)
|
| 75 | {
|
| 76 | return dw->rg_region.vaddr;
|
| 77 | }
|
| 78 |
|
| 79 | static void dw_xdata_stop(struct dw_xdata *dw)
|
| 80 | {
|
| 81 | u32 burst;
|
| 82 |
|
| 83 | mutex_lock(&dw->mutex);
|
| 84 |
|
| 85 | burst = readl(&(__dw_regs(dw)->burst_cnt));
|
| 86 |
|
| 87 | if (burst & BURST_REPEAT) {
|
| 88 | burst &= ~(u32)BURST_REPEAT;
|
| 89 | writel(burst, &(__dw_regs(dw)->burst_cnt));
|
| 90 | }
|
| 91 |
|
| 92 | mutex_unlock(&dw->mutex);
|
| 93 | }
|
| 94 |
|
| 95 | static void dw_xdata_start(struct dw_xdata *dw, bool write)
|
| 96 | {
|
| 97 | struct device *dev = &dw->pdev->dev;
|
| 98 | u32 control, status;
|
| 99 |
|
| 100 |
|
| 101 | dw_xdata_stop(dw);
|
| 102 |
|
| 103 | mutex_lock(&dw->mutex);
|
| 104 |
|
| 105 |
|
| 106 | writel(0x0, &(__dw_regs(dw)->status));
|
| 107 |
|
| 108 |
|
| 109 | writel(BURST_REPEAT | BURST_VALUE, &(__dw_regs(dw)->burst_cnt));
|
| 110 |
|
| 111 |
|
| 112 | writel(PATTERN_VALUE, &(__dw_regs(dw)->pattern));
|
| 113 |
|
| 114 |
|
| 115 | control = CONTROL_DOORBELL | CONTROL_PATTERN_INC | CONTROL_NO_ADDR_INC;
|
| 116 | if (write) {
|
| 117 | control |= CONTROL_IS_WRITE;
|
| 118 | control |= CONTROL_LENGTH(dw->max_wr_len);
|
| 119 | } else {
|
| 120 | control |= CONTROL_LENGTH(dw->max_rd_len);
|
| 121 | }
|
| 122 | writel(control, &(__dw_regs(dw)->control));
|
| 123 |
|
| 124 | |
| 125 | * The xData HW block needs about 100 ms to initiate the traffic
|
| 126 | * generation according this HW block datasheet.
|
| 127 |
|
| 128 | usleep_range(100, 150);
|
| 129 |
|
| 130 | status = readl(&(__dw_regs(dw)->status));
|
| 131 |
|
| 132 | mutex_unlock(&dw->mutex);
|
| 133 |
|
| 134 | if (!(status & STATUS_DONE))
|
| 135 | dev_dbg(dev, "xData: started %s direction\n",
|
| 136 | str_write_read(write));
|
| 137 | }
|
| 138 |
|
| 139 | static void dw_xdata_perf_meas(struct dw_xdata *dw, u64 *data, bool write)
|
| 140 | {
|
| 141 | if (write) {
|
| 142 | *data = readl(&(__dw_regs(dw)->wr_cnt_msb));
|
| 143 | *data <<= 32;
|
| 144 | *data |= readl(&(__dw_regs(dw)->wr_cnt_lsb));
|
| 145 | } else {
|
| 146 | *data = readl(&(__dw_regs(dw)->rd_cnt_msb));
|
| 147 | *data <<= 32;
|
| 148 | *data |= readl(&(__dw_regs(dw)->rd_cnt_lsb));
|
| 149 | }
|
| 150 | }
|
| 151 |
|
| 152 | static u64 dw_xdata_perf_diff(u64 *m1, u64 *m2, u64 time)
|
| 153 | {
|
| 154 | u64 rate = (*m1 - *m2);
|
| 155 |
|
| 156 | rate *= (1000 * 1000 * 1000);
|
| 157 | rate >>= 20;
|
| 158 | rate = DIV_ROUND_CLOSEST_ULL(rate, time);
|
| 159 |
|
| 160 | return rate;
|
| 161 | }
|
| 162 |
|
| 163 | static void dw_xdata_perf(struct dw_xdata *dw, u64 *rate, bool write)
|
| 164 | {
|
| 165 | struct device *dev = &dw->pdev->dev;
|
| 166 | u64 data[2], time[2], diff;
|
| 167 |
|
| 168 | mutex_lock(&dw->mutex);
|
| 169 |
|
| 170 |
|
| 171 | writel(0x0, &(__dw_regs(dw)->perf_control));
|
| 172 | dw_xdata_perf_meas(dw, &data[0], write);
|
| 173 | time[0] = jiffies;
|
| 174 | writel((u32)XPERF_CONTROL_ENABLE, &(__dw_regs(dw)->perf_control));
|
| 175 |
|
| 176 | |
| 177 | * Wait 100ms between the 1st count frame acquisition and the 2nd
|
| 178 | * count frame acquisition, in order to calculate the speed later
|
| 179 |
|
| 180 | mdelay(100);
|
| 181 |
|
| 182 |
|
| 183 | writel(0x0, &(__dw_regs(dw)->perf_control));
|
| 184 | dw_xdata_perf_meas(dw, &data[1], write);
|
| 185 | time[1] = jiffies;
|
| 186 | writel((u32)XPERF_CONTROL_ENABLE, &(__dw_regs(dw)->perf_control));
|
| 187 |
|
| 188 | |
| 189 | * Speed calculation
|
| 190 | *
|
| 191 | * rate = (2nd count frames - 1st count frames) / (time elapsed)
|
| 192 |
|
| 193 | diff = jiffies_to_nsecs(time[1] - time[0]);
|
| 194 | *rate = dw_xdata_perf_diff(&data[1], &data[0], diff);
|
| 195 |
|
| 196 | mutex_unlock(&dw->mutex);
|
| 197 |
|
| 198 | dev_dbg(dev, "xData: time=%llu us, %s=%llu MB/s\n",
|
| 199 | diff, str_write_read(write), *rate);
|
| 200 | }
|
| 201 |
|
| 202 | static struct dw_xdata *misc_dev_to_dw(struct miscdevice *misc_dev)
|
| 203 | {
|
| 204 | return container_of(misc_dev, struct dw_xdata, misc_dev);
|
| 205 | }
|
| 206 |
|
| 207 | static ssize_t write_show(struct device *dev, struct device_attribute *attr,
|
| 208 | char *buf)
|
| 209 | {
|
| 210 | struct miscdevice *misc_dev = dev_get_drvdata(dev);
|
| 211 | struct dw_xdata *dw = misc_dev_to_dw(misc_dev);
|
| 212 | u64 rate;
|
| 213 |
|
| 214 | dw_xdata_perf(dw, &rate, true);
|
| 215 |
|
| 216 | return sysfs_emit(buf, "%llu\n", rate);
|
| 217 | }
|
| 218 |
|
| 219 | static ssize_t write_store(struct device *dev, struct device_attribute *attr,
|
| 220 | const char *buf, size_t size)
|
| 221 | {
|
| 222 | struct miscdevice *misc_dev = dev_get_drvdata(dev);
|
| 223 | struct dw_xdata *dw = misc_dev_to_dw(misc_dev);
|
| 224 | bool enabled;
|
| 225 | int ret;
|
| 226 |
|
| 227 | ret = kstrtobool(buf, &enabled);
|
| 228 | if (ret < 0)
|
| 229 | return ret;
|
| 230 |
|
| 231 | if (enabled) {
|
| 232 | dev_dbg(dev, "xData: requested write transfer\n");
|
| 233 | dw_xdata_start(dw, true);
|
| 234 | } else {
|
| 235 | dev_dbg(dev, "xData: requested stop transfer\n");
|
| 236 | dw_xdata_stop(dw);
|
| 237 | }
|
| 238 |
|
| 239 | return size;
|
| 240 | }
|
| 241 |
|
| 242 | static DEVICE_ATTR_RW(write);
|
| 243 |
|
| 244 | static ssize_t read_show(struct device *dev, struct device_attribute *attr,
|
| 245 | char *buf)
|
| 246 | {
|
| 247 | struct miscdevice *misc_dev = dev_get_drvdata(dev);
|
| 248 | struct dw_xdata *dw = misc_dev_to_dw(misc_dev);
|
| 249 | u64 rate;
|
| 250 |
|
| 251 | dw_xdata_perf(dw, &rate, false);
|
| 252 |
|
| 253 | return sysfs_emit(buf, "%llu\n", rate);
|
| 254 | }
|
| 255 |
|
| 256 | static ssize_t read_store(struct device *dev, struct device_attribute *attr,
|
| 257 | const char *buf, size_t size)
|
| 258 | {
|
| 259 | struct miscdevice *misc_dev = dev_get_drvdata(dev);
|
| 260 | struct dw_xdata *dw = misc_dev_to_dw(misc_dev);
|
| 261 | bool enabled;
|
| 262 | int ret;
|
| 263 |
|
| 264 | ret = kstrtobool(buf, &enabled);
|
| 265 | if (ret < 0)
|
| 266 | return ret;
|
| 267 |
|
| 268 | if (enabled) {
|
| 269 | dev_dbg(dev, "xData: requested read transfer\n");
|
| 270 | dw_xdata_start(dw, false);
|
| 271 | } else {
|
| 272 | dev_dbg(dev, "xData: requested stop transfer\n");
|
| 273 | dw_xdata_stop(dw);
|
| 274 | }
|
| 275 |
|
| 276 | return size;
|
| 277 | }
|
| 278 |
|
| 279 | static DEVICE_ATTR_RW(read);
|
| 280 |
|
| 281 | static struct attribute *xdata_attrs[] = {
|
| 282 | &dev_attr_write.attr,
|
| 283 | &dev_attr_read.attr,
|
| 284 | NULL,
|
| 285 | };
|
| 286 |
|
| 287 | ATTRIBUTE_GROUPS(xdata);
|
| 288 |
|
| 289 | static int dw_xdata_pcie_probe(struct pci_dev *pdev,
|
| 290 | const struct pci_device_id *pid)
|
| 291 | {
|
| 292 | struct device *dev = &pdev->dev;
|
| 293 | struct dw_xdata *dw;
|
| 294 | char name[24];
|
| 295 | u64 addr;
|
| 296 | int err;
|
| 297 | int id;
|
| 298 |
|
| 299 |
|
| 300 | err = pcim_enable_device(pdev);
|
| 301 | if (err) {
|
| 302 | dev_err(dev, "enabling device failed\n");
|
| 303 | return err;
|
| 304 | }
|
| 305 |
|
| 306 |
|
| 307 | err = pcim_iomap_regions(pdev, BIT(BAR_0), pci_name(pdev));
|
| 308 | if (err) {
|
| 309 | dev_err(dev, "xData BAR I/O remapping failed\n");
|
| 310 | return err;
|
| 311 | }
|
| 312 |
|
| 313 | pci_set_master(pdev);
|
| 314 |
|
| 315 |
|
| 316 | dw = devm_kzalloc(dev, sizeof(*dw), GFP_KERNEL);
|
| 317 | if (!dw)
|
| 318 | return -ENOMEM;
|
| 319 |
|
| 320 |
|
| 321 | mutex_init(&dw->mutex);
|
| 322 |
|
| 323 | dw->rg_region.vaddr = pcim_iomap_table(pdev)[BAR_0];
|
| 324 | if (!dw->rg_region.vaddr)
|
| 325 | return -ENOMEM;
|
| 326 |
|
| 327 | dw->rg_region.paddr = pdev->resource[BAR_0].start;
|
| 328 |
|
| 329 | dw->max_wr_len = pcie_get_mps(pdev);
|
| 330 | dw->max_wr_len >>= 2;
|
| 331 |
|
| 332 | dw->max_rd_len = pcie_get_readrq(pdev);
|
| 333 | dw->max_rd_len >>= 2;
|
| 334 |
|
| 335 | dw->pdev = pdev;
|
| 336 |
|
| 337 | id = ida_alloc(&xdata_ida, GFP_KERNEL);
|
| 338 | if (id < 0) {
|
| 339 | dev_err(dev, "xData: unable to get id\n");
|
| 340 | return id;
|
| 341 | }
|
| 342 |
|
| 343 | snprintf(name, sizeof(name), DW_XDATA_DRIVER_NAME ".%d", id);
|
| 344 | dw->misc_dev.name = kstrdup(name, GFP_KERNEL);
|
| 345 | if (!dw->misc_dev.name) {
|
| 346 | err = -ENOMEM;
|
| 347 | goto err_ida_remove;
|
| 348 | }
|
| 349 |
|
| 350 | dw->misc_dev.minor = MISC_DYNAMIC_MINOR;
|
| 351 | dw->misc_dev.parent = dev;
|
| 352 | dw->misc_dev.groups = xdata_groups;
|
| 353 |
|
| 354 | writel(0x0, &(__dw_regs(dw)->RAM_addr));
|
| 355 | writel(0x0, &(__dw_regs(dw)->RAM_port));
|
| 356 |
|
| 357 | addr = dw->rg_region.paddr + DW_XDATA_EP_MEM_OFFSET;
|
| 358 | writel(lower_32_bits(addr), &(__dw_regs(dw)->addr_lsb));
|
| 359 | writel(upper_32_bits(addr), &(__dw_regs(dw)->addr_msb));
|
| 360 | dev_dbg(dev, "xData: target address = 0x%.16llx\n", addr);
|
| 361 |
|
| 362 | dev_dbg(dev, "xData: wr_len = %zu, rd_len = %zu\n",
|
| 363 | dw->max_wr_len * 4, dw->max_rd_len * 4);
|
| 364 |
|
| 365 |
|
| 366 | pci_set_drvdata(pdev, dw);
|
| 367 |
|
| 368 |
|
| 369 | err = misc_register(&dw->misc_dev);
|
| 370 | if (err) {
|
| 371 | dev_err(dev, "xData: failed to register device\n");
|
| 372 | goto err_kfree_name;
|
| 373 | }
|
| 374 |
|
| 375 | return 0;
|
| 376 |
|
| 377 | err_kfree_name:
|
| 378 | kfree(dw->misc_dev.name);
|
| 379 |
|
| 380 | err_ida_remove:
|
| 381 | ida_free(&xdata_ida, id);
|
| 382 |
|
| 383 | return err;
|
| 384 | }
|
| 385 |
|
| 386 | static void dw_xdata_pcie_remove(struct pci_dev *pdev)
|
| 387 | {
|
| 388 | struct dw_xdata *dw = pci_get_drvdata(pdev);
|
| 389 | int id;
|
| 390 |
|
| 391 | if (sscanf(dw->misc_dev.name, DW_XDATA_DRIVER_NAME ".%d", &id) != 1)
|
| 392 | return;
|
| 393 |
|
| 394 | if (id < 0)
|
| 395 | return;
|
| 396 |
|
| 397 | dw_xdata_stop(dw);
|
| 398 | misc_deregister(&dw->misc_dev);
|
| 399 | kfree(dw->misc_dev.name);
|
| 400 | ida_free(&xdata_ida, id);
|
| 401 | }
|
| 402 |
|
| 403 | static const struct pci_device_id dw_xdata_pcie_id_table[] = {
|
| 404 | { PCI_DEVICE_DATA(SYNOPSYS, EDDA, NULL) },
|
| 405 | { }
|
| 406 | };
|
| 407 | MODULE_DEVICE_TABLE(pci, dw_xdata_pcie_id_table);
|
| 408 |
|
| 409 | static struct pci_driver dw_xdata_pcie_driver = {
|
| 410 | .name = DW_XDATA_DRIVER_NAME,
|
| 411 | .id_table = dw_xdata_pcie_id_table,
|
| 412 | .probe = dw_xdata_pcie_probe,
|
| 413 | .remove = dw_xdata_pcie_remove,
|
| 414 | };
|
| 415 |
|
| 416 | module_pci_driver(dw_xdata_pcie_driver);
|
| 417 |
|
| 418 | MODULE_LICENSE("GPL v2");
|
| 419 | MODULE_DESCRIPTION("Synopsys DesignWare xData PCIe driver");
|
| 420 | MODULE_AUTHOR("Gustavo Pimentel <[email protected]>");
|
| 421 |
|