1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2020 Intel Corporation. */ 3 4 #include <linux/io-64-nonatomic-lo-hi.h> 5 #include <linux/firmware.h> 6 #include <linux/device.h> 7 #include <linux/slab.h> 8 #include <linux/idr.h> 9 #include <linux/pci.h> 10 #include <cxlmem.h> 11 #include "trace.h" 12 #include "core.h" 13 14 static DECLARE_RWSEM(cxl_memdev_rwsem); 15 16 /* 17 * An entire PCI topology full of devices should be enough for any 18 * config 19 */ 20 #define CXL_MEM_MAX_DEVS 65536 21 22 static int cxl_mem_major; 23 static DEFINE_IDA(cxl_memdev_ida); 24 25 static void cxl_memdev_release(struct device *dev) 26 { 27 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 28 29 ida_free(&cxl_memdev_ida, cxlmd->id); 30 devm_cxl_memdev_edac_release(cxlmd); 31 kfree(cxlmd); 32 } 33 34 static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid, 35 kgid_t *gid) 36 { 37 return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev)); 38 } 39 40 static ssize_t firmware_version_show(struct device *dev, 41 struct device_attribute *attr, char *buf) 42 { 43 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 44 struct cxl_dev_state *cxlds = cxlmd->cxlds; 45 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 46 47 if (!mds) 48 return sysfs_emit(buf, "\n"); 49 return sysfs_emit(buf, "%.16s\n", mds->firmware_version); 50 } 51 static DEVICE_ATTR_RO(firmware_version); 52 53 static ssize_t payload_max_show(struct device *dev, 54 struct device_attribute *attr, char *buf) 55 { 56 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 57 struct cxl_dev_state *cxlds = cxlmd->cxlds; 58 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 59 60 if (!mds) 61 return sysfs_emit(buf, "\n"); 62 return sysfs_emit(buf, "%zu\n", cxlds->cxl_mbox.payload_size); 63 } 64 static DEVICE_ATTR_RO(payload_max); 65 66 static ssize_t label_storage_size_show(struct device *dev, 67 struct device_attribute *attr, char *buf) 68 { 69 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 70 struct cxl_dev_state *cxlds = cxlmd->cxlds; 71 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 72 73 if (!mds) 74 return sysfs_emit(buf, "\n"); 75 return sysfs_emit(buf, "%zu\n", mds->lsa_size); 76 } 77 static DEVICE_ATTR_RO(label_storage_size); 78 79 static resource_size_t cxl_ram_size(struct cxl_dev_state *cxlds) 80 { 81 /* Static RAM is only expected at partition 0. */ 82 if (cxlds->part[0].mode != CXL_PARTMODE_RAM) 83 return 0; 84 return resource_size(&cxlds->part[0].res); 85 } 86 87 static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, 88 char *buf) 89 { 90 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 91 struct cxl_dev_state *cxlds = cxlmd->cxlds; 92 unsigned long long len = cxl_ram_size(cxlds); 93 94 return sysfs_emit(buf, "%#llx\n", len); 95 } 96 97 static struct device_attribute dev_attr_ram_size = 98 __ATTR(size, 0444, ram_size_show, NULL); 99 100 static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, 101 char *buf) 102 { 103 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 104 struct cxl_dev_state *cxlds = cxlmd->cxlds; 105 unsigned long long len = cxl_pmem_size(cxlds); 106 107 return sysfs_emit(buf, "%#llx\n", len); 108 } 109 110 static struct device_attribute dev_attr_pmem_size = 111 __ATTR(size, 0444, pmem_size_show, NULL); 112 113 static ssize_t serial_show(struct device *dev, struct device_attribute *attr, 114 char *buf) 115 { 116 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 117 struct cxl_dev_state *cxlds = cxlmd->cxlds; 118 119 return sysfs_emit(buf, "%#llx\n", cxlds->serial); 120 } 121 static DEVICE_ATTR_RO(serial); 122 123 static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr, 124 char *buf) 125 { 126 return sysfs_emit(buf, "%d\n", dev_to_node(dev)); 127 } 128 static DEVICE_ATTR_RO(numa_node); 129 130 static ssize_t security_state_show(struct device *dev, 131 struct device_attribute *attr, 132 char *buf) 133 { 134 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 135 struct cxl_dev_state *cxlds = cxlmd->cxlds; 136 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; 137 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 138 unsigned long state = mds->security.state; 139 int rc = 0; 140 141 /* sync with latest submission state */ 142 mutex_lock(&cxl_mbox->mbox_mutex); 143 if (mds->security.sanitize_active) 144 rc = sysfs_emit(buf, "sanitize\n"); 145 mutex_unlock(&cxl_mbox->mbox_mutex); 146 if (rc) 147 return rc; 148 149 if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET)) 150 return sysfs_emit(buf, "disabled\n"); 151 if (state & CXL_PMEM_SEC_STATE_FROZEN || 152 state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT || 153 state & CXL_PMEM_SEC_STATE_USER_PLIMIT) 154 return sysfs_emit(buf, "frozen\n"); 155 if (state & CXL_PMEM_SEC_STATE_LOCKED) 156 return sysfs_emit(buf, "locked\n"); 157 158 return sysfs_emit(buf, "unlocked\n"); 159 } 160 static struct device_attribute dev_attr_security_state = 161 __ATTR(state, 0444, security_state_show, NULL); 162 163 static ssize_t security_sanitize_store(struct device *dev, 164 struct device_attribute *attr, 165 const char *buf, size_t len) 166 { 167 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 168 bool sanitize; 169 ssize_t rc; 170 171 if (kstrtobool(buf, &sanitize) || !sanitize) 172 return -EINVAL; 173 174 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SANITIZE); 175 if (rc) 176 return rc; 177 178 return len; 179 } 180 static struct device_attribute dev_attr_security_sanitize = 181 __ATTR(sanitize, 0200, NULL, security_sanitize_store); 182 183 static ssize_t security_erase_store(struct device *dev, 184 struct device_attribute *attr, 185 const char *buf, size_t len) 186 { 187 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 188 ssize_t rc; 189 bool erase; 190 191 if (kstrtobool(buf, &erase) || !erase) 192 return -EINVAL; 193 194 rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SECURE_ERASE); 195 if (rc) 196 return rc; 197 198 return len; 199 } 200 static struct device_attribute dev_attr_security_erase = 201 __ATTR(erase, 0200, NULL, security_erase_store); 202 203 static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd) 204 { 205 struct cxl_dev_state *cxlds = cxlmd->cxlds; 206 u64 offset, length; 207 int rc = 0; 208 209 /* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */ 210 for (int i = 0; i < cxlds->nr_partitions; i++) { 211 const struct resource *res = &cxlds->part[i].res; 212 213 offset = res->start; 214 length = resource_size(res); 215 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 216 /* 217 * Invalid Physical Address is not an error for 218 * volatile addresses. Device support is optional. 219 */ 220 if (rc == -EFAULT && cxlds->part[i].mode == CXL_PARTMODE_RAM) 221 rc = 0; 222 } 223 return rc; 224 } 225 226 int cxl_trigger_poison_list(struct cxl_memdev *cxlmd) 227 { 228 struct cxl_port *port; 229 int rc; 230 231 port = cxlmd->endpoint; 232 if (!port || !is_cxl_endpoint(port)) 233 return -EINVAL; 234 235 rc = down_read_interruptible(&cxl_region_rwsem); 236 if (rc) 237 return rc; 238 239 rc = down_read_interruptible(&cxl_dpa_rwsem); 240 if (rc) { 241 up_read(&cxl_region_rwsem); 242 return rc; 243 } 244 245 if (cxl_num_decoders_committed(port) == 0) { 246 /* No regions mapped to this memdev */ 247 rc = cxl_get_poison_by_memdev(cxlmd); 248 } else { 249 /* Regions mapped, collect poison by endpoint */ 250 rc = cxl_get_poison_by_endpoint(port); 251 } 252 up_read(&cxl_dpa_rwsem); 253 up_read(&cxl_region_rwsem); 254 255 return rc; 256 } 257 EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, "CXL"); 258 259 static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa) 260 { 261 struct cxl_dev_state *cxlds = cxlmd->cxlds; 262 263 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 264 return 0; 265 266 if (!resource_size(&cxlds->dpa_res)) { 267 dev_dbg(cxlds->dev, "device has no dpa resource\n"); 268 return -EINVAL; 269 } 270 if (dpa < cxlds->dpa_res.start || dpa > cxlds->dpa_res.end) { 271 dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n", 272 dpa, &cxlds->dpa_res); 273 return -EINVAL; 274 } 275 if (!IS_ALIGNED(dpa, 64)) { 276 dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa); 277 return -EINVAL; 278 } 279 280 return 0; 281 } 282 283 int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa) 284 { 285 struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; 286 struct cxl_mbox_inject_poison inject; 287 struct cxl_poison_record record; 288 struct cxl_mbox_cmd mbox_cmd; 289 struct cxl_region *cxlr; 290 int rc; 291 292 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 293 return 0; 294 295 rc = down_read_interruptible(&cxl_region_rwsem); 296 if (rc) 297 return rc; 298 299 rc = down_read_interruptible(&cxl_dpa_rwsem); 300 if (rc) { 301 up_read(&cxl_region_rwsem); 302 return rc; 303 } 304 305 rc = cxl_validate_poison_dpa(cxlmd, dpa); 306 if (rc) 307 goto out; 308 309 inject.address = cpu_to_le64(dpa); 310 mbox_cmd = (struct cxl_mbox_cmd) { 311 .opcode = CXL_MBOX_OP_INJECT_POISON, 312 .size_in = sizeof(inject), 313 .payload_in = &inject, 314 }; 315 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 316 if (rc) 317 goto out; 318 319 cxlr = cxl_dpa_to_region(cxlmd, dpa); 320 if (cxlr) 321 dev_warn_once(cxl_mbox->host, 322 "poison inject dpa:%#llx region: %s\n", dpa, 323 dev_name(&cxlr->dev)); 324 325 record = (struct cxl_poison_record) { 326 .address = cpu_to_le64(dpa), 327 .length = cpu_to_le32(1), 328 }; 329 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT); 330 out: 331 up_read(&cxl_dpa_rwsem); 332 up_read(&cxl_region_rwsem); 333 334 return rc; 335 } 336 EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, "CXL"); 337 338 int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa) 339 { 340 struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; 341 struct cxl_mbox_clear_poison clear; 342 struct cxl_poison_record record; 343 struct cxl_mbox_cmd mbox_cmd; 344 struct cxl_region *cxlr; 345 int rc; 346 347 if (!IS_ENABLED(CONFIG_DEBUG_FS)) 348 return 0; 349 350 rc = down_read_interruptible(&cxl_region_rwsem); 351 if (rc) 352 return rc; 353 354 rc = down_read_interruptible(&cxl_dpa_rwsem); 355 if (rc) { 356 up_read(&cxl_region_rwsem); 357 return rc; 358 } 359 360 rc = cxl_validate_poison_dpa(cxlmd, dpa); 361 if (rc) 362 goto out; 363 364 /* 365 * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command 366 * is defined to accept 64 bytes of write-data, along with the 367 * address to clear. This driver uses zeroes as write-data. 368 */ 369 clear = (struct cxl_mbox_clear_poison) { 370 .address = cpu_to_le64(dpa) 371 }; 372 373 mbox_cmd = (struct cxl_mbox_cmd) { 374 .opcode = CXL_MBOX_OP_CLEAR_POISON, 375 .size_in = sizeof(clear), 376 .payload_in = &clear, 377 }; 378 379 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 380 if (rc) 381 goto out; 382 383 cxlr = cxl_dpa_to_region(cxlmd, dpa); 384 if (cxlr) 385 dev_warn_once(cxl_mbox->host, 386 "poison clear dpa:%#llx region: %s\n", dpa, 387 dev_name(&cxlr->dev)); 388 389 record = (struct cxl_poison_record) { 390 .address = cpu_to_le64(dpa), 391 .length = cpu_to_le32(1), 392 }; 393 trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR); 394 out: 395 up_read(&cxl_dpa_rwsem); 396 up_read(&cxl_region_rwsem); 397 398 return rc; 399 } 400 EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, "CXL"); 401 402 static struct attribute *cxl_memdev_attributes[] = { 403 &dev_attr_serial.attr, 404 &dev_attr_firmware_version.attr, 405 &dev_attr_payload_max.attr, 406 &dev_attr_label_storage_size.attr, 407 &dev_attr_numa_node.attr, 408 NULL, 409 }; 410 411 static struct cxl_dpa_perf *to_pmem_perf(struct cxl_dev_state *cxlds) 412 { 413 for (int i = 0; i < cxlds->nr_partitions; i++) 414 if (cxlds->part[i].mode == CXL_PARTMODE_PMEM) 415 return &cxlds->part[i].perf; 416 return NULL; 417 } 418 419 static ssize_t pmem_qos_class_show(struct device *dev, 420 struct device_attribute *attr, char *buf) 421 { 422 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 423 struct cxl_dev_state *cxlds = cxlmd->cxlds; 424 425 return sysfs_emit(buf, "%d\n", to_pmem_perf(cxlds)->qos_class); 426 } 427 428 static struct device_attribute dev_attr_pmem_qos_class = 429 __ATTR(qos_class, 0444, pmem_qos_class_show, NULL); 430 431 static struct attribute *cxl_memdev_pmem_attributes[] = { 432 &dev_attr_pmem_size.attr, 433 &dev_attr_pmem_qos_class.attr, 434 NULL, 435 }; 436 437 static struct cxl_dpa_perf *to_ram_perf(struct cxl_dev_state *cxlds) 438 { 439 if (cxlds->part[0].mode != CXL_PARTMODE_RAM) 440 return NULL; 441 return &cxlds->part[0].perf; 442 } 443 444 static ssize_t ram_qos_class_show(struct device *dev, 445 struct device_attribute *attr, char *buf) 446 { 447 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 448 struct cxl_dev_state *cxlds = cxlmd->cxlds; 449 450 return sysfs_emit(buf, "%d\n", to_ram_perf(cxlds)->qos_class); 451 } 452 453 static struct device_attribute dev_attr_ram_qos_class = 454 __ATTR(qos_class, 0444, ram_qos_class_show, NULL); 455 456 static struct attribute *cxl_memdev_ram_attributes[] = { 457 &dev_attr_ram_size.attr, 458 &dev_attr_ram_qos_class.attr, 459 NULL, 460 }; 461 462 static struct attribute *cxl_memdev_security_attributes[] = { 463 &dev_attr_security_state.attr, 464 &dev_attr_security_sanitize.attr, 465 &dev_attr_security_erase.attr, 466 NULL, 467 }; 468 469 static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a, 470 int n) 471 { 472 if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr) 473 return 0; 474 return a->mode; 475 } 476 477 static struct attribute_group cxl_memdev_attribute_group = { 478 .attrs = cxl_memdev_attributes, 479 .is_visible = cxl_memdev_visible, 480 }; 481 482 static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n) 483 { 484 struct device *dev = kobj_to_dev(kobj); 485 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 486 struct cxl_dpa_perf *perf = to_ram_perf(cxlmd->cxlds); 487 488 if (a == &dev_attr_ram_qos_class.attr && 489 (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID)) 490 return 0; 491 492 return a->mode; 493 } 494 495 static struct attribute_group cxl_memdev_ram_attribute_group = { 496 .name = "ram", 497 .attrs = cxl_memdev_ram_attributes, 498 .is_visible = cxl_ram_visible, 499 }; 500 501 static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n) 502 { 503 struct device *dev = kobj_to_dev(kobj); 504 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 505 struct cxl_dpa_perf *perf = to_pmem_perf(cxlmd->cxlds); 506 507 if (a == &dev_attr_pmem_qos_class.attr && 508 (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID)) 509 return 0; 510 511 return a->mode; 512 } 513 514 static struct attribute_group cxl_memdev_pmem_attribute_group = { 515 .name = "pmem", 516 .attrs = cxl_memdev_pmem_attributes, 517 .is_visible = cxl_pmem_visible, 518 }; 519 520 static umode_t cxl_memdev_security_visible(struct kobject *kobj, 521 struct attribute *a, int n) 522 { 523 struct device *dev = kobj_to_dev(kobj); 524 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 525 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 526 527 if (a == &dev_attr_security_sanitize.attr && 528 !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds)) 529 return 0; 530 531 if (a == &dev_attr_security_erase.attr && 532 !test_bit(CXL_SEC_ENABLED_SECURE_ERASE, mds->security.enabled_cmds)) 533 return 0; 534 535 return a->mode; 536 } 537 538 static struct attribute_group cxl_memdev_security_attribute_group = { 539 .name = "security", 540 .attrs = cxl_memdev_security_attributes, 541 .is_visible = cxl_memdev_security_visible, 542 }; 543 544 static const struct attribute_group *cxl_memdev_attribute_groups[] = { 545 &cxl_memdev_attribute_group, 546 &cxl_memdev_ram_attribute_group, 547 &cxl_memdev_pmem_attribute_group, 548 &cxl_memdev_security_attribute_group, 549 NULL, 550 }; 551 552 void cxl_memdev_update_perf(struct cxl_memdev *cxlmd) 553 { 554 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_ram_attribute_group); 555 sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_pmem_attribute_group); 556 } 557 EXPORT_SYMBOL_NS_GPL(cxl_memdev_update_perf, "CXL"); 558 559 static const struct device_type cxl_memdev_type = { 560 .name = "cxl_memdev", 561 .release = cxl_memdev_release, 562 .devnode = cxl_memdev_devnode, 563 .groups = cxl_memdev_attribute_groups, 564 }; 565 566 bool is_cxl_memdev(const struct device *dev) 567 { 568 return dev->type == &cxl_memdev_type; 569 } 570 EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, "CXL"); 571 572 /** 573 * set_exclusive_cxl_commands() - atomically disable user cxl commands 574 * @mds: The device state to operate on 575 * @cmds: bitmap of commands to mark exclusive 576 * 577 * Grab the cxl_memdev_rwsem in write mode to flush in-flight 578 * invocations of the ioctl path and then disable future execution of 579 * commands with the command ids set in @cmds. 580 */ 581 void set_exclusive_cxl_commands(struct cxl_memdev_state *mds, 582 unsigned long *cmds) 583 { 584 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 585 586 guard(rwsem_write)(&cxl_memdev_rwsem); 587 bitmap_or(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds, 588 cmds, CXL_MEM_COMMAND_ID_MAX); 589 } 590 EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, "CXL"); 591 592 /** 593 * clear_exclusive_cxl_commands() - atomically enable user cxl commands 594 * @mds: The device state to modify 595 * @cmds: bitmap of commands to mark available for userspace 596 */ 597 void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds, 598 unsigned long *cmds) 599 { 600 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 601 602 guard(rwsem_write)(&cxl_memdev_rwsem); 603 bitmap_andnot(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds, 604 cmds, CXL_MEM_COMMAND_ID_MAX); 605 } 606 EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, "CXL"); 607 608 static void cxl_memdev_shutdown(struct device *dev) 609 { 610 struct cxl_memdev *cxlmd = to_cxl_memdev(dev); 611 612 guard(rwsem_write)(&cxl_memdev_rwsem); 613 cxlmd->cxlds = NULL; 614 } 615 616 static void cxl_memdev_unregister(void *_cxlmd) 617 { 618 struct cxl_memdev *cxlmd = _cxlmd; 619 struct device *dev = &cxlmd->dev; 620 621 cdev_device_del(&cxlmd->cdev, dev); 622 cxl_memdev_shutdown(dev); 623 put_device(dev); 624 } 625 626 static void detach_memdev(struct work_struct *work) 627 { 628 struct cxl_memdev *cxlmd; 629 630 cxlmd = container_of(work, typeof(*cxlmd), detach_work); 631 device_release_driver(&cxlmd->dev); 632 put_device(&cxlmd->dev); 633 } 634 635 static struct lock_class_key cxl_memdev_key; 636 637 static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds, 638 const struct file_operations *fops) 639 { 640 struct cxl_memdev *cxlmd; 641 struct device *dev; 642 struct cdev *cdev; 643 int rc; 644 645 cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL); 646 if (!cxlmd) 647 return ERR_PTR(-ENOMEM); 648 649 rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL); 650 if (rc < 0) 651 goto err; 652 cxlmd->id = rc; 653 cxlmd->depth = -1; 654 655 dev = &cxlmd->dev; 656 device_initialize(dev); 657 lockdep_set_class(&dev->mutex, &cxl_memdev_key); 658 dev->parent = cxlds->dev; 659 dev->bus = &cxl_bus_type; 660 dev->devt = MKDEV(cxl_mem_major, cxlmd->id); 661 dev->type = &cxl_memdev_type; 662 device_set_pm_not_required(dev); 663 INIT_WORK(&cxlmd->detach_work, detach_memdev); 664 665 cdev = &cxlmd->cdev; 666 cdev_init(cdev, fops); 667 return cxlmd; 668 669 err: 670 kfree(cxlmd); 671 return ERR_PTR(rc); 672 } 673 674 static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd, 675 unsigned long arg) 676 { 677 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds); 678 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 679 680 switch (cmd) { 681 case CXL_MEM_QUERY_COMMANDS: 682 return cxl_query_cmd(cxl_mbox, (void __user *)arg); 683 case CXL_MEM_SEND_COMMAND: 684 return cxl_send_cmd(cxl_mbox, (void __user *)arg); 685 default: 686 return -ENOTTY; 687 } 688 } 689 690 static long cxl_memdev_ioctl(struct file *file, unsigned int cmd, 691 unsigned long arg) 692 { 693 struct cxl_memdev *cxlmd = file->private_data; 694 struct cxl_dev_state *cxlds; 695 696 guard(rwsem_read)(&cxl_memdev_rwsem); 697 cxlds = cxlmd->cxlds; 698 if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM) 699 return __cxl_memdev_ioctl(cxlmd, cmd, arg); 700 701 return -ENXIO; 702 } 703 704 static int cxl_memdev_open(struct inode *inode, struct file *file) 705 { 706 struct cxl_memdev *cxlmd = 707 container_of(inode->i_cdev, typeof(*cxlmd), cdev); 708 709 get_device(&cxlmd->dev); 710 file->private_data = cxlmd; 711 712 return 0; 713 } 714 715 static int cxl_memdev_release_file(struct inode *inode, struct file *file) 716 { 717 struct cxl_memdev *cxlmd = 718 container_of(inode->i_cdev, typeof(*cxlmd), cdev); 719 720 put_device(&cxlmd->dev); 721 722 return 0; 723 } 724 725 /** 726 * cxl_mem_get_fw_info - Get Firmware info 727 * @mds: The device data for the operation 728 * 729 * Retrieve firmware info for the device specified. 730 * 731 * Return: 0 if no error: or the result of the mailbox command. 732 * 733 * See CXL-3.0 8.2.9.3.1 Get FW Info 734 */ 735 static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds) 736 { 737 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 738 struct cxl_mbox_get_fw_info info; 739 struct cxl_mbox_cmd mbox_cmd; 740 int rc; 741 742 mbox_cmd = (struct cxl_mbox_cmd) { 743 .opcode = CXL_MBOX_OP_GET_FW_INFO, 744 .size_out = sizeof(info), 745 .payload_out = &info, 746 }; 747 748 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 749 if (rc < 0) 750 return rc; 751 752 mds->fw.num_slots = info.num_slots; 753 mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK, 754 info.slot_info); 755 756 return 0; 757 } 758 759 /** 760 * cxl_mem_activate_fw - Activate Firmware 761 * @mds: The device data for the operation 762 * @slot: slot number to activate 763 * 764 * Activate firmware in a given slot for the device specified. 765 * 766 * Return: 0 if no error: or the result of the mailbox command. 767 * 768 * See CXL-3.0 8.2.9.3.3 Activate FW 769 */ 770 static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot) 771 { 772 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 773 struct cxl_mbox_activate_fw activate; 774 struct cxl_mbox_cmd mbox_cmd; 775 776 if (slot == 0 || slot > mds->fw.num_slots) 777 return -EINVAL; 778 779 mbox_cmd = (struct cxl_mbox_cmd) { 780 .opcode = CXL_MBOX_OP_ACTIVATE_FW, 781 .size_in = sizeof(activate), 782 .payload_in = &activate, 783 }; 784 785 /* Only offline activation supported for now */ 786 activate.action = CXL_FW_ACTIVATE_OFFLINE; 787 activate.slot = slot; 788 789 return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 790 } 791 792 /** 793 * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer 794 * @mds: The device data for the operation 795 * 796 * Abort an in-progress firmware transfer for the device specified. 797 * 798 * Return: 0 if no error: or the result of the mailbox command. 799 * 800 * See CXL-3.0 8.2.9.3.2 Transfer FW 801 */ 802 static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds) 803 { 804 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 805 struct cxl_mbox_transfer_fw *transfer; 806 struct cxl_mbox_cmd mbox_cmd; 807 int rc; 808 809 transfer = kzalloc(struct_size(transfer, data, 0), GFP_KERNEL); 810 if (!transfer) 811 return -ENOMEM; 812 813 /* Set a 1s poll interval and a total wait time of 30s */ 814 mbox_cmd = (struct cxl_mbox_cmd) { 815 .opcode = CXL_MBOX_OP_TRANSFER_FW, 816 .size_in = sizeof(*transfer), 817 .payload_in = transfer, 818 .poll_interval_ms = 1000, 819 .poll_count = 30, 820 }; 821 822 transfer->action = CXL_FW_TRANSFER_ACTION_ABORT; 823 824 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 825 kfree(transfer); 826 return rc; 827 } 828 829 static void cxl_fw_cleanup(struct fw_upload *fwl) 830 { 831 struct cxl_memdev_state *mds = fwl->dd_handle; 832 833 mds->fw.next_slot = 0; 834 } 835 836 static int cxl_fw_do_cancel(struct fw_upload *fwl) 837 { 838 struct cxl_memdev_state *mds = fwl->dd_handle; 839 struct cxl_dev_state *cxlds = &mds->cxlds; 840 struct cxl_memdev *cxlmd = cxlds->cxlmd; 841 int rc; 842 843 rc = cxl_mem_abort_fw_xfer(mds); 844 if (rc < 0) 845 dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc); 846 847 return FW_UPLOAD_ERR_CANCELED; 848 } 849 850 static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data, 851 u32 size) 852 { 853 struct cxl_memdev_state *mds = fwl->dd_handle; 854 struct cxl_mbox_transfer_fw *transfer; 855 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 856 857 if (!size) 858 return FW_UPLOAD_ERR_INVALID_SIZE; 859 860 mds->fw.oneshot = struct_size(transfer, data, size) < 861 cxl_mbox->payload_size; 862 863 if (cxl_mem_get_fw_info(mds)) 864 return FW_UPLOAD_ERR_HW_ERROR; 865 866 /* 867 * So far no state has been changed, hence no other cleanup is 868 * necessary. Simply return the cancelled status. 869 */ 870 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 871 return FW_UPLOAD_ERR_CANCELED; 872 873 return FW_UPLOAD_ERR_NONE; 874 } 875 876 static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data, 877 u32 offset, u32 size, u32 *written) 878 { 879 struct cxl_memdev_state *mds = fwl->dd_handle; 880 struct cxl_dev_state *cxlds = &mds->cxlds; 881 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; 882 struct cxl_memdev *cxlmd = cxlds->cxlmd; 883 struct cxl_mbox_transfer_fw *transfer; 884 struct cxl_mbox_cmd mbox_cmd; 885 u32 cur_size, remaining; 886 size_t size_in; 887 int rc; 888 889 *written = 0; 890 891 /* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */ 892 if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) { 893 dev_err(&cxlmd->dev, 894 "misaligned offset for FW transfer slice (%u)\n", 895 offset); 896 return FW_UPLOAD_ERR_RW_ERROR; 897 } 898 899 /* 900 * Pick transfer size based on mds->payload_size @size must bw 128-byte 901 * aligned, ->payload_size is a power of 2 starting at 256 bytes, and 902 * sizeof(*transfer) is 128. These constraints imply that @cur_size 903 * will always be 128b aligned. 904 */ 905 cur_size = min_t(size_t, size, cxl_mbox->payload_size - sizeof(*transfer)); 906 907 remaining = size - cur_size; 908 size_in = struct_size(transfer, data, cur_size); 909 910 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 911 return cxl_fw_do_cancel(fwl); 912 913 /* 914 * Slot numbers are 1-indexed 915 * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1') 916 * Check for rollover using modulo, and 1-index it by adding 1 917 */ 918 mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1; 919 920 /* Do the transfer via mailbox cmd */ 921 transfer = kzalloc(size_in, GFP_KERNEL); 922 if (!transfer) 923 return FW_UPLOAD_ERR_RW_ERROR; 924 925 transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT); 926 memcpy(transfer->data, data + offset, cur_size); 927 if (mds->fw.oneshot) { 928 transfer->action = CXL_FW_TRANSFER_ACTION_FULL; 929 transfer->slot = mds->fw.next_slot; 930 } else { 931 if (offset == 0) { 932 transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE; 933 } else if (remaining == 0) { 934 transfer->action = CXL_FW_TRANSFER_ACTION_END; 935 transfer->slot = mds->fw.next_slot; 936 } else { 937 transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE; 938 } 939 } 940 941 mbox_cmd = (struct cxl_mbox_cmd) { 942 .opcode = CXL_MBOX_OP_TRANSFER_FW, 943 .size_in = size_in, 944 .payload_in = transfer, 945 .poll_interval_ms = 1000, 946 .poll_count = 30, 947 }; 948 949 rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); 950 if (rc < 0) { 951 rc = FW_UPLOAD_ERR_RW_ERROR; 952 goto out_free; 953 } 954 955 *written = cur_size; 956 957 /* Activate FW if oneshot or if the last slice was written */ 958 if (mds->fw.oneshot || remaining == 0) { 959 dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n", 960 mds->fw.next_slot); 961 rc = cxl_mem_activate_fw(mds, mds->fw.next_slot); 962 if (rc < 0) { 963 dev_err(&cxlmd->dev, "Error activating firmware: %d\n", 964 rc); 965 rc = FW_UPLOAD_ERR_HW_ERROR; 966 goto out_free; 967 } 968 } 969 970 rc = FW_UPLOAD_ERR_NONE; 971 972 out_free: 973 kfree(transfer); 974 return rc; 975 } 976 977 static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl) 978 { 979 struct cxl_memdev_state *mds = fwl->dd_handle; 980 981 /* 982 * cxl_internal_send_cmd() handles background operations synchronously. 983 * No need to wait for completions here - any errors would've been 984 * reported and handled during the ->write() call(s). 985 * Just check if a cancel request was received, and return success. 986 */ 987 if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state)) 988 return cxl_fw_do_cancel(fwl); 989 990 return FW_UPLOAD_ERR_NONE; 991 } 992 993 static void cxl_fw_cancel(struct fw_upload *fwl) 994 { 995 struct cxl_memdev_state *mds = fwl->dd_handle; 996 997 set_bit(CXL_FW_CANCEL, mds->fw.state); 998 } 999 1000 static const struct fw_upload_ops cxl_memdev_fw_ops = { 1001 .prepare = cxl_fw_prepare, 1002 .write = cxl_fw_write, 1003 .poll_complete = cxl_fw_poll_complete, 1004 .cancel = cxl_fw_cancel, 1005 .cleanup = cxl_fw_cleanup, 1006 }; 1007 1008 static void cxl_remove_fw_upload(void *fwl) 1009 { 1010 firmware_upload_unregister(fwl); 1011 } 1012 1013 int devm_cxl_setup_fw_upload(struct device *host, struct cxl_memdev_state *mds) 1014 { 1015 struct cxl_dev_state *cxlds = &mds->cxlds; 1016 struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox; 1017 struct device *dev = &cxlds->cxlmd->dev; 1018 struct fw_upload *fwl; 1019 1020 if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, cxl_mbox->enabled_cmds)) 1021 return 0; 1022 1023 fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev), 1024 &cxl_memdev_fw_ops, mds); 1025 if (IS_ERR(fwl)) 1026 return PTR_ERR(fwl); 1027 return devm_add_action_or_reset(host, cxl_remove_fw_upload, fwl); 1028 } 1029 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_fw_upload, "CXL"); 1030 1031 static const struct file_operations cxl_memdev_fops = { 1032 .owner = THIS_MODULE, 1033 .unlocked_ioctl = cxl_memdev_ioctl, 1034 .open = cxl_memdev_open, 1035 .release = cxl_memdev_release_file, 1036 .compat_ioctl = compat_ptr_ioctl, 1037 .llseek = noop_llseek, 1038 }; 1039 1040 struct cxl_memdev *devm_cxl_add_memdev(struct device *host, 1041 struct cxl_dev_state *cxlds) 1042 { 1043 struct cxl_memdev *cxlmd; 1044 struct device *dev; 1045 struct cdev *cdev; 1046 int rc; 1047 1048 cxlmd = cxl_memdev_alloc(cxlds, &cxl_memdev_fops); 1049 if (IS_ERR(cxlmd)) 1050 return cxlmd; 1051 1052 dev = &cxlmd->dev; 1053 rc = dev_set_name(dev, "mem%d", cxlmd->id); 1054 if (rc) 1055 goto err; 1056 1057 /* 1058 * Activate ioctl operations, no cxl_memdev_rwsem manipulation 1059 * needed as this is ordered with cdev_add() publishing the device. 1060 */ 1061 cxlmd->cxlds = cxlds; 1062 cxlds->cxlmd = cxlmd; 1063 1064 cdev = &cxlmd->cdev; 1065 rc = cdev_device_add(cdev, dev); 1066 if (rc) 1067 goto err; 1068 1069 rc = devm_add_action_or_reset(host, cxl_memdev_unregister, cxlmd); 1070 if (rc) 1071 return ERR_PTR(rc); 1072 return cxlmd; 1073 1074 err: 1075 /* 1076 * The cdev was briefly live, shutdown any ioctl operations that 1077 * saw that state. 1078 */ 1079 cxl_memdev_shutdown(dev); 1080 put_device(dev); 1081 return ERR_PTR(rc); 1082 } 1083 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, "CXL"); 1084 1085 static void sanitize_teardown_notifier(void *data) 1086 { 1087 struct cxl_memdev_state *mds = data; 1088 struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; 1089 struct kernfs_node *state; 1090 1091 /* 1092 * Prevent new irq triggered invocations of the workqueue and 1093 * flush inflight invocations. 1094 */ 1095 mutex_lock(&cxl_mbox->mbox_mutex); 1096 state = mds->security.sanitize_node; 1097 mds->security.sanitize_node = NULL; 1098 mutex_unlock(&cxl_mbox->mbox_mutex); 1099 1100 cancel_delayed_work_sync(&mds->security.poll_dwork); 1101 sysfs_put(state); 1102 } 1103 1104 int devm_cxl_sanitize_setup_notifier(struct device *host, 1105 struct cxl_memdev *cxlmd) 1106 { 1107 struct cxl_dev_state *cxlds = cxlmd->cxlds; 1108 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 1109 struct kernfs_node *sec; 1110 1111 if (!test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds)) 1112 return 0; 1113 1114 /* 1115 * Note, the expectation is that @cxlmd would have failed to be 1116 * created if these sysfs_get_dirent calls fail. 1117 */ 1118 sec = sysfs_get_dirent(cxlmd->dev.kobj.sd, "security"); 1119 if (!sec) 1120 return -ENOENT; 1121 mds->security.sanitize_node = sysfs_get_dirent(sec, "state"); 1122 sysfs_put(sec); 1123 if (!mds->security.sanitize_node) 1124 return -ENOENT; 1125 1126 return devm_add_action_or_reset(host, sanitize_teardown_notifier, mds); 1127 } 1128 EXPORT_SYMBOL_NS_GPL(devm_cxl_sanitize_setup_notifier, "CXL"); 1129 1130 __init int cxl_memdev_init(void) 1131 { 1132 dev_t devt; 1133 int rc; 1134 1135 rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl"); 1136 if (rc) 1137 return rc; 1138 1139 cxl_mem_major = MAJOR(devt); 1140 1141 return 0; 1142 } 1143 1144 void cxl_memdev_exit(void) 1145 { 1146 unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS); 1147 } 1148