1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */ 3 #include <linux/seq_file.h> 4 #include <linux/device.h> 5 #include <linux/delay.h> 6 7 #include "cxlmem.h" 8 #include "core.h" 9 10 /** 11 * DOC: cxl core hdm 12 * 13 * Compute Express Link Host Managed Device Memory, starting with the 14 * CXL 2.0 specification, is managed by an array of HDM Decoder register 15 * instances per CXL port and per CXL endpoint. Define common helpers 16 * for enumerating these registers and capabilities. 17 */ 18 19 DECLARE_RWSEM(cxl_dpa_rwsem); 20 21 static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, 22 int *target_map) 23 { 24 int rc; 25 26 rc = cxl_decoder_add_locked(cxld, target_map); 27 if (rc) { 28 put_device(&cxld->dev); 29 dev_err(&port->dev, "Failed to add decoder\n"); 30 return rc; 31 } 32 33 rc = cxl_decoder_autoremove(&port->dev, cxld); 34 if (rc) 35 return rc; 36 37 dev_dbg(port->uport_dev, "%s added to %s\n", 38 dev_name(&cxld->dev), dev_name(&port->dev)); 39 40 return 0; 41 } 42 43 /* 44 * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure) 45 * single ported host-bridges need not publish a decoder capability when a 46 * passthrough decode can be assumed, i.e. all transactions that the uport sees 47 * are claimed and passed to the single dport. Disable the range until the first 48 * CXL region is enumerated / activated. 49 */ 50 int devm_cxl_add_passthrough_decoder(struct cxl_port *port) 51 { 52 struct cxl_switch_decoder *cxlsd; 53 struct cxl_dport *dport = NULL; 54 int single_port_map[1]; 55 unsigned long index; 56 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev); 57 58 /* 59 * Capability checks are moot for passthrough decoders, support 60 * any and all possibilities. 61 */ 62 cxlhdm->interleave_mask = ~0U; 63 cxlhdm->iw_cap_mask = ~0UL; 64 65 cxlsd = cxl_switch_decoder_alloc(port, 1); 66 if (IS_ERR(cxlsd)) 67 return PTR_ERR(cxlsd); 68 69 device_lock_assert(&port->dev); 70 71 xa_for_each(&port->dports, index, dport) 72 break; 73 single_port_map[0] = dport->port_id; 74 75 return add_hdm_decoder(port, &cxlsd->cxld, single_port_map); 76 } 77 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, "CXL"); 78 79 static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm) 80 { 81 u32 hdm_cap; 82 83 hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET); 84 cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap); 85 cxlhdm->target_count = 86 FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap); 87 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap)) 88 cxlhdm->interleave_mask |= GENMASK(11, 8); 89 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_14_12, hdm_cap)) 90 cxlhdm->interleave_mask |= GENMASK(14, 12); 91 cxlhdm->iw_cap_mask = BIT(1) | BIT(2) | BIT(4) | BIT(8); 92 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_3_6_12_WAY, hdm_cap)) 93 cxlhdm->iw_cap_mask |= BIT(3) | BIT(6) | BIT(12); 94 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_16_WAY, hdm_cap)) 95 cxlhdm->iw_cap_mask |= BIT(16); 96 } 97 98 static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info) 99 { 100 struct cxl_hdm *cxlhdm; 101 void __iomem *hdm; 102 u32 ctrl; 103 int i; 104 105 if (!info) 106 return false; 107 108 cxlhdm = dev_get_drvdata(&info->port->dev); 109 hdm = cxlhdm->regs.hdm_decoder; 110 111 if (!hdm) 112 return true; 113 114 /* 115 * If HDM decoders are present and the driver is in control of 116 * Mem_Enable skip DVSEC based emulation 117 */ 118 if (!info->mem_enabled) 119 return false; 120 121 /* 122 * If any decoders are committed already, there should not be any 123 * emulated DVSEC decoders. 124 */ 125 for (i = 0; i < cxlhdm->decoder_count; i++) { 126 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i)); 127 dev_dbg(&info->port->dev, 128 "decoder%d.%d: committed: %ld base: %#x_%.8x size: %#x_%.8x\n", 129 info->port->id, i, 130 FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl), 131 readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i)), 132 readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(i)), 133 readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i)), 134 readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i))); 135 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl)) 136 return false; 137 } 138 139 return true; 140 } 141 142 /** 143 * devm_cxl_setup_hdm - map HDM decoder component registers 144 * @port: cxl_port to map 145 * @info: cached DVSEC range register info 146 */ 147 struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port, 148 struct cxl_endpoint_dvsec_info *info) 149 { 150 struct cxl_register_map *reg_map = &port->reg_map; 151 struct device *dev = &port->dev; 152 struct cxl_hdm *cxlhdm; 153 int rc; 154 155 cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL); 156 if (!cxlhdm) 157 return ERR_PTR(-ENOMEM); 158 cxlhdm->port = port; 159 dev_set_drvdata(dev, cxlhdm); 160 161 /* Memory devices can configure device HDM using DVSEC range regs. */ 162 if (reg_map->resource == CXL_RESOURCE_NONE) { 163 if (!info || !info->mem_enabled) { 164 dev_err(dev, "No component registers mapped\n"); 165 return ERR_PTR(-ENXIO); 166 } 167 168 cxlhdm->decoder_count = info->ranges; 169 return cxlhdm; 170 } 171 172 if (!reg_map->component_map.hdm_decoder.valid) { 173 dev_dbg(&port->dev, "HDM decoder registers not implemented\n"); 174 /* unique error code to indicate no HDM decoder capability */ 175 return ERR_PTR(-ENODEV); 176 } 177 178 rc = cxl_map_component_regs(reg_map, &cxlhdm->regs, 179 BIT(CXL_CM_CAP_CAP_ID_HDM)); 180 if (rc) { 181 dev_err(dev, "Failed to map HDM capability.\n"); 182 return ERR_PTR(rc); 183 } 184 185 parse_hdm_decoder_caps(cxlhdm); 186 if (cxlhdm->decoder_count == 0) { 187 dev_err(dev, "Spec violation. Caps invalid\n"); 188 return ERR_PTR(-ENXIO); 189 } 190 191 /* 192 * Now that the hdm capability is parsed, decide if range 193 * register emulation is needed and fixup cxlhdm accordingly. 194 */ 195 if (should_emulate_decoders(info)) { 196 dev_dbg(dev, "Fallback map %d range register%s\n", info->ranges, 197 info->ranges > 1 ? "s" : ""); 198 cxlhdm->decoder_count = info->ranges; 199 } 200 201 return cxlhdm; 202 } 203 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, "CXL"); 204 205 static void __cxl_dpa_debug(struct seq_file *file, struct resource *r, int depth) 206 { 207 unsigned long long start = r->start, end = r->end; 208 209 seq_printf(file, "%*s%08llx-%08llx : %s\n", depth * 2, "", start, end, 210 r->name); 211 } 212 213 void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds) 214 { 215 struct resource *p1, *p2; 216 217 guard(rwsem_read)(&cxl_dpa_rwsem); 218 for (p1 = cxlds->dpa_res.child; p1; p1 = p1->sibling) { 219 __cxl_dpa_debug(file, p1, 0); 220 for (p2 = p1->child; p2; p2 = p2->sibling) 221 __cxl_dpa_debug(file, p2, 1); 222 } 223 } 224 EXPORT_SYMBOL_NS_GPL(cxl_dpa_debug, "CXL"); 225 226 /* See request_skip() kernel-doc */ 227 static resource_size_t __adjust_skip(struct cxl_dev_state *cxlds, 228 const resource_size_t skip_base, 229 const resource_size_t skip_len, 230 const char *requester) 231 { 232 const resource_size_t skip_end = skip_base + skip_len - 1; 233 234 for (int i = 0; i < cxlds->nr_partitions; i++) { 235 const struct resource *part_res = &cxlds->part[i].res; 236 resource_size_t adjust_start, adjust_end, size; 237 238 adjust_start = max(skip_base, part_res->start); 239 adjust_end = min(skip_end, part_res->end); 240 241 if (adjust_end < adjust_start) 242 continue; 243 244 size = adjust_end - adjust_start + 1; 245 246 if (!requester) 247 __release_region(&cxlds->dpa_res, adjust_start, size); 248 else if (!__request_region(&cxlds->dpa_res, adjust_start, size, 249 requester, 0)) 250 return adjust_start - skip_base; 251 } 252 253 return skip_len; 254 } 255 #define release_skip(c, b, l) __adjust_skip((c), (b), (l), NULL) 256 257 /* 258 * Must be called in a context that synchronizes against this decoder's 259 * port ->remove() callback (like an endpoint decoder sysfs attribute) 260 */ 261 static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled) 262 { 263 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 264 struct cxl_port *port = cxled_to_port(cxled); 265 struct cxl_dev_state *cxlds = cxlmd->cxlds; 266 struct resource *res = cxled->dpa_res; 267 resource_size_t skip_start; 268 269 lockdep_assert_held_write(&cxl_dpa_rwsem); 270 271 /* save @skip_start, before @res is released */ 272 skip_start = res->start - cxled->skip; 273 __release_region(&cxlds->dpa_res, res->start, resource_size(res)); 274 if (cxled->skip) 275 release_skip(cxlds, skip_start, cxled->skip); 276 cxled->skip = 0; 277 cxled->dpa_res = NULL; 278 put_device(&cxled->cxld.dev); 279 port->hdm_end--; 280 } 281 282 static void cxl_dpa_release(void *cxled) 283 { 284 guard(rwsem_write)(&cxl_dpa_rwsem); 285 __cxl_dpa_release(cxled); 286 } 287 288 /* 289 * Must be called from context that will not race port device 290 * unregistration, like decoder sysfs attribute methods 291 */ 292 static void devm_cxl_dpa_release(struct cxl_endpoint_decoder *cxled) 293 { 294 struct cxl_port *port = cxled_to_port(cxled); 295 296 lockdep_assert_held_write(&cxl_dpa_rwsem); 297 devm_remove_action(&port->dev, cxl_dpa_release, cxled); 298 __cxl_dpa_release(cxled); 299 } 300 301 /** 302 * request_skip() - Track DPA 'skip' in @cxlds->dpa_res resource tree 303 * @cxlds: CXL.mem device context that parents @cxled 304 * @cxled: Endpoint decoder establishing new allocation that skips lower DPA 305 * @skip_base: DPA < start of new DPA allocation (DPAnew) 306 * @skip_len: @skip_base + @skip_len == DPAnew 307 * 308 * DPA 'skip' arises from out-of-sequence DPA allocation events relative 309 * to free capacity across multiple partitions. It is a wasteful event 310 * as usable DPA gets thrown away, but if a deployment has, for example, 311 * a dual RAM+PMEM device, wants to use PMEM, and has unallocated RAM 312 * DPA, the free RAM DPA must be sacrificed to start allocating PMEM. 313 * See third "Implementation Note" in CXL 3.1 8.2.4.19.13 "Decoder 314 * Protection" for more details. 315 * 316 * A 'skip' always covers the last allocated DPA in a previous partition 317 * to the start of the current partition to allocate. Allocations never 318 * start in the middle of a partition, and allocations are always 319 * de-allocated in reverse order (see cxl_dpa_free(), or natural devm 320 * unwind order from forced in-order allocation). 321 * 322 * If @cxlds->nr_partitions was guaranteed to be <= 2 then the 'skip' 323 * would always be contained to a single partition. Given 324 * @cxlds->nr_partitions may be > 2 it results in cases where the 'skip' 325 * might span "tail capacity of partition[0], all of partition[1], ..., 326 * all of partition[N-1]" to support allocating from partition[N]. That 327 * in turn interacts with the partition 'struct resource' boundaries 328 * within @cxlds->dpa_res whereby 'skip' requests need to be divided by 329 * partition. I.e. this is a quirk of using a 'struct resource' tree to 330 * detect range conflicts while also tracking partition boundaries in 331 * @cxlds->dpa_res. 332 */ 333 static int request_skip(struct cxl_dev_state *cxlds, 334 struct cxl_endpoint_decoder *cxled, 335 const resource_size_t skip_base, 336 const resource_size_t skip_len) 337 { 338 resource_size_t skipped = __adjust_skip(cxlds, skip_base, skip_len, 339 dev_name(&cxled->cxld.dev)); 340 341 if (skipped == skip_len) 342 return 0; 343 344 dev_dbg(cxlds->dev, 345 "%s: failed to reserve skipped space (%pa %pa %pa)\n", 346 dev_name(&cxled->cxld.dev), &skip_base, &skip_len, &skipped); 347 348 release_skip(cxlds, skip_base, skipped); 349 350 return -EBUSY; 351 } 352 353 static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled, 354 resource_size_t base, resource_size_t len, 355 resource_size_t skipped) 356 { 357 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 358 struct cxl_port *port = cxled_to_port(cxled); 359 struct cxl_dev_state *cxlds = cxlmd->cxlds; 360 struct device *dev = &port->dev; 361 struct resource *res; 362 int rc; 363 364 lockdep_assert_held_write(&cxl_dpa_rwsem); 365 366 if (!len) { 367 dev_warn(dev, "decoder%d.%d: empty reservation attempted\n", 368 port->id, cxled->cxld.id); 369 return -EINVAL; 370 } 371 372 if (cxled->dpa_res) { 373 dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n", 374 port->id, cxled->cxld.id, cxled->dpa_res); 375 return -EBUSY; 376 } 377 378 if (port->hdm_end + 1 != cxled->cxld.id) { 379 /* 380 * Assumes alloc and commit order is always in hardware instance 381 * order per expectations from 8.2.5.12.20 Committing Decoder 382 * Programming that enforce decoder[m] committed before 383 * decoder[m+1] commit start. 384 */ 385 dev_dbg(dev, "decoder%d.%d: expected decoder%d.%d\n", port->id, 386 cxled->cxld.id, port->id, port->hdm_end + 1); 387 return -EBUSY; 388 } 389 390 if (skipped) { 391 rc = request_skip(cxlds, cxled, base - skipped, skipped); 392 if (rc) 393 return rc; 394 } 395 res = __request_region(&cxlds->dpa_res, base, len, 396 dev_name(&cxled->cxld.dev), 0); 397 if (!res) { 398 dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n", 399 port->id, cxled->cxld.id); 400 if (skipped) 401 release_skip(cxlds, base - skipped, skipped); 402 return -EBUSY; 403 } 404 cxled->dpa_res = res; 405 cxled->skip = skipped; 406 407 /* 408 * When allocating new capacity, ->part is already set, when 409 * discovering decoder settings at initial enumeration, ->part 410 * is not set. 411 */ 412 if (cxled->part < 0) 413 for (int i = 0; cxlds->nr_partitions; i++) 414 if (resource_contains(&cxlds->part[i].res, res)) { 415 cxled->part = i; 416 break; 417 } 418 419 if (cxled->part < 0) 420 dev_warn(dev, "decoder%d.%d: %pr does not map any partition\n", 421 port->id, cxled->cxld.id, res); 422 423 port->hdm_end++; 424 get_device(&cxled->cxld.dev); 425 return 0; 426 } 427 428 static int add_dpa_res(struct device *dev, struct resource *parent, 429 struct resource *res, resource_size_t start, 430 resource_size_t size, const char *type) 431 { 432 int rc; 433 434 *res = (struct resource) { 435 .name = type, 436 .start = start, 437 .end = start + size - 1, 438 .flags = IORESOURCE_MEM, 439 }; 440 if (resource_size(res) == 0) { 441 dev_dbg(dev, "DPA(%s): no capacity\n", res->name); 442 return 0; 443 } 444 rc = request_resource(parent, res); 445 if (rc) { 446 dev_err(dev, "DPA(%s): failed to track %pr (%d)\n", res->name, 447 res, rc); 448 return rc; 449 } 450 451 dev_dbg(dev, "DPA(%s): %pr\n", res->name, res); 452 453 return 0; 454 } 455 456 static const char *cxl_mode_name(enum cxl_partition_mode mode) 457 { 458 switch (mode) { 459 case CXL_PARTMODE_RAM: 460 return "ram"; 461 case CXL_PARTMODE_PMEM: 462 return "pmem"; 463 default: 464 return ""; 465 }; 466 } 467 468 /* if this fails the caller must destroy @cxlds, there is no recovery */ 469 int cxl_dpa_setup(struct cxl_dev_state *cxlds, const struct cxl_dpa_info *info) 470 { 471 struct device *dev = cxlds->dev; 472 473 guard(rwsem_write)(&cxl_dpa_rwsem); 474 475 if (cxlds->nr_partitions) 476 return -EBUSY; 477 478 if (!info->size || !info->nr_partitions) { 479 cxlds->dpa_res = DEFINE_RES_MEM(0, 0); 480 cxlds->nr_partitions = 0; 481 return 0; 482 } 483 484 cxlds->dpa_res = DEFINE_RES_MEM(0, info->size); 485 486 for (int i = 0; i < info->nr_partitions; i++) { 487 const struct cxl_dpa_part_info *part = &info->part[i]; 488 int rc; 489 490 cxlds->part[i].perf.qos_class = CXL_QOS_CLASS_INVALID; 491 cxlds->part[i].mode = part->mode; 492 493 /* Require ordered + contiguous partitions */ 494 if (i) { 495 const struct cxl_dpa_part_info *prev = &info->part[i - 1]; 496 497 if (prev->range.end + 1 != part->range.start) 498 return -EINVAL; 499 } 500 rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->part[i].res, 501 part->range.start, range_len(&part->range), 502 cxl_mode_name(part->mode)); 503 if (rc) 504 return rc; 505 cxlds->nr_partitions++; 506 } 507 508 return 0; 509 } 510 EXPORT_SYMBOL_GPL(cxl_dpa_setup); 511 512 int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled, 513 resource_size_t base, resource_size_t len, 514 resource_size_t skipped) 515 { 516 struct cxl_port *port = cxled_to_port(cxled); 517 int rc; 518 519 down_write(&cxl_dpa_rwsem); 520 rc = __cxl_dpa_reserve(cxled, base, len, skipped); 521 up_write(&cxl_dpa_rwsem); 522 523 if (rc) 524 return rc; 525 526 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled); 527 } 528 EXPORT_SYMBOL_NS_GPL(devm_cxl_dpa_reserve, "CXL"); 529 530 resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled) 531 { 532 guard(rwsem_read)(&cxl_dpa_rwsem); 533 if (cxled->dpa_res) 534 return resource_size(cxled->dpa_res); 535 536 return 0; 537 } 538 539 resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled) 540 { 541 resource_size_t base = -1; 542 543 lockdep_assert_held(&cxl_dpa_rwsem); 544 if (cxled->dpa_res) 545 base = cxled->dpa_res->start; 546 547 return base; 548 } 549 550 int cxl_dpa_free(struct cxl_endpoint_decoder *cxled) 551 { 552 struct cxl_port *port = cxled_to_port(cxled); 553 struct device *dev = &cxled->cxld.dev; 554 555 guard(rwsem_write)(&cxl_dpa_rwsem); 556 if (!cxled->dpa_res) 557 return 0; 558 if (cxled->cxld.region) { 559 dev_dbg(dev, "decoder assigned to: %s\n", 560 dev_name(&cxled->cxld.region->dev)); 561 return -EBUSY; 562 } 563 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) { 564 dev_dbg(dev, "decoder enabled\n"); 565 return -EBUSY; 566 } 567 if (cxled->cxld.id != port->hdm_end) { 568 dev_dbg(dev, "expected decoder%d.%d\n", port->id, 569 port->hdm_end); 570 return -EBUSY; 571 } 572 573 devm_cxl_dpa_release(cxled); 574 return 0; 575 } 576 577 int cxl_dpa_set_part(struct cxl_endpoint_decoder *cxled, 578 enum cxl_partition_mode mode) 579 { 580 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 581 struct cxl_dev_state *cxlds = cxlmd->cxlds; 582 struct device *dev = &cxled->cxld.dev; 583 int part; 584 585 guard(rwsem_write)(&cxl_dpa_rwsem); 586 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) 587 return -EBUSY; 588 589 for (part = 0; part < cxlds->nr_partitions; part++) 590 if (cxlds->part[part].mode == mode) 591 break; 592 593 if (part >= cxlds->nr_partitions) { 594 dev_dbg(dev, "unsupported mode: %d\n", mode); 595 return -EINVAL; 596 } 597 598 if (!resource_size(&cxlds->part[part].res)) { 599 dev_dbg(dev, "no available capacity for mode: %d\n", mode); 600 return -ENXIO; 601 } 602 603 cxled->part = part; 604 return 0; 605 } 606 607 static int __cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, u64 size) 608 { 609 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 610 struct cxl_dev_state *cxlds = cxlmd->cxlds; 611 struct device *dev = &cxled->cxld.dev; 612 struct resource *res, *prev = NULL; 613 resource_size_t start, avail, skip, skip_start; 614 struct resource *p, *last; 615 int part; 616 617 guard(rwsem_write)(&cxl_dpa_rwsem); 618 if (cxled->cxld.region) { 619 dev_dbg(dev, "decoder attached to %s\n", 620 dev_name(&cxled->cxld.region->dev)); 621 return -EBUSY; 622 } 623 624 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) { 625 dev_dbg(dev, "decoder enabled\n"); 626 return -EBUSY; 627 } 628 629 part = cxled->part; 630 if (part < 0) { 631 dev_dbg(dev, "partition not set\n"); 632 return -EBUSY; 633 } 634 635 res = &cxlds->part[part].res; 636 for (p = res->child, last = NULL; p; p = p->sibling) 637 last = p; 638 if (last) 639 start = last->end + 1; 640 else 641 start = res->start; 642 643 /* 644 * To allocate at partition N, a skip needs to be calculated for all 645 * unallocated space at lower partitions indices. 646 * 647 * If a partition has any allocations, the search can end because a 648 * previous cxl_dpa_alloc() invocation is assumed to have accounted for 649 * all previous partitions. 650 */ 651 skip_start = CXL_RESOURCE_NONE; 652 for (int i = part; i; i--) { 653 prev = &cxlds->part[i - 1].res; 654 for (p = prev->child, last = NULL; p; p = p->sibling) 655 last = p; 656 if (last) { 657 skip_start = last->end + 1; 658 break; 659 } 660 skip_start = prev->start; 661 } 662 663 avail = res->end - start + 1; 664 if (skip_start == CXL_RESOURCE_NONE) 665 skip = 0; 666 else 667 skip = res->start - skip_start; 668 669 if (size > avail) { 670 dev_dbg(dev, "%llu exceeds available %s capacity: %llu\n", size, 671 res->name, (u64)avail); 672 return -ENOSPC; 673 } 674 675 return __cxl_dpa_reserve(cxled, start, size, skip); 676 } 677 678 int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, u64 size) 679 { 680 struct cxl_port *port = cxled_to_port(cxled); 681 int rc; 682 683 rc = __cxl_dpa_alloc(cxled, size); 684 if (rc) 685 return rc; 686 687 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled); 688 } 689 690 static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl) 691 { 692 u16 eig; 693 u8 eiw; 694 695 /* 696 * Input validation ensures these warns never fire, but otherwise 697 * suppress unititalized variable usage warnings. 698 */ 699 if (WARN_ONCE(ways_to_eiw(cxld->interleave_ways, &eiw), 700 "invalid interleave_ways: %d\n", cxld->interleave_ways)) 701 return; 702 if (WARN_ONCE(granularity_to_eig(cxld->interleave_granularity, &eig), 703 "invalid interleave_granularity: %d\n", 704 cxld->interleave_granularity)) 705 return; 706 707 u32p_replace_bits(ctrl, eig, CXL_HDM_DECODER0_CTRL_IG_MASK); 708 u32p_replace_bits(ctrl, eiw, CXL_HDM_DECODER0_CTRL_IW_MASK); 709 *ctrl |= CXL_HDM_DECODER0_CTRL_COMMIT; 710 } 711 712 static void cxld_set_type(struct cxl_decoder *cxld, u32 *ctrl) 713 { 714 u32p_replace_bits(ctrl, 715 !!(cxld->target_type == CXL_DECODER_HOSTONLYMEM), 716 CXL_HDM_DECODER0_CTRL_HOSTONLY); 717 } 718 719 static void cxlsd_set_targets(struct cxl_switch_decoder *cxlsd, u64 *tgt) 720 { 721 struct cxl_dport **t = &cxlsd->target[0]; 722 int ways = cxlsd->cxld.interleave_ways; 723 724 *tgt = FIELD_PREP(GENMASK(7, 0), t[0]->port_id); 725 if (ways > 1) 726 *tgt |= FIELD_PREP(GENMASK(15, 8), t[1]->port_id); 727 if (ways > 2) 728 *tgt |= FIELD_PREP(GENMASK(23, 16), t[2]->port_id); 729 if (ways > 3) 730 *tgt |= FIELD_PREP(GENMASK(31, 24), t[3]->port_id); 731 if (ways > 4) 732 *tgt |= FIELD_PREP(GENMASK_ULL(39, 32), t[4]->port_id); 733 if (ways > 5) 734 *tgt |= FIELD_PREP(GENMASK_ULL(47, 40), t[5]->port_id); 735 if (ways > 6) 736 *tgt |= FIELD_PREP(GENMASK_ULL(55, 48), t[6]->port_id); 737 if (ways > 7) 738 *tgt |= FIELD_PREP(GENMASK_ULL(63, 56), t[7]->port_id); 739 } 740 741 /* 742 * Per CXL 2.0 8.2.5.12.20 Committing Decoder Programming, hardware must set 743 * committed or error within 10ms, but just be generous with 20ms to account for 744 * clock skew and other marginal behavior 745 */ 746 #define COMMIT_TIMEOUT_MS 20 747 static int cxld_await_commit(void __iomem *hdm, int id) 748 { 749 u32 ctrl; 750 int i; 751 752 for (i = 0; i < COMMIT_TIMEOUT_MS; i++) { 753 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); 754 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMIT_ERROR, ctrl)) { 755 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT; 756 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); 757 return -EIO; 758 } 759 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl)) 760 return 0; 761 fsleep(1000); 762 } 763 764 return -ETIMEDOUT; 765 } 766 767 static int cxl_decoder_commit(struct cxl_decoder *cxld) 768 { 769 struct cxl_port *port = to_cxl_port(cxld->dev.parent); 770 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev); 771 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 772 int id = cxld->id, rc; 773 u64 base, size; 774 u32 ctrl; 775 776 if (cxld->flags & CXL_DECODER_F_ENABLE) 777 return 0; 778 779 if (cxl_num_decoders_committed(port) != id) { 780 dev_dbg(&port->dev, 781 "%s: out of order commit, expected decoder%d.%d\n", 782 dev_name(&cxld->dev), port->id, 783 cxl_num_decoders_committed(port)); 784 return -EBUSY; 785 } 786 787 /* 788 * For endpoint decoders hosted on CXL memory devices that 789 * support the sanitize operation, make sure sanitize is not in-flight. 790 */ 791 if (is_endpoint_decoder(&cxld->dev)) { 792 struct cxl_endpoint_decoder *cxled = 793 to_cxl_endpoint_decoder(&cxld->dev); 794 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 795 struct cxl_memdev_state *mds = 796 to_cxl_memdev_state(cxlmd->cxlds); 797 798 if (mds && mds->security.sanitize_active) { 799 dev_dbg(&cxlmd->dev, 800 "attempted to commit %s during sanitize\n", 801 dev_name(&cxld->dev)); 802 return -EBUSY; 803 } 804 } 805 806 down_read(&cxl_dpa_rwsem); 807 /* common decoder settings */ 808 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id)); 809 cxld_set_interleave(cxld, &ctrl); 810 cxld_set_type(cxld, &ctrl); 811 base = cxld->hpa_range.start; 812 size = range_len(&cxld->hpa_range); 813 814 writel(upper_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id)); 815 writel(lower_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id)); 816 writel(upper_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id)); 817 writel(lower_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id)); 818 819 if (is_switch_decoder(&cxld->dev)) { 820 struct cxl_switch_decoder *cxlsd = 821 to_cxl_switch_decoder(&cxld->dev); 822 void __iomem *tl_hi = hdm + CXL_HDM_DECODER0_TL_HIGH(id); 823 void __iomem *tl_lo = hdm + CXL_HDM_DECODER0_TL_LOW(id); 824 u64 targets; 825 826 cxlsd_set_targets(cxlsd, &targets); 827 writel(upper_32_bits(targets), tl_hi); 828 writel(lower_32_bits(targets), tl_lo); 829 } else { 830 struct cxl_endpoint_decoder *cxled = 831 to_cxl_endpoint_decoder(&cxld->dev); 832 void __iomem *sk_hi = hdm + CXL_HDM_DECODER0_SKIP_HIGH(id); 833 void __iomem *sk_lo = hdm + CXL_HDM_DECODER0_SKIP_LOW(id); 834 835 writel(upper_32_bits(cxled->skip), sk_hi); 836 writel(lower_32_bits(cxled->skip), sk_lo); 837 } 838 839 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); 840 up_read(&cxl_dpa_rwsem); 841 842 port->commit_end++; 843 rc = cxld_await_commit(hdm, cxld->id); 844 if (rc) { 845 dev_dbg(&port->dev, "%s: error %d committing decoder\n", 846 dev_name(&cxld->dev), rc); 847 cxld->reset(cxld); 848 return rc; 849 } 850 cxld->flags |= CXL_DECODER_F_ENABLE; 851 852 return 0; 853 } 854 855 static int commit_reap(struct device *dev, void *data) 856 { 857 struct cxl_port *port = to_cxl_port(dev->parent); 858 struct cxl_decoder *cxld; 859 860 if (!is_switch_decoder(dev) && !is_endpoint_decoder(dev)) 861 return 0; 862 863 cxld = to_cxl_decoder(dev); 864 if (port->commit_end == cxld->id && 865 ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) { 866 port->commit_end--; 867 dev_dbg(&port->dev, "reap: %s commit_end: %d\n", 868 dev_name(&cxld->dev), port->commit_end); 869 } 870 871 return 0; 872 } 873 874 void cxl_port_commit_reap(struct cxl_decoder *cxld) 875 { 876 struct cxl_port *port = to_cxl_port(cxld->dev.parent); 877 878 lockdep_assert_held_write(&cxl_region_rwsem); 879 880 /* 881 * Once the highest committed decoder is disabled, free any other 882 * decoders that were pinned allocated by out-of-order release. 883 */ 884 port->commit_end--; 885 dev_dbg(&port->dev, "reap: %s commit_end: %d\n", dev_name(&cxld->dev), 886 port->commit_end); 887 device_for_each_child_reverse_from(&port->dev, &cxld->dev, NULL, 888 commit_reap); 889 } 890 EXPORT_SYMBOL_NS_GPL(cxl_port_commit_reap, "CXL"); 891 892 static void cxl_decoder_reset(struct cxl_decoder *cxld) 893 { 894 struct cxl_port *port = to_cxl_port(cxld->dev.parent); 895 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev); 896 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 897 int id = cxld->id; 898 u32 ctrl; 899 900 if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0) 901 return; 902 903 if (port->commit_end == id) 904 cxl_port_commit_reap(cxld); 905 else 906 dev_dbg(&port->dev, 907 "%s: out of order reset, expected decoder%d.%d\n", 908 dev_name(&cxld->dev), port->id, port->commit_end); 909 910 down_read(&cxl_dpa_rwsem); 911 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); 912 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT; 913 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); 914 915 writel(0, hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id)); 916 writel(0, hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id)); 917 writel(0, hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id)); 918 writel(0, hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id)); 919 up_read(&cxl_dpa_rwsem); 920 921 cxld->flags &= ~CXL_DECODER_F_ENABLE; 922 923 /* Userspace is now responsible for reconfiguring this decoder */ 924 if (is_endpoint_decoder(&cxld->dev)) { 925 struct cxl_endpoint_decoder *cxled; 926 927 cxled = to_cxl_endpoint_decoder(&cxld->dev); 928 cxled->state = CXL_DECODER_STATE_MANUAL; 929 } 930 } 931 932 static int cxl_setup_hdm_decoder_from_dvsec( 933 struct cxl_port *port, struct cxl_decoder *cxld, u64 *dpa_base, 934 int which, struct cxl_endpoint_dvsec_info *info) 935 { 936 struct cxl_endpoint_decoder *cxled; 937 u64 len; 938 int rc; 939 940 if (!is_cxl_endpoint(port)) 941 return -EOPNOTSUPP; 942 943 cxled = to_cxl_endpoint_decoder(&cxld->dev); 944 len = range_len(&info->dvsec_range[which]); 945 if (!len) 946 return -ENOENT; 947 948 cxld->target_type = CXL_DECODER_HOSTONLYMEM; 949 cxld->commit = NULL; 950 cxld->reset = NULL; 951 cxld->hpa_range = info->dvsec_range[which]; 952 953 /* 954 * Set the emulated decoder as locked pending additional support to 955 * change the range registers at run time. 956 */ 957 cxld->flags |= CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK; 958 port->commit_end = cxld->id; 959 960 rc = devm_cxl_dpa_reserve(cxled, *dpa_base, len, 0); 961 if (rc) { 962 dev_err(&port->dev, 963 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)", 964 port->id, cxld->id, *dpa_base, *dpa_base + len - 1, rc); 965 return rc; 966 } 967 *dpa_base += len; 968 cxled->state = CXL_DECODER_STATE_AUTO; 969 970 return 0; 971 } 972 973 static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, 974 int *target_map, void __iomem *hdm, int which, 975 u64 *dpa_base, struct cxl_endpoint_dvsec_info *info) 976 { 977 struct cxl_endpoint_decoder *cxled = NULL; 978 u64 size, base, skip, dpa_size, lo, hi; 979 bool committed; 980 u32 remainder; 981 int i, rc; 982 u32 ctrl; 983 union { 984 u64 value; 985 unsigned char target_id[8]; 986 } target_list; 987 988 if (should_emulate_decoders(info)) 989 return cxl_setup_hdm_decoder_from_dvsec(port, cxld, dpa_base, 990 which, info); 991 992 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which)); 993 lo = readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which)); 994 hi = readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(which)); 995 base = (hi << 32) + lo; 996 lo = readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which)); 997 hi = readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(which)); 998 size = (hi << 32) + lo; 999 committed = !!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED); 1000 cxld->commit = cxl_decoder_commit; 1001 cxld->reset = cxl_decoder_reset; 1002 1003 if (!committed) 1004 size = 0; 1005 if (base == U64_MAX || size == U64_MAX) { 1006 dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n", 1007 port->id, cxld->id); 1008 return -ENXIO; 1009 } 1010 1011 if (info) 1012 cxled = to_cxl_endpoint_decoder(&cxld->dev); 1013 cxld->hpa_range = (struct range) { 1014 .start = base, 1015 .end = base + size - 1, 1016 }; 1017 1018 /* decoders are enabled if committed */ 1019 if (committed) { 1020 cxld->flags |= CXL_DECODER_F_ENABLE; 1021 if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK) 1022 cxld->flags |= CXL_DECODER_F_LOCK; 1023 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl)) 1024 cxld->target_type = CXL_DECODER_HOSTONLYMEM; 1025 else 1026 cxld->target_type = CXL_DECODER_DEVMEM; 1027 1028 guard(rwsem_write)(&cxl_region_rwsem); 1029 if (cxld->id != cxl_num_decoders_committed(port)) { 1030 dev_warn(&port->dev, 1031 "decoder%d.%d: Committed out of order\n", 1032 port->id, cxld->id); 1033 return -ENXIO; 1034 } 1035 1036 if (size == 0) { 1037 dev_warn(&port->dev, 1038 "decoder%d.%d: Committed with zero size\n", 1039 port->id, cxld->id); 1040 return -ENXIO; 1041 } 1042 port->commit_end = cxld->id; 1043 } else { 1044 if (cxled) { 1045 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1046 struct cxl_dev_state *cxlds = cxlmd->cxlds; 1047 1048 /* 1049 * Default by devtype until a device arrives that needs 1050 * more precision. 1051 */ 1052 if (cxlds->type == CXL_DEVTYPE_CLASSMEM) 1053 cxld->target_type = CXL_DECODER_HOSTONLYMEM; 1054 else 1055 cxld->target_type = CXL_DECODER_DEVMEM; 1056 } else { 1057 /* To be overridden by region type at commit time */ 1058 cxld->target_type = CXL_DECODER_HOSTONLYMEM; 1059 } 1060 1061 if (!FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl) && 1062 cxld->target_type == CXL_DECODER_HOSTONLYMEM) { 1063 ctrl |= CXL_HDM_DECODER0_CTRL_HOSTONLY; 1064 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which)); 1065 } 1066 } 1067 rc = eiw_to_ways(FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl), 1068 &cxld->interleave_ways); 1069 if (rc) { 1070 dev_warn(&port->dev, 1071 "decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n", 1072 port->id, cxld->id, ctrl); 1073 return rc; 1074 } 1075 rc = eig_to_granularity(FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl), 1076 &cxld->interleave_granularity); 1077 if (rc) { 1078 dev_warn(&port->dev, 1079 "decoder%d.%d: Invalid interleave granularity (ctrl: %#x)\n", 1080 port->id, cxld->id, ctrl); 1081 return rc; 1082 } 1083 1084 dev_dbg(&port->dev, "decoder%d.%d: range: %#llx-%#llx iw: %d ig: %d\n", 1085 port->id, cxld->id, cxld->hpa_range.start, cxld->hpa_range.end, 1086 cxld->interleave_ways, cxld->interleave_granularity); 1087 1088 if (!cxled) { 1089 lo = readl(hdm + CXL_HDM_DECODER0_TL_LOW(which)); 1090 hi = readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which)); 1091 target_list.value = (hi << 32) + lo; 1092 for (i = 0; i < cxld->interleave_ways; i++) 1093 target_map[i] = target_list.target_id[i]; 1094 1095 return 0; 1096 } 1097 1098 if (!committed) 1099 return 0; 1100 1101 dpa_size = div_u64_rem(size, cxld->interleave_ways, &remainder); 1102 if (remainder) { 1103 dev_err(&port->dev, 1104 "decoder%d.%d: invalid committed configuration size: %#llx ways: %d\n", 1105 port->id, cxld->id, size, cxld->interleave_ways); 1106 return -ENXIO; 1107 } 1108 lo = readl(hdm + CXL_HDM_DECODER0_SKIP_LOW(which)); 1109 hi = readl(hdm + CXL_HDM_DECODER0_SKIP_HIGH(which)); 1110 skip = (hi << 32) + lo; 1111 rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip); 1112 if (rc) { 1113 dev_err(&port->dev, 1114 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)", 1115 port->id, cxld->id, *dpa_base, 1116 *dpa_base + dpa_size + skip - 1, rc); 1117 return rc; 1118 } 1119 *dpa_base += dpa_size + skip; 1120 1121 cxled->state = CXL_DECODER_STATE_AUTO; 1122 1123 return 0; 1124 } 1125 1126 static void cxl_settle_decoders(struct cxl_hdm *cxlhdm) 1127 { 1128 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 1129 int committed, i; 1130 u32 ctrl; 1131 1132 if (!hdm) 1133 return; 1134 1135 /* 1136 * Since the register resource was recently claimed via request_region() 1137 * be careful about trusting the "not-committed" status until the commit 1138 * timeout has elapsed. The commit timeout is 10ms (CXL 2.0 1139 * 8.2.5.12.20), but double it to be tolerant of any clock skew between 1140 * host and target. 1141 */ 1142 for (i = 0, committed = 0; i < cxlhdm->decoder_count; i++) { 1143 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i)); 1144 if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED) 1145 committed++; 1146 } 1147 1148 /* ensure that future checks of committed can be trusted */ 1149 if (committed != cxlhdm->decoder_count) 1150 msleep(20); 1151 } 1152 1153 /** 1154 * devm_cxl_enumerate_decoders - add decoder objects per HDM register set 1155 * @cxlhdm: Structure to populate with HDM capabilities 1156 * @info: cached DVSEC range register info 1157 */ 1158 int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm, 1159 struct cxl_endpoint_dvsec_info *info) 1160 { 1161 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 1162 struct cxl_port *port = cxlhdm->port; 1163 int i; 1164 u64 dpa_base = 0; 1165 1166 cxl_settle_decoders(cxlhdm); 1167 1168 for (i = 0; i < cxlhdm->decoder_count; i++) { 1169 int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 }; 1170 int rc, target_count = cxlhdm->target_count; 1171 struct cxl_decoder *cxld; 1172 1173 if (is_cxl_endpoint(port)) { 1174 struct cxl_endpoint_decoder *cxled; 1175 1176 cxled = cxl_endpoint_decoder_alloc(port); 1177 if (IS_ERR(cxled)) { 1178 dev_warn(&port->dev, 1179 "Failed to allocate decoder%d.%d\n", 1180 port->id, i); 1181 return PTR_ERR(cxled); 1182 } 1183 cxld = &cxled->cxld; 1184 } else { 1185 struct cxl_switch_decoder *cxlsd; 1186 1187 cxlsd = cxl_switch_decoder_alloc(port, target_count); 1188 if (IS_ERR(cxlsd)) { 1189 dev_warn(&port->dev, 1190 "Failed to allocate decoder%d.%d\n", 1191 port->id, i); 1192 return PTR_ERR(cxlsd); 1193 } 1194 cxld = &cxlsd->cxld; 1195 } 1196 1197 rc = init_hdm_decoder(port, cxld, target_map, hdm, i, 1198 &dpa_base, info); 1199 if (rc) { 1200 dev_warn(&port->dev, 1201 "Failed to initialize decoder%d.%d\n", 1202 port->id, i); 1203 put_device(&cxld->dev); 1204 return rc; 1205 } 1206 rc = add_hdm_decoder(port, cxld, target_map); 1207 if (rc) { 1208 dev_warn(&port->dev, 1209 "Failed to add decoder%d.%d\n", port->id, i); 1210 return rc; 1211 } 1212 } 1213 1214 return 0; 1215 } 1216 EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, "CXL"); 1217