1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3 #include <linux/seq_file.h>
4 #include <linux/device.h>
5 #include <linux/delay.h>
6
7 #include "cxlmem.h"
8 #include "core.h"
9
10 /**
11 * DOC: cxl core hdm
12 *
13 * Compute Express Link Host Managed Device Memory, starting with the
14 * CXL 2.0 specification, is managed by an array of HDM Decoder register
15 * instances per CXL port and per CXL endpoint. Define common helpers
16 * for enumerating these registers and capabilities.
17 */
18
19 struct cxl_rwsem cxl_rwsem = {
20 .region = __RWSEM_INITIALIZER(cxl_rwsem.region),
21 .dpa = __RWSEM_INITIALIZER(cxl_rwsem.dpa),
22 };
23
add_hdm_decoder(struct cxl_port * port,struct cxl_decoder * cxld,int * target_map)24 static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
25 int *target_map)
26 {
27 int rc;
28
29 rc = cxl_decoder_add_locked(cxld, target_map);
30 if (rc) {
31 put_device(&cxld->dev);
32 dev_err(&port->dev, "Failed to add decoder\n");
33 return rc;
34 }
35
36 rc = cxl_decoder_autoremove(&port->dev, cxld);
37 if (rc)
38 return rc;
39
40 dev_dbg(port->uport_dev, "%s added to %s\n",
41 dev_name(&cxld->dev), dev_name(&port->dev));
42
43 return 0;
44 }
45
46 /*
47 * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure)
48 * single ported host-bridges need not publish a decoder capability when a
49 * passthrough decode can be assumed, i.e. all transactions that the uport sees
50 * are claimed and passed to the single dport. Disable the range until the first
51 * CXL region is enumerated / activated.
52 */
devm_cxl_add_passthrough_decoder(struct cxl_port * port)53 int devm_cxl_add_passthrough_decoder(struct cxl_port *port)
54 {
55 struct cxl_switch_decoder *cxlsd;
56 struct cxl_dport *dport = NULL;
57 int single_port_map[1];
58 unsigned long index;
59 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
60
61 /*
62 * Capability checks are moot for passthrough decoders, support
63 * any and all possibilities.
64 */
65 cxlhdm->interleave_mask = ~0U;
66 cxlhdm->iw_cap_mask = ~0UL;
67
68 cxlsd = cxl_switch_decoder_alloc(port, 1);
69 if (IS_ERR(cxlsd))
70 return PTR_ERR(cxlsd);
71
72 device_lock_assert(&port->dev);
73
74 xa_for_each(&port->dports, index, dport)
75 break;
76 single_port_map[0] = dport->port_id;
77
78 return add_hdm_decoder(port, &cxlsd->cxld, single_port_map);
79 }
80 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, "CXL");
81
parse_hdm_decoder_caps(struct cxl_hdm * cxlhdm)82 static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm)
83 {
84 u32 hdm_cap;
85
86 hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET);
87 cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap);
88 cxlhdm->target_count =
89 FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap);
90 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap))
91 cxlhdm->interleave_mask |= GENMASK(11, 8);
92 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_14_12, hdm_cap))
93 cxlhdm->interleave_mask |= GENMASK(14, 12);
94 cxlhdm->iw_cap_mask = BIT(1) | BIT(2) | BIT(4) | BIT(8);
95 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_3_6_12_WAY, hdm_cap))
96 cxlhdm->iw_cap_mask |= BIT(3) | BIT(6) | BIT(12);
97 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_16_WAY, hdm_cap))
98 cxlhdm->iw_cap_mask |= BIT(16);
99 }
100
should_emulate_decoders(struct cxl_endpoint_dvsec_info * info)101 static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info)
102 {
103 struct cxl_hdm *cxlhdm;
104 void __iomem *hdm;
105 u32 ctrl;
106 int i;
107
108 if (!info)
109 return false;
110
111 cxlhdm = dev_get_drvdata(&info->port->dev);
112 hdm = cxlhdm->regs.hdm_decoder;
113
114 if (!hdm)
115 return true;
116
117 /*
118 * If HDM decoders are present and the driver is in control of
119 * Mem_Enable skip DVSEC based emulation
120 */
121 if (!info->mem_enabled)
122 return false;
123
124 /*
125 * If any decoders are committed already, there should not be any
126 * emulated DVSEC decoders.
127 */
128 for (i = 0; i < cxlhdm->decoder_count; i++) {
129 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
130 dev_dbg(&info->port->dev,
131 "decoder%d.%d: committed: %ld base: %#x_%.8x size: %#x_%.8x\n",
132 info->port->id, i,
133 FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl),
134 readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i)),
135 readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(i)),
136 readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i)),
137 readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i)));
138 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
139 return false;
140 }
141
142 return true;
143 }
144
145 /**
146 * devm_cxl_setup_hdm - map HDM decoder component registers
147 * @port: cxl_port to map
148 * @info: cached DVSEC range register info
149 */
devm_cxl_setup_hdm(struct cxl_port * port,struct cxl_endpoint_dvsec_info * info)150 struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port,
151 struct cxl_endpoint_dvsec_info *info)
152 {
153 struct cxl_register_map *reg_map = &port->reg_map;
154 struct device *dev = &port->dev;
155 struct cxl_hdm *cxlhdm;
156 int rc;
157
158 cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL);
159 if (!cxlhdm)
160 return ERR_PTR(-ENOMEM);
161 cxlhdm->port = port;
162 dev_set_drvdata(dev, cxlhdm);
163
164 /* Memory devices can configure device HDM using DVSEC range regs. */
165 if (reg_map->resource == CXL_RESOURCE_NONE) {
166 if (!info || !info->mem_enabled) {
167 dev_err(dev, "No component registers mapped\n");
168 return ERR_PTR(-ENXIO);
169 }
170
171 cxlhdm->decoder_count = info->ranges;
172 return cxlhdm;
173 }
174
175 if (!reg_map->component_map.hdm_decoder.valid) {
176 dev_dbg(&port->dev, "HDM decoder registers not implemented\n");
177 /* unique error code to indicate no HDM decoder capability */
178 return ERR_PTR(-ENODEV);
179 }
180
181 rc = cxl_map_component_regs(reg_map, &cxlhdm->regs,
182 BIT(CXL_CM_CAP_CAP_ID_HDM));
183 if (rc) {
184 dev_err(dev, "Failed to map HDM capability.\n");
185 return ERR_PTR(rc);
186 }
187
188 parse_hdm_decoder_caps(cxlhdm);
189 if (cxlhdm->decoder_count == 0) {
190 dev_err(dev, "Spec violation. Caps invalid\n");
191 return ERR_PTR(-ENXIO);
192 }
193
194 /*
195 * Now that the hdm capability is parsed, decide if range
196 * register emulation is needed and fixup cxlhdm accordingly.
197 */
198 if (should_emulate_decoders(info)) {
199 dev_dbg(dev, "Fallback map %d range register%s\n", info->ranges,
200 info->ranges > 1 ? "s" : "");
201 cxlhdm->decoder_count = info->ranges;
202 }
203
204 return cxlhdm;
205 }
206 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, "CXL");
207
__cxl_dpa_debug(struct seq_file * file,struct resource * r,int depth)208 static void __cxl_dpa_debug(struct seq_file *file, struct resource *r, int depth)
209 {
210 unsigned long long start = r->start, end = r->end;
211
212 seq_printf(file, "%*s%08llx-%08llx : %s\n", depth * 2, "", start, end,
213 r->name);
214 }
215
cxl_dpa_debug(struct seq_file * file,struct cxl_dev_state * cxlds)216 void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds)
217 {
218 struct resource *p1, *p2;
219
220 guard(rwsem_read)(&cxl_rwsem.dpa);
221 for (p1 = cxlds->dpa_res.child; p1; p1 = p1->sibling) {
222 __cxl_dpa_debug(file, p1, 0);
223 for (p2 = p1->child; p2; p2 = p2->sibling)
224 __cxl_dpa_debug(file, p2, 1);
225 }
226 }
227 EXPORT_SYMBOL_NS_GPL(cxl_dpa_debug, "CXL");
228
229 /* See request_skip() kernel-doc */
__adjust_skip(struct cxl_dev_state * cxlds,const resource_size_t skip_base,const resource_size_t skip_len,const char * requester)230 static resource_size_t __adjust_skip(struct cxl_dev_state *cxlds,
231 const resource_size_t skip_base,
232 const resource_size_t skip_len,
233 const char *requester)
234 {
235 const resource_size_t skip_end = skip_base + skip_len - 1;
236
237 for (int i = 0; i < cxlds->nr_partitions; i++) {
238 const struct resource *part_res = &cxlds->part[i].res;
239 resource_size_t adjust_start, adjust_end, size;
240
241 adjust_start = max(skip_base, part_res->start);
242 adjust_end = min(skip_end, part_res->end);
243
244 if (adjust_end < adjust_start)
245 continue;
246
247 size = adjust_end - adjust_start + 1;
248
249 if (!requester)
250 __release_region(&cxlds->dpa_res, adjust_start, size);
251 else if (!__request_region(&cxlds->dpa_res, adjust_start, size,
252 requester, 0))
253 return adjust_start - skip_base;
254 }
255
256 return skip_len;
257 }
258 #define release_skip(c, b, l) __adjust_skip((c), (b), (l), NULL)
259
260 /*
261 * Must be called in a context that synchronizes against this decoder's
262 * port ->remove() callback (like an endpoint decoder sysfs attribute)
263 */
__cxl_dpa_release(struct cxl_endpoint_decoder * cxled)264 static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
265 {
266 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
267 struct cxl_port *port = cxled_to_port(cxled);
268 struct cxl_dev_state *cxlds = cxlmd->cxlds;
269 struct resource *res = cxled->dpa_res;
270 resource_size_t skip_start;
271
272 lockdep_assert_held_write(&cxl_rwsem.dpa);
273
274 /* save @skip_start, before @res is released */
275 skip_start = res->start - cxled->skip;
276 __release_region(&cxlds->dpa_res, res->start, resource_size(res));
277 if (cxled->skip)
278 release_skip(cxlds, skip_start, cxled->skip);
279 cxled->skip = 0;
280 cxled->dpa_res = NULL;
281 put_device(&cxled->cxld.dev);
282 port->hdm_end--;
283 }
284
cxl_dpa_release(void * cxled)285 static void cxl_dpa_release(void *cxled)
286 {
287 guard(rwsem_write)(&cxl_rwsem.dpa);
288 __cxl_dpa_release(cxled);
289 }
290
291 /*
292 * Must be called from context that will not race port device
293 * unregistration, like decoder sysfs attribute methods
294 */
devm_cxl_dpa_release(struct cxl_endpoint_decoder * cxled)295 static void devm_cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
296 {
297 struct cxl_port *port = cxled_to_port(cxled);
298
299 lockdep_assert_held_write(&cxl_rwsem.dpa);
300 devm_remove_action(&port->dev, cxl_dpa_release, cxled);
301 __cxl_dpa_release(cxled);
302 }
303
304 /**
305 * request_skip() - Track DPA 'skip' in @cxlds->dpa_res resource tree
306 * @cxlds: CXL.mem device context that parents @cxled
307 * @cxled: Endpoint decoder establishing new allocation that skips lower DPA
308 * @skip_base: DPA < start of new DPA allocation (DPAnew)
309 * @skip_len: @skip_base + @skip_len == DPAnew
310 *
311 * DPA 'skip' arises from out-of-sequence DPA allocation events relative
312 * to free capacity across multiple partitions. It is a wasteful event
313 * as usable DPA gets thrown away, but if a deployment has, for example,
314 * a dual RAM+PMEM device, wants to use PMEM, and has unallocated RAM
315 * DPA, the free RAM DPA must be sacrificed to start allocating PMEM.
316 * See third "Implementation Note" in CXL 3.1 8.2.4.19.13 "Decoder
317 * Protection" for more details.
318 *
319 * A 'skip' always covers the last allocated DPA in a previous partition
320 * to the start of the current partition to allocate. Allocations never
321 * start in the middle of a partition, and allocations are always
322 * de-allocated in reverse order (see cxl_dpa_free(), or natural devm
323 * unwind order from forced in-order allocation).
324 *
325 * If @cxlds->nr_partitions was guaranteed to be <= 2 then the 'skip'
326 * would always be contained to a single partition. Given
327 * @cxlds->nr_partitions may be > 2 it results in cases where the 'skip'
328 * might span "tail capacity of partition[0], all of partition[1], ...,
329 * all of partition[N-1]" to support allocating from partition[N]. That
330 * in turn interacts with the partition 'struct resource' boundaries
331 * within @cxlds->dpa_res whereby 'skip' requests need to be divided by
332 * partition. I.e. this is a quirk of using a 'struct resource' tree to
333 * detect range conflicts while also tracking partition boundaries in
334 * @cxlds->dpa_res.
335 */
request_skip(struct cxl_dev_state * cxlds,struct cxl_endpoint_decoder * cxled,const resource_size_t skip_base,const resource_size_t skip_len)336 static int request_skip(struct cxl_dev_state *cxlds,
337 struct cxl_endpoint_decoder *cxled,
338 const resource_size_t skip_base,
339 const resource_size_t skip_len)
340 {
341 resource_size_t skipped = __adjust_skip(cxlds, skip_base, skip_len,
342 dev_name(&cxled->cxld.dev));
343
344 if (skipped == skip_len)
345 return 0;
346
347 dev_dbg(cxlds->dev,
348 "%s: failed to reserve skipped space (%pa %pa %pa)\n",
349 dev_name(&cxled->cxld.dev), &skip_base, &skip_len, &skipped);
350
351 release_skip(cxlds, skip_base, skipped);
352
353 return -EBUSY;
354 }
355
__cxl_dpa_reserve(struct cxl_endpoint_decoder * cxled,resource_size_t base,resource_size_t len,resource_size_t skipped)356 static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
357 resource_size_t base, resource_size_t len,
358 resource_size_t skipped)
359 {
360 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
361 struct cxl_port *port = cxled_to_port(cxled);
362 struct cxl_dev_state *cxlds = cxlmd->cxlds;
363 struct device *dev = &port->dev;
364 struct resource *res;
365 int rc;
366
367 lockdep_assert_held_write(&cxl_rwsem.dpa);
368
369 if (!len) {
370 dev_warn(dev, "decoder%d.%d: empty reservation attempted\n",
371 port->id, cxled->cxld.id);
372 return -EINVAL;
373 }
374
375 if (cxled->dpa_res) {
376 dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n",
377 port->id, cxled->cxld.id, cxled->dpa_res);
378 return -EBUSY;
379 }
380
381 if (port->hdm_end + 1 != cxled->cxld.id) {
382 /*
383 * Assumes alloc and commit order is always in hardware instance
384 * order per expectations from 8.2.5.12.20 Committing Decoder
385 * Programming that enforce decoder[m] committed before
386 * decoder[m+1] commit start.
387 */
388 dev_dbg(dev, "decoder%d.%d: expected decoder%d.%d\n", port->id,
389 cxled->cxld.id, port->id, port->hdm_end + 1);
390 return -EBUSY;
391 }
392
393 if (skipped) {
394 rc = request_skip(cxlds, cxled, base - skipped, skipped);
395 if (rc)
396 return rc;
397 }
398 res = __request_region(&cxlds->dpa_res, base, len,
399 dev_name(&cxled->cxld.dev), 0);
400 if (!res) {
401 dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n",
402 port->id, cxled->cxld.id);
403 if (skipped)
404 release_skip(cxlds, base - skipped, skipped);
405 return -EBUSY;
406 }
407 cxled->dpa_res = res;
408 cxled->skip = skipped;
409
410 /*
411 * When allocating new capacity, ->part is already set, when
412 * discovering decoder settings at initial enumeration, ->part
413 * is not set.
414 */
415 if (cxled->part < 0)
416 for (int i = 0; cxlds->nr_partitions; i++)
417 if (resource_contains(&cxlds->part[i].res, res)) {
418 cxled->part = i;
419 break;
420 }
421
422 if (cxled->part < 0)
423 dev_warn(dev, "decoder%d.%d: %pr does not map any partition\n",
424 port->id, cxled->cxld.id, res);
425
426 port->hdm_end++;
427 get_device(&cxled->cxld.dev);
428 return 0;
429 }
430
add_dpa_res(struct device * dev,struct resource * parent,struct resource * res,resource_size_t start,resource_size_t size,const char * type)431 static int add_dpa_res(struct device *dev, struct resource *parent,
432 struct resource *res, resource_size_t start,
433 resource_size_t size, const char *type)
434 {
435 int rc;
436
437 *res = (struct resource) {
438 .name = type,
439 .start = start,
440 .end = start + size - 1,
441 .flags = IORESOURCE_MEM,
442 };
443 if (resource_size(res) == 0) {
444 dev_dbg(dev, "DPA(%s): no capacity\n", res->name);
445 return 0;
446 }
447 rc = request_resource(parent, res);
448 if (rc) {
449 dev_err(dev, "DPA(%s): failed to track %pr (%d)\n", res->name,
450 res, rc);
451 return rc;
452 }
453
454 dev_dbg(dev, "DPA(%s): %pr\n", res->name, res);
455
456 return 0;
457 }
458
cxl_mode_name(enum cxl_partition_mode mode)459 static const char *cxl_mode_name(enum cxl_partition_mode mode)
460 {
461 switch (mode) {
462 case CXL_PARTMODE_RAM:
463 return "ram";
464 case CXL_PARTMODE_PMEM:
465 return "pmem";
466 default:
467 return "";
468 };
469 }
470
471 /* if this fails the caller must destroy @cxlds, there is no recovery */
cxl_dpa_setup(struct cxl_dev_state * cxlds,const struct cxl_dpa_info * info)472 int cxl_dpa_setup(struct cxl_dev_state *cxlds, const struct cxl_dpa_info *info)
473 {
474 struct device *dev = cxlds->dev;
475
476 guard(rwsem_write)(&cxl_rwsem.dpa);
477
478 if (cxlds->nr_partitions)
479 return -EBUSY;
480
481 if (!info->size || !info->nr_partitions) {
482 cxlds->dpa_res = DEFINE_RES_MEM(0, 0);
483 cxlds->nr_partitions = 0;
484 return 0;
485 }
486
487 cxlds->dpa_res = DEFINE_RES_MEM(0, info->size);
488
489 for (int i = 0; i < info->nr_partitions; i++) {
490 const struct cxl_dpa_part_info *part = &info->part[i];
491 int rc;
492
493 cxlds->part[i].perf.qos_class = CXL_QOS_CLASS_INVALID;
494 cxlds->part[i].mode = part->mode;
495
496 /* Require ordered + contiguous partitions */
497 if (i) {
498 const struct cxl_dpa_part_info *prev = &info->part[i - 1];
499
500 if (prev->range.end + 1 != part->range.start)
501 return -EINVAL;
502 }
503 rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->part[i].res,
504 part->range.start, range_len(&part->range),
505 cxl_mode_name(part->mode));
506 if (rc)
507 return rc;
508 cxlds->nr_partitions++;
509 }
510
511 return 0;
512 }
513 EXPORT_SYMBOL_GPL(cxl_dpa_setup);
514
devm_cxl_dpa_reserve(struct cxl_endpoint_decoder * cxled,resource_size_t base,resource_size_t len,resource_size_t skipped)515 int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
516 resource_size_t base, resource_size_t len,
517 resource_size_t skipped)
518 {
519 struct cxl_port *port = cxled_to_port(cxled);
520 int rc;
521
522 scoped_guard(rwsem_write, &cxl_rwsem.dpa)
523 rc = __cxl_dpa_reserve(cxled, base, len, skipped);
524
525 if (rc)
526 return rc;
527
528 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
529 }
530 EXPORT_SYMBOL_NS_GPL(devm_cxl_dpa_reserve, "CXL");
531
cxl_dpa_size(struct cxl_endpoint_decoder * cxled)532 resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled)
533 {
534 guard(rwsem_read)(&cxl_rwsem.dpa);
535 if (cxled->dpa_res)
536 return resource_size(cxled->dpa_res);
537
538 return 0;
539 }
540
cxl_dpa_resource_start(struct cxl_endpoint_decoder * cxled)541 resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled)
542 {
543 resource_size_t base = -1;
544
545 lockdep_assert_held(&cxl_rwsem.dpa);
546 if (cxled->dpa_res)
547 base = cxled->dpa_res->start;
548
549 return base;
550 }
551
cxl_resource_contains_addr(const struct resource * res,const resource_size_t addr)552 bool cxl_resource_contains_addr(const struct resource *res, const resource_size_t addr)
553 {
554 struct resource _addr = DEFINE_RES_MEM(addr, 1);
555
556 return resource_contains(res, &_addr);
557 }
558
cxl_dpa_free(struct cxl_endpoint_decoder * cxled)559 int cxl_dpa_free(struct cxl_endpoint_decoder *cxled)
560 {
561 struct cxl_port *port = cxled_to_port(cxled);
562 struct device *dev = &cxled->cxld.dev;
563
564 guard(rwsem_write)(&cxl_rwsem.dpa);
565 if (!cxled->dpa_res)
566 return 0;
567 if (cxled->cxld.region) {
568 dev_dbg(dev, "decoder assigned to: %s\n",
569 dev_name(&cxled->cxld.region->dev));
570 return -EBUSY;
571 }
572 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
573 dev_dbg(dev, "decoder enabled\n");
574 return -EBUSY;
575 }
576 if (cxled->cxld.id != port->hdm_end) {
577 dev_dbg(dev, "expected decoder%d.%d\n", port->id,
578 port->hdm_end);
579 return -EBUSY;
580 }
581
582 devm_cxl_dpa_release(cxled);
583 return 0;
584 }
585
cxl_dpa_set_part(struct cxl_endpoint_decoder * cxled,enum cxl_partition_mode mode)586 int cxl_dpa_set_part(struct cxl_endpoint_decoder *cxled,
587 enum cxl_partition_mode mode)
588 {
589 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
590 struct cxl_dev_state *cxlds = cxlmd->cxlds;
591 struct device *dev = &cxled->cxld.dev;
592 int part;
593
594 guard(rwsem_write)(&cxl_rwsem.dpa);
595 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE)
596 return -EBUSY;
597
598 for (part = 0; part < cxlds->nr_partitions; part++)
599 if (cxlds->part[part].mode == mode)
600 break;
601
602 if (part >= cxlds->nr_partitions) {
603 dev_dbg(dev, "unsupported mode: %d\n", mode);
604 return -EINVAL;
605 }
606
607 if (!resource_size(&cxlds->part[part].res)) {
608 dev_dbg(dev, "no available capacity for mode: %d\n", mode);
609 return -ENXIO;
610 }
611
612 cxled->part = part;
613 return 0;
614 }
615
__cxl_dpa_alloc(struct cxl_endpoint_decoder * cxled,u64 size)616 static int __cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, u64 size)
617 {
618 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
619 struct cxl_dev_state *cxlds = cxlmd->cxlds;
620 struct device *dev = &cxled->cxld.dev;
621 struct resource *res, *prev = NULL;
622 resource_size_t start, avail, skip, skip_start;
623 struct resource *p, *last;
624 int part;
625
626 guard(rwsem_write)(&cxl_rwsem.dpa);
627 if (cxled->cxld.region) {
628 dev_dbg(dev, "decoder attached to %s\n",
629 dev_name(&cxled->cxld.region->dev));
630 return -EBUSY;
631 }
632
633 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
634 dev_dbg(dev, "decoder enabled\n");
635 return -EBUSY;
636 }
637
638 part = cxled->part;
639 if (part < 0) {
640 dev_dbg(dev, "partition not set\n");
641 return -EBUSY;
642 }
643
644 res = &cxlds->part[part].res;
645 for (p = res->child, last = NULL; p; p = p->sibling)
646 last = p;
647 if (last)
648 start = last->end + 1;
649 else
650 start = res->start;
651
652 /*
653 * To allocate at partition N, a skip needs to be calculated for all
654 * unallocated space at lower partitions indices.
655 *
656 * If a partition has any allocations, the search can end because a
657 * previous cxl_dpa_alloc() invocation is assumed to have accounted for
658 * all previous partitions.
659 */
660 skip_start = CXL_RESOURCE_NONE;
661 for (int i = part; i; i--) {
662 prev = &cxlds->part[i - 1].res;
663 for (p = prev->child, last = NULL; p; p = p->sibling)
664 last = p;
665 if (last) {
666 skip_start = last->end + 1;
667 break;
668 }
669 skip_start = prev->start;
670 }
671
672 avail = res->end - start + 1;
673 if (skip_start == CXL_RESOURCE_NONE)
674 skip = 0;
675 else
676 skip = res->start - skip_start;
677
678 if (size > avail) {
679 dev_dbg(dev, "%llu exceeds available %s capacity: %llu\n", size,
680 res->name, (u64)avail);
681 return -ENOSPC;
682 }
683
684 return __cxl_dpa_reserve(cxled, start, size, skip);
685 }
686
cxl_dpa_alloc(struct cxl_endpoint_decoder * cxled,u64 size)687 int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, u64 size)
688 {
689 struct cxl_port *port = cxled_to_port(cxled);
690 int rc;
691
692 rc = __cxl_dpa_alloc(cxled, size);
693 if (rc)
694 return rc;
695
696 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
697 }
698
cxld_set_interleave(struct cxl_decoder * cxld,u32 * ctrl)699 static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl)
700 {
701 u16 eig;
702 u8 eiw;
703
704 /*
705 * Input validation ensures these warns never fire, but otherwise
706 * suppress unititalized variable usage warnings.
707 */
708 if (WARN_ONCE(ways_to_eiw(cxld->interleave_ways, &eiw),
709 "invalid interleave_ways: %d\n", cxld->interleave_ways))
710 return;
711 if (WARN_ONCE(granularity_to_eig(cxld->interleave_granularity, &eig),
712 "invalid interleave_granularity: %d\n",
713 cxld->interleave_granularity))
714 return;
715
716 u32p_replace_bits(ctrl, eig, CXL_HDM_DECODER0_CTRL_IG_MASK);
717 u32p_replace_bits(ctrl, eiw, CXL_HDM_DECODER0_CTRL_IW_MASK);
718 *ctrl |= CXL_HDM_DECODER0_CTRL_COMMIT;
719 }
720
cxld_set_type(struct cxl_decoder * cxld,u32 * ctrl)721 static void cxld_set_type(struct cxl_decoder *cxld, u32 *ctrl)
722 {
723 u32p_replace_bits(ctrl,
724 !!(cxld->target_type == CXL_DECODER_HOSTONLYMEM),
725 CXL_HDM_DECODER0_CTRL_HOSTONLY);
726 }
727
cxlsd_set_targets(struct cxl_switch_decoder * cxlsd,u64 * tgt)728 static void cxlsd_set_targets(struct cxl_switch_decoder *cxlsd, u64 *tgt)
729 {
730 struct cxl_dport **t = &cxlsd->target[0];
731 int ways = cxlsd->cxld.interleave_ways;
732
733 *tgt = FIELD_PREP(GENMASK(7, 0), t[0]->port_id);
734 if (ways > 1)
735 *tgt |= FIELD_PREP(GENMASK(15, 8), t[1]->port_id);
736 if (ways > 2)
737 *tgt |= FIELD_PREP(GENMASK(23, 16), t[2]->port_id);
738 if (ways > 3)
739 *tgt |= FIELD_PREP(GENMASK(31, 24), t[3]->port_id);
740 if (ways > 4)
741 *tgt |= FIELD_PREP(GENMASK_ULL(39, 32), t[4]->port_id);
742 if (ways > 5)
743 *tgt |= FIELD_PREP(GENMASK_ULL(47, 40), t[5]->port_id);
744 if (ways > 6)
745 *tgt |= FIELD_PREP(GENMASK_ULL(55, 48), t[6]->port_id);
746 if (ways > 7)
747 *tgt |= FIELD_PREP(GENMASK_ULL(63, 56), t[7]->port_id);
748 }
749
750 /*
751 * Per CXL 2.0 8.2.5.12.20 Committing Decoder Programming, hardware must set
752 * committed or error within 10ms, but just be generous with 20ms to account for
753 * clock skew and other marginal behavior
754 */
755 #define COMMIT_TIMEOUT_MS 20
cxld_await_commit(void __iomem * hdm,int id)756 static int cxld_await_commit(void __iomem *hdm, int id)
757 {
758 u32 ctrl;
759 int i;
760
761 for (i = 0; i < COMMIT_TIMEOUT_MS; i++) {
762 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
763 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMIT_ERROR, ctrl)) {
764 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
765 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
766 return -EIO;
767 }
768 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))
769 return 0;
770 fsleep(1000);
771 }
772
773 return -ETIMEDOUT;
774 }
775
setup_hw_decoder(struct cxl_decoder * cxld,void __iomem * hdm)776 static void setup_hw_decoder(struct cxl_decoder *cxld, void __iomem *hdm)
777 {
778 int id = cxld->id;
779 u64 base, size;
780 u32 ctrl;
781
782 /* common decoder settings */
783 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id));
784 cxld_set_interleave(cxld, &ctrl);
785 cxld_set_type(cxld, &ctrl);
786 base = cxld->hpa_range.start;
787 size = range_len(&cxld->hpa_range);
788
789 writel(upper_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
790 writel(lower_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
791 writel(upper_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
792 writel(lower_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
793
794 if (is_switch_decoder(&cxld->dev)) {
795 struct cxl_switch_decoder *cxlsd =
796 to_cxl_switch_decoder(&cxld->dev);
797 void __iomem *tl_hi = hdm + CXL_HDM_DECODER0_TL_HIGH(id);
798 void __iomem *tl_lo = hdm + CXL_HDM_DECODER0_TL_LOW(id);
799 u64 targets;
800
801 cxlsd_set_targets(cxlsd, &targets);
802 writel(upper_32_bits(targets), tl_hi);
803 writel(lower_32_bits(targets), tl_lo);
804 } else {
805 struct cxl_endpoint_decoder *cxled =
806 to_cxl_endpoint_decoder(&cxld->dev);
807 void __iomem *sk_hi = hdm + CXL_HDM_DECODER0_SKIP_HIGH(id);
808 void __iomem *sk_lo = hdm + CXL_HDM_DECODER0_SKIP_LOW(id);
809
810 writel(upper_32_bits(cxled->skip), sk_hi);
811 writel(lower_32_bits(cxled->skip), sk_lo);
812 }
813
814 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
815 }
816
cxl_decoder_commit(struct cxl_decoder * cxld)817 static int cxl_decoder_commit(struct cxl_decoder *cxld)
818 {
819 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
820 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
821 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
822 int id = cxld->id, rc;
823
824 if (cxld->flags & CXL_DECODER_F_ENABLE)
825 return 0;
826
827 if (cxl_num_decoders_committed(port) != id) {
828 dev_dbg(&port->dev,
829 "%s: out of order commit, expected decoder%d.%d\n",
830 dev_name(&cxld->dev), port->id,
831 cxl_num_decoders_committed(port));
832 return -EBUSY;
833 }
834
835 /*
836 * For endpoint decoders hosted on CXL memory devices that
837 * support the sanitize operation, make sure sanitize is not in-flight.
838 */
839 if (is_endpoint_decoder(&cxld->dev)) {
840 struct cxl_endpoint_decoder *cxled =
841 to_cxl_endpoint_decoder(&cxld->dev);
842 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
843 struct cxl_memdev_state *mds =
844 to_cxl_memdev_state(cxlmd->cxlds);
845
846 if (mds && mds->security.sanitize_active) {
847 dev_dbg(&cxlmd->dev,
848 "attempted to commit %s during sanitize\n",
849 dev_name(&cxld->dev));
850 return -EBUSY;
851 }
852 }
853
854 scoped_guard(rwsem_read, &cxl_rwsem.dpa)
855 setup_hw_decoder(cxld, hdm);
856
857 port->commit_end++;
858 rc = cxld_await_commit(hdm, cxld->id);
859 if (rc) {
860 dev_dbg(&port->dev, "%s: error %d committing decoder\n",
861 dev_name(&cxld->dev), rc);
862 cxld->reset(cxld);
863 return rc;
864 }
865 cxld->flags |= CXL_DECODER_F_ENABLE;
866
867 return 0;
868 }
869
commit_reap(struct device * dev,void * data)870 static int commit_reap(struct device *dev, void *data)
871 {
872 struct cxl_port *port = to_cxl_port(dev->parent);
873 struct cxl_decoder *cxld;
874
875 if (!is_switch_decoder(dev) && !is_endpoint_decoder(dev))
876 return 0;
877
878 cxld = to_cxl_decoder(dev);
879 if (port->commit_end == cxld->id &&
880 ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
881 port->commit_end--;
882 dev_dbg(&port->dev, "reap: %s commit_end: %d\n",
883 dev_name(&cxld->dev), port->commit_end);
884 }
885
886 return 0;
887 }
888
cxl_port_commit_reap(struct cxl_decoder * cxld)889 void cxl_port_commit_reap(struct cxl_decoder *cxld)
890 {
891 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
892
893 lockdep_assert_held_write(&cxl_rwsem.region);
894
895 /*
896 * Once the highest committed decoder is disabled, free any other
897 * decoders that were pinned allocated by out-of-order release.
898 */
899 port->commit_end--;
900 dev_dbg(&port->dev, "reap: %s commit_end: %d\n", dev_name(&cxld->dev),
901 port->commit_end);
902 device_for_each_child_reverse_from(&port->dev, &cxld->dev, NULL,
903 commit_reap);
904 }
905 EXPORT_SYMBOL_NS_GPL(cxl_port_commit_reap, "CXL");
906
cxl_decoder_reset(struct cxl_decoder * cxld)907 static void cxl_decoder_reset(struct cxl_decoder *cxld)
908 {
909 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
910 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
911 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
912 int id = cxld->id;
913 u32 ctrl;
914
915 if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
916 return;
917
918 if (port->commit_end == id)
919 cxl_port_commit_reap(cxld);
920 else
921 dev_dbg(&port->dev,
922 "%s: out of order reset, expected decoder%d.%d\n",
923 dev_name(&cxld->dev), port->id, port->commit_end);
924
925 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
926 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;
927 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));
928
929 writel(0, hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));
930 writel(0, hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));
931 writel(0, hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));
932 writel(0, hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));
933
934 cxld->flags &= ~CXL_DECODER_F_ENABLE;
935
936 /* Userspace is now responsible for reconfiguring this decoder */
937 if (is_endpoint_decoder(&cxld->dev)) {
938 struct cxl_endpoint_decoder *cxled;
939
940 cxled = to_cxl_endpoint_decoder(&cxld->dev);
941 cxled->state = CXL_DECODER_STATE_MANUAL;
942 }
943 }
944
cxl_setup_hdm_decoder_from_dvsec(struct cxl_port * port,struct cxl_decoder * cxld,u64 * dpa_base,int which,struct cxl_endpoint_dvsec_info * info)945 static int cxl_setup_hdm_decoder_from_dvsec(
946 struct cxl_port *port, struct cxl_decoder *cxld, u64 *dpa_base,
947 int which, struct cxl_endpoint_dvsec_info *info)
948 {
949 struct cxl_endpoint_decoder *cxled;
950 u64 len;
951 int rc;
952
953 if (!is_cxl_endpoint(port))
954 return -EOPNOTSUPP;
955
956 cxled = to_cxl_endpoint_decoder(&cxld->dev);
957 len = range_len(&info->dvsec_range[which]);
958 if (!len)
959 return -ENOENT;
960
961 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
962 cxld->commit = NULL;
963 cxld->reset = NULL;
964 cxld->hpa_range = info->dvsec_range[which];
965
966 /*
967 * Set the emulated decoder as locked pending additional support to
968 * change the range registers at run time.
969 */
970 cxld->flags |= CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK;
971 port->commit_end = cxld->id;
972
973 rc = devm_cxl_dpa_reserve(cxled, *dpa_base, len, 0);
974 if (rc) {
975 dev_err(&port->dev,
976 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
977 port->id, cxld->id, *dpa_base, *dpa_base + len - 1, rc);
978 return rc;
979 }
980 *dpa_base += len;
981 cxled->state = CXL_DECODER_STATE_AUTO;
982
983 return 0;
984 }
985
init_hdm_decoder(struct cxl_port * port,struct cxl_decoder * cxld,int * target_map,void __iomem * hdm,int which,u64 * dpa_base,struct cxl_endpoint_dvsec_info * info)986 static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
987 int *target_map, void __iomem *hdm, int which,
988 u64 *dpa_base, struct cxl_endpoint_dvsec_info *info)
989 {
990 struct cxl_endpoint_decoder *cxled = NULL;
991 u64 size, base, skip, dpa_size, lo, hi;
992 bool committed;
993 u32 remainder;
994 int i, rc;
995 u32 ctrl;
996 union {
997 u64 value;
998 unsigned char target_id[8];
999 } target_list;
1000
1001 if (should_emulate_decoders(info))
1002 return cxl_setup_hdm_decoder_from_dvsec(port, cxld, dpa_base,
1003 which, info);
1004
1005 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
1006 lo = readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which));
1007 hi = readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(which));
1008 base = (hi << 32) + lo;
1009 lo = readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which));
1010 hi = readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(which));
1011 size = (hi << 32) + lo;
1012 committed = !!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED);
1013 cxld->commit = cxl_decoder_commit;
1014 cxld->reset = cxl_decoder_reset;
1015
1016 if (!committed)
1017 size = 0;
1018 if (base == U64_MAX || size == U64_MAX) {
1019 dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n",
1020 port->id, cxld->id);
1021 return -ENXIO;
1022 }
1023
1024 if (info)
1025 cxled = to_cxl_endpoint_decoder(&cxld->dev);
1026 cxld->hpa_range = (struct range) {
1027 .start = base,
1028 .end = base + size - 1,
1029 };
1030
1031 /* decoders are enabled if committed */
1032 if (committed) {
1033 cxld->flags |= CXL_DECODER_F_ENABLE;
1034 if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK)
1035 cxld->flags |= CXL_DECODER_F_LOCK;
1036 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl))
1037 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
1038 else
1039 cxld->target_type = CXL_DECODER_DEVMEM;
1040
1041 guard(rwsem_write)(&cxl_rwsem.region);
1042 if (cxld->id != cxl_num_decoders_committed(port)) {
1043 dev_warn(&port->dev,
1044 "decoder%d.%d: Committed out of order\n",
1045 port->id, cxld->id);
1046 return -ENXIO;
1047 }
1048
1049 if (size == 0) {
1050 dev_warn(&port->dev,
1051 "decoder%d.%d: Committed with zero size\n",
1052 port->id, cxld->id);
1053 return -ENXIO;
1054 }
1055 port->commit_end = cxld->id;
1056 } else {
1057 if (cxled) {
1058 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1059 struct cxl_dev_state *cxlds = cxlmd->cxlds;
1060
1061 /*
1062 * Default by devtype until a device arrives that needs
1063 * more precision.
1064 */
1065 if (cxlds->type == CXL_DEVTYPE_CLASSMEM)
1066 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
1067 else
1068 cxld->target_type = CXL_DECODER_DEVMEM;
1069 } else {
1070 /* To be overridden by region type at commit time */
1071 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
1072 }
1073
1074 if (!FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl) &&
1075 cxld->target_type == CXL_DECODER_HOSTONLYMEM) {
1076 ctrl |= CXL_HDM_DECODER0_CTRL_HOSTONLY;
1077 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
1078 }
1079 }
1080 rc = eiw_to_ways(FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl),
1081 &cxld->interleave_ways);
1082 if (rc) {
1083 dev_warn(&port->dev,
1084 "decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n",
1085 port->id, cxld->id, ctrl);
1086 return rc;
1087 }
1088 rc = eig_to_granularity(FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl),
1089 &cxld->interleave_granularity);
1090 if (rc) {
1091 dev_warn(&port->dev,
1092 "decoder%d.%d: Invalid interleave granularity (ctrl: %#x)\n",
1093 port->id, cxld->id, ctrl);
1094 return rc;
1095 }
1096
1097 dev_dbg(&port->dev, "decoder%d.%d: range: %#llx-%#llx iw: %d ig: %d\n",
1098 port->id, cxld->id, cxld->hpa_range.start, cxld->hpa_range.end,
1099 cxld->interleave_ways, cxld->interleave_granularity);
1100
1101 if (!cxled) {
1102 lo = readl(hdm + CXL_HDM_DECODER0_TL_LOW(which));
1103 hi = readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which));
1104 target_list.value = (hi << 32) + lo;
1105 for (i = 0; i < cxld->interleave_ways; i++)
1106 target_map[i] = target_list.target_id[i];
1107
1108 return 0;
1109 }
1110
1111 if (!committed)
1112 return 0;
1113
1114 dpa_size = div_u64_rem(size, cxld->interleave_ways, &remainder);
1115 if (remainder) {
1116 dev_err(&port->dev,
1117 "decoder%d.%d: invalid committed configuration size: %#llx ways: %d\n",
1118 port->id, cxld->id, size, cxld->interleave_ways);
1119 return -ENXIO;
1120 }
1121 lo = readl(hdm + CXL_HDM_DECODER0_SKIP_LOW(which));
1122 hi = readl(hdm + CXL_HDM_DECODER0_SKIP_HIGH(which));
1123 skip = (hi << 32) + lo;
1124 rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip);
1125 if (rc) {
1126 dev_err(&port->dev,
1127 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",
1128 port->id, cxld->id, *dpa_base,
1129 *dpa_base + dpa_size + skip - 1, rc);
1130 return rc;
1131 }
1132 *dpa_base += dpa_size + skip;
1133
1134 cxled->state = CXL_DECODER_STATE_AUTO;
1135
1136 return 0;
1137 }
1138
cxl_settle_decoders(struct cxl_hdm * cxlhdm)1139 static void cxl_settle_decoders(struct cxl_hdm *cxlhdm)
1140 {
1141 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
1142 int committed, i;
1143 u32 ctrl;
1144
1145 if (!hdm)
1146 return;
1147
1148 /*
1149 * Since the register resource was recently claimed via request_region()
1150 * be careful about trusting the "not-committed" status until the commit
1151 * timeout has elapsed. The commit timeout is 10ms (CXL 2.0
1152 * 8.2.5.12.20), but double it to be tolerant of any clock skew between
1153 * host and target.
1154 */
1155 for (i = 0, committed = 0; i < cxlhdm->decoder_count; i++) {
1156 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
1157 if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED)
1158 committed++;
1159 }
1160
1161 /* ensure that future checks of committed can be trusted */
1162 if (committed != cxlhdm->decoder_count)
1163 msleep(20);
1164 }
1165
1166 /**
1167 * devm_cxl_enumerate_decoders - add decoder objects per HDM register set
1168 * @cxlhdm: Structure to populate with HDM capabilities
1169 * @info: cached DVSEC range register info
1170 */
devm_cxl_enumerate_decoders(struct cxl_hdm * cxlhdm,struct cxl_endpoint_dvsec_info * info)1171 int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm,
1172 struct cxl_endpoint_dvsec_info *info)
1173 {
1174 void __iomem *hdm = cxlhdm->regs.hdm_decoder;
1175 struct cxl_port *port = cxlhdm->port;
1176 int i;
1177 u64 dpa_base = 0;
1178
1179 cxl_settle_decoders(cxlhdm);
1180
1181 for (i = 0; i < cxlhdm->decoder_count; i++) {
1182 int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 };
1183 int rc, target_count = cxlhdm->target_count;
1184 struct cxl_decoder *cxld;
1185
1186 if (is_cxl_endpoint(port)) {
1187 struct cxl_endpoint_decoder *cxled;
1188
1189 cxled = cxl_endpoint_decoder_alloc(port);
1190 if (IS_ERR(cxled)) {
1191 dev_warn(&port->dev,
1192 "Failed to allocate decoder%d.%d\n",
1193 port->id, i);
1194 return PTR_ERR(cxled);
1195 }
1196 cxld = &cxled->cxld;
1197 } else {
1198 struct cxl_switch_decoder *cxlsd;
1199
1200 cxlsd = cxl_switch_decoder_alloc(port, target_count);
1201 if (IS_ERR(cxlsd)) {
1202 dev_warn(&port->dev,
1203 "Failed to allocate decoder%d.%d\n",
1204 port->id, i);
1205 return PTR_ERR(cxlsd);
1206 }
1207 cxld = &cxlsd->cxld;
1208 }
1209
1210 rc = init_hdm_decoder(port, cxld, target_map, hdm, i,
1211 &dpa_base, info);
1212 if (rc) {
1213 dev_warn(&port->dev,
1214 "Failed to initialize decoder%d.%d\n",
1215 port->id, i);
1216 put_device(&cxld->dev);
1217 return rc;
1218 }
1219 rc = add_hdm_decoder(port, cxld, target_map);
1220 if (rc) {
1221 dev_warn(&port->dev,
1222 "Failed to add decoder%d.%d\n", port->id, i);
1223 return rc;
1224 }
1225 }
1226
1227 return 0;
1228 }
1229 EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, "CXL");
1230