1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2023 Intel Corporation. All rights reserved. */
3 #include <linux/acpi.h>
4 #include <linux/xarray.h>
5 #include <linux/fw_table.h>
6 #include <linux/node.h>
7 #include <linux/overflow.h>
8 #include "cxlpci.h"
9 #include "cxlmem.h"
10 #include "core.h"
11 #include "cxl.h"
12
13 struct dsmas_entry {
14 struct range dpa_range;
15 u8 handle;
16 struct access_coordinate coord[ACCESS_COORDINATE_MAX];
17 struct access_coordinate cdat_coord[ACCESS_COORDINATE_MAX];
18 int entries;
19 int qos_class;
20 };
21
cdat_normalize(u16 entry,u64 base,u8 type)22 static u32 cdat_normalize(u16 entry, u64 base, u8 type)
23 {
24 u32 value;
25
26 /*
27 * Check for invalid and overflow values
28 */
29 if (entry == 0xffff || !entry)
30 return 0;
31 else if (base > (UINT_MAX / (entry)))
32 return 0;
33
34 /*
35 * CDAT fields follow the format of HMAT fields. See table 5 Device
36 * Scoped Latency and Bandwidth Information Structure in Coherent Device
37 * Attribute Table (CDAT) Specification v1.01.
38 */
39 value = entry * base;
40 switch (type) {
41 case ACPI_HMAT_ACCESS_LATENCY:
42 case ACPI_HMAT_READ_LATENCY:
43 case ACPI_HMAT_WRITE_LATENCY:
44 value = DIV_ROUND_UP(value, 1000);
45 break;
46 default:
47 break;
48 }
49 return value;
50 }
51
cdat_dsmas_handler(union acpi_subtable_headers * header,void * arg,const unsigned long end)52 static int cdat_dsmas_handler(union acpi_subtable_headers *header, void *arg,
53 const unsigned long end)
54 {
55 struct acpi_cdat_header *hdr = &header->cdat;
56 struct acpi_cdat_dsmas *dsmas;
57 int size = sizeof(*hdr) + sizeof(*dsmas);
58 struct xarray *dsmas_xa = arg;
59 struct dsmas_entry *dent;
60 u16 len;
61 int rc;
62
63 len = le16_to_cpu((__force __le16)hdr->length);
64 if (len != size || (unsigned long)hdr + len > end) {
65 pr_warn("Malformed DSMAS table length: (%u:%u)\n", size, len);
66 return -EINVAL;
67 }
68
69 /* Skip common header */
70 dsmas = (struct acpi_cdat_dsmas *)(hdr + 1);
71
72 dent = kzalloc(sizeof(*dent), GFP_KERNEL);
73 if (!dent)
74 return -ENOMEM;
75
76 dent->handle = dsmas->dsmad_handle;
77 dent->dpa_range.start = le64_to_cpu((__force __le64)dsmas->dpa_base_address);
78 dent->dpa_range.end = le64_to_cpu((__force __le64)dsmas->dpa_base_address) +
79 le64_to_cpu((__force __le64)dsmas->dpa_length) - 1;
80
81 rc = xa_insert(dsmas_xa, dent->handle, dent, GFP_KERNEL);
82 if (rc) {
83 kfree(dent);
84 return rc;
85 }
86
87 return 0;
88 }
89
__cxl_access_coordinate_set(struct access_coordinate * coord,int access,unsigned int val)90 static void __cxl_access_coordinate_set(struct access_coordinate *coord,
91 int access, unsigned int val)
92 {
93 switch (access) {
94 case ACPI_HMAT_ACCESS_LATENCY:
95 coord->read_latency = val;
96 coord->write_latency = val;
97 break;
98 case ACPI_HMAT_READ_LATENCY:
99 coord->read_latency = val;
100 break;
101 case ACPI_HMAT_WRITE_LATENCY:
102 coord->write_latency = val;
103 break;
104 case ACPI_HMAT_ACCESS_BANDWIDTH:
105 coord->read_bandwidth = val;
106 coord->write_bandwidth = val;
107 break;
108 case ACPI_HMAT_READ_BANDWIDTH:
109 coord->read_bandwidth = val;
110 break;
111 case ACPI_HMAT_WRITE_BANDWIDTH:
112 coord->write_bandwidth = val;
113 break;
114 }
115 }
116
cxl_access_coordinate_set(struct access_coordinate * coord,int access,unsigned int val)117 static void cxl_access_coordinate_set(struct access_coordinate *coord,
118 int access, unsigned int val)
119 {
120 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++)
121 __cxl_access_coordinate_set(&coord[i], access, val);
122 }
123
cdat_dslbis_handler(union acpi_subtable_headers * header,void * arg,const unsigned long end)124 static int cdat_dslbis_handler(union acpi_subtable_headers *header, void *arg,
125 const unsigned long end)
126 {
127 struct acpi_cdat_header *hdr = &header->cdat;
128 struct acpi_cdat_dslbis *dslbis;
129 int size = sizeof(*hdr) + sizeof(*dslbis);
130 struct xarray *dsmas_xa = arg;
131 struct dsmas_entry *dent;
132 __le64 le_base;
133 __le16 le_val;
134 u64 val;
135 u16 len;
136
137 len = le16_to_cpu((__force __le16)hdr->length);
138 if (len != size || (unsigned long)hdr + len > end) {
139 pr_warn("Malformed DSLBIS table length: (%u:%u)\n", size, len);
140 return -EINVAL;
141 }
142
143 /* Skip common header */
144 dslbis = (struct acpi_cdat_dslbis *)(hdr + 1);
145
146 /* Skip unrecognized data type */
147 if (dslbis->data_type > ACPI_HMAT_WRITE_BANDWIDTH)
148 return 0;
149
150 /* Not a memory type, skip */
151 if ((dslbis->flags & ACPI_HMAT_MEMORY_HIERARCHY) != ACPI_HMAT_MEMORY)
152 return 0;
153
154 dent = xa_load(dsmas_xa, dslbis->handle);
155 if (!dent) {
156 pr_warn("No matching DSMAS entry for DSLBIS entry.\n");
157 return 0;
158 }
159
160 le_base = (__force __le64)dslbis->entry_base_unit;
161 le_val = (__force __le16)dslbis->entry[0];
162 val = cdat_normalize(le16_to_cpu(le_val), le64_to_cpu(le_base),
163 dslbis->data_type);
164
165 cxl_access_coordinate_set(dent->cdat_coord, dslbis->data_type, val);
166
167 return 0;
168 }
169
cdat_table_parse_output(int rc)170 static int cdat_table_parse_output(int rc)
171 {
172 if (rc < 0)
173 return rc;
174 if (rc == 0)
175 return -ENOENT;
176
177 return 0;
178 }
179
cxl_cdat_endpoint_process(struct cxl_port * port,struct xarray * dsmas_xa)180 static int cxl_cdat_endpoint_process(struct cxl_port *port,
181 struct xarray *dsmas_xa)
182 {
183 int rc;
184
185 rc = cdat_table_parse(ACPI_CDAT_TYPE_DSMAS, cdat_dsmas_handler,
186 dsmas_xa, port->cdat.table, port->cdat.length);
187 rc = cdat_table_parse_output(rc);
188 if (rc)
189 return rc;
190
191 rc = cdat_table_parse(ACPI_CDAT_TYPE_DSLBIS, cdat_dslbis_handler,
192 dsmas_xa, port->cdat.table, port->cdat.length);
193 return cdat_table_parse_output(rc);
194 }
195
cxl_port_perf_data_calculate(struct cxl_port * port,struct xarray * dsmas_xa)196 static int cxl_port_perf_data_calculate(struct cxl_port *port,
197 struct xarray *dsmas_xa)
198 {
199 struct access_coordinate ep_c[ACCESS_COORDINATE_MAX];
200 struct dsmas_entry *dent;
201 int valid_entries = 0;
202 unsigned long index;
203 int rc;
204
205 rc = cxl_endpoint_get_perf_coordinates(port, ep_c);
206 if (rc) {
207 dev_dbg(&port->dev, "Failed to retrieve ep perf coordinates.\n");
208 return rc;
209 }
210
211 struct cxl_root *cxl_root __free(put_cxl_root) = find_cxl_root(port);
212
213 if (!cxl_root)
214 return -ENODEV;
215
216 if (!cxl_root->ops || !cxl_root->ops->qos_class)
217 return -EOPNOTSUPP;
218
219 xa_for_each(dsmas_xa, index, dent) {
220 int qos_class;
221
222 cxl_coordinates_combine(dent->coord, dent->cdat_coord, ep_c);
223 dent->entries = 1;
224 rc = cxl_root->ops->qos_class(cxl_root,
225 &dent->coord[ACCESS_COORDINATE_CPU],
226 1, &qos_class);
227 if (rc != 1)
228 continue;
229
230 valid_entries++;
231 dent->qos_class = qos_class;
232 }
233
234 if (!valid_entries)
235 return -ENOENT;
236
237 return 0;
238 }
239
update_perf_entry(struct device * dev,struct dsmas_entry * dent,struct cxl_dpa_perf * dpa_perf)240 static void update_perf_entry(struct device *dev, struct dsmas_entry *dent,
241 struct cxl_dpa_perf *dpa_perf)
242 {
243 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
244 dpa_perf->coord[i] = dent->coord[i];
245 dpa_perf->cdat_coord[i] = dent->cdat_coord[i];
246 }
247 dpa_perf->dpa_range = dent->dpa_range;
248 dpa_perf->qos_class = dent->qos_class;
249 dev_dbg(dev,
250 "DSMAS: dpa: %pra qos: %d read_bw: %d write_bw %d read_lat: %d write_lat: %d\n",
251 &dent->dpa_range, dpa_perf->qos_class,
252 dent->coord[ACCESS_COORDINATE_CPU].read_bandwidth,
253 dent->coord[ACCESS_COORDINATE_CPU].write_bandwidth,
254 dent->coord[ACCESS_COORDINATE_CPU].read_latency,
255 dent->coord[ACCESS_COORDINATE_CPU].write_latency);
256 }
257
cxl_memdev_set_qos_class(struct cxl_dev_state * cxlds,struct xarray * dsmas_xa)258 static void cxl_memdev_set_qos_class(struct cxl_dev_state *cxlds,
259 struct xarray *dsmas_xa)
260 {
261 struct device *dev = cxlds->dev;
262 struct dsmas_entry *dent;
263 unsigned long index;
264
265 xa_for_each(dsmas_xa, index, dent) {
266 bool found = false;
267
268 for (int i = 0; i < cxlds->nr_partitions; i++) {
269 struct resource *res = &cxlds->part[i].res;
270 struct range range = {
271 .start = res->start,
272 .end = res->end,
273 };
274
275 if (range_contains(&range, &dent->dpa_range)) {
276 update_perf_entry(dev, dent,
277 &cxlds->part[i].perf);
278 found = true;
279 break;
280 }
281 }
282
283 if (!found)
284 dev_dbg(dev, "no partition for dsmas dpa: %pra\n",
285 &dent->dpa_range);
286 }
287 }
288
match_cxlrd_qos_class(struct device * dev,void * data)289 static int match_cxlrd_qos_class(struct device *dev, void *data)
290 {
291 int dev_qos_class = *(int *)data;
292 struct cxl_root_decoder *cxlrd;
293
294 if (!is_root_decoder(dev))
295 return 0;
296
297 cxlrd = to_cxl_root_decoder(dev);
298 if (cxlrd->qos_class == CXL_QOS_CLASS_INVALID)
299 return 0;
300
301 if (cxlrd->qos_class == dev_qos_class)
302 return 1;
303
304 return 0;
305 }
306
reset_dpa_perf(struct cxl_dpa_perf * dpa_perf)307 static void reset_dpa_perf(struct cxl_dpa_perf *dpa_perf)
308 {
309 *dpa_perf = (struct cxl_dpa_perf) {
310 .qos_class = CXL_QOS_CLASS_INVALID,
311 };
312 }
313
cxl_qos_match(struct cxl_port * root_port,struct cxl_dpa_perf * dpa_perf)314 static bool cxl_qos_match(struct cxl_port *root_port,
315 struct cxl_dpa_perf *dpa_perf)
316 {
317 if (dpa_perf->qos_class == CXL_QOS_CLASS_INVALID)
318 return false;
319
320 if (!device_for_each_child(&root_port->dev, &dpa_perf->qos_class,
321 match_cxlrd_qos_class))
322 return false;
323
324 return true;
325 }
326
match_cxlrd_hb(struct device * dev,void * data)327 static int match_cxlrd_hb(struct device *dev, void *data)
328 {
329 struct device *host_bridge = data;
330 struct cxl_switch_decoder *cxlsd;
331 struct cxl_root_decoder *cxlrd;
332
333 if (!is_root_decoder(dev))
334 return 0;
335
336 cxlrd = to_cxl_root_decoder(dev);
337 cxlsd = &cxlrd->cxlsd;
338
339 guard(rwsem_read)(&cxl_region_rwsem);
340 for (int i = 0; i < cxlsd->nr_targets; i++) {
341 if (host_bridge == cxlsd->target[i]->dport_dev)
342 return 1;
343 }
344
345 return 0;
346 }
347
cxl_qos_class_verify(struct cxl_memdev * cxlmd)348 static void cxl_qos_class_verify(struct cxl_memdev *cxlmd)
349 {
350 struct cxl_dev_state *cxlds = cxlmd->cxlds;
351 struct cxl_port *root_port;
352
353 struct cxl_root *cxl_root __free(put_cxl_root) =
354 find_cxl_root(cxlmd->endpoint);
355
356 /*
357 * No need to reset_dpa_perf() here as find_cxl_root() is guaranteed to
358 * succeed when called in the cxl_endpoint_port_probe() path.
359 */
360 if (!cxl_root)
361 return;
362
363 root_port = &cxl_root->port;
364
365 /*
366 * Save userspace from needing to check if a qos class has any matches
367 * by hiding qos class info if the memdev is not mapped by a root
368 * decoder, or the partition class does not match any root decoder
369 * class.
370 */
371 if (!device_for_each_child(&root_port->dev,
372 cxlmd->endpoint->host_bridge,
373 match_cxlrd_hb)) {
374 for (int i = 0; i < cxlds->nr_partitions; i++) {
375 struct cxl_dpa_perf *perf = &cxlds->part[i].perf;
376
377 reset_dpa_perf(perf);
378 }
379 return;
380 }
381
382 for (int i = 0; i < cxlds->nr_partitions; i++) {
383 struct cxl_dpa_perf *perf = &cxlds->part[i].perf;
384
385 if (!cxl_qos_match(root_port, perf))
386 reset_dpa_perf(perf);
387 }
388 }
389
discard_dsmas(struct xarray * xa)390 static void discard_dsmas(struct xarray *xa)
391 {
392 unsigned long index;
393 void *ent;
394
395 xa_for_each(xa, index, ent) {
396 xa_erase(xa, index);
397 kfree(ent);
398 }
399 xa_destroy(xa);
400 }
DEFINE_FREE(dsmas,struct xarray *,if (_T)discard_dsmas (_T))401 DEFINE_FREE(dsmas, struct xarray *, if (_T) discard_dsmas(_T))
402
403 void cxl_endpoint_parse_cdat(struct cxl_port *port)
404 {
405 struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev);
406 struct cxl_dev_state *cxlds = cxlmd->cxlds;
407 struct xarray __dsmas_xa;
408 struct xarray *dsmas_xa __free(dsmas) = &__dsmas_xa;
409 int rc;
410
411 xa_init(&__dsmas_xa);
412 if (!port->cdat.table)
413 return;
414
415 rc = cxl_cdat_endpoint_process(port, dsmas_xa);
416 if (rc < 0) {
417 dev_dbg(&port->dev, "Failed to parse CDAT: %d\n", rc);
418 return;
419 }
420
421 rc = cxl_port_perf_data_calculate(port, dsmas_xa);
422 if (rc) {
423 dev_dbg(&port->dev, "Failed to do perf coord calculations.\n");
424 return;
425 }
426
427 cxl_memdev_set_qos_class(cxlds, dsmas_xa);
428 cxl_qos_class_verify(cxlmd);
429 cxl_memdev_update_perf(cxlmd);
430 }
431 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_parse_cdat, "CXL");
432
cdat_sslbis_handler(union acpi_subtable_headers * header,void * arg,const unsigned long end)433 static int cdat_sslbis_handler(union acpi_subtable_headers *header, void *arg,
434 const unsigned long end)
435 {
436 struct acpi_cdat_sslbis_table {
437 struct acpi_cdat_header header;
438 struct acpi_cdat_sslbis sslbis_header;
439 struct acpi_cdat_sslbe entries[];
440 } *tbl = (struct acpi_cdat_sslbis_table *)header;
441 int size = sizeof(header->cdat) + sizeof(tbl->sslbis_header);
442 struct acpi_cdat_sslbis *sslbis;
443 struct cxl_port *port = arg;
444 struct device *dev = &port->dev;
445 int remain, entries, i;
446 u16 len;
447
448 len = le16_to_cpu((__force __le16)header->cdat.length);
449 remain = len - size;
450 if (!remain || remain % sizeof(tbl->entries[0]) ||
451 (unsigned long)header + len > end) {
452 dev_warn(dev, "Malformed SSLBIS table length: (%u)\n", len);
453 return -EINVAL;
454 }
455
456 sslbis = &tbl->sslbis_header;
457 /* Unrecognized data type, we can skip */
458 if (sslbis->data_type > ACPI_HMAT_WRITE_BANDWIDTH)
459 return 0;
460
461 entries = remain / sizeof(tbl->entries[0]);
462 if (struct_size(tbl, entries, entries) != len)
463 return -EINVAL;
464
465 for (i = 0; i < entries; i++) {
466 u16 x = le16_to_cpu((__force __le16)tbl->entries[i].portx_id);
467 u16 y = le16_to_cpu((__force __le16)tbl->entries[i].porty_id);
468 __le64 le_base;
469 __le16 le_val;
470 struct cxl_dport *dport;
471 unsigned long index;
472 u16 dsp_id;
473 u64 val;
474
475 switch (x) {
476 case ACPI_CDAT_SSLBIS_US_PORT:
477 dsp_id = y;
478 break;
479 case ACPI_CDAT_SSLBIS_ANY_PORT:
480 switch (y) {
481 case ACPI_CDAT_SSLBIS_US_PORT:
482 dsp_id = x;
483 break;
484 case ACPI_CDAT_SSLBIS_ANY_PORT:
485 dsp_id = ACPI_CDAT_SSLBIS_ANY_PORT;
486 break;
487 default:
488 dsp_id = y;
489 break;
490 }
491 break;
492 default:
493 dsp_id = x;
494 break;
495 }
496
497 le_base = (__force __le64)tbl->sslbis_header.entry_base_unit;
498 le_val = (__force __le16)tbl->entries[i].latency_or_bandwidth;
499 val = cdat_normalize(le16_to_cpu(le_val), le64_to_cpu(le_base),
500 sslbis->data_type);
501
502 xa_for_each(&port->dports, index, dport) {
503 if (dsp_id == ACPI_CDAT_SSLBIS_ANY_PORT ||
504 dsp_id == dport->port_id) {
505 cxl_access_coordinate_set(dport->coord,
506 sslbis->data_type,
507 val);
508 }
509 }
510 }
511
512 return 0;
513 }
514
cxl_switch_parse_cdat(struct cxl_port * port)515 void cxl_switch_parse_cdat(struct cxl_port *port)
516 {
517 int rc;
518
519 if (!port->cdat.table)
520 return;
521
522 rc = cdat_table_parse(ACPI_CDAT_TYPE_SSLBIS, cdat_sslbis_handler,
523 port, port->cdat.table, port->cdat.length);
524 rc = cdat_table_parse_output(rc);
525 if (rc)
526 dev_dbg(&port->dev, "Failed to parse SSLBIS: %d\n", rc);
527 }
528 EXPORT_SYMBOL_NS_GPL(cxl_switch_parse_cdat, "CXL");
529
__cxl_coordinates_combine(struct access_coordinate * out,struct access_coordinate * c1,struct access_coordinate * c2)530 static void __cxl_coordinates_combine(struct access_coordinate *out,
531 struct access_coordinate *c1,
532 struct access_coordinate *c2)
533 {
534 if (c1->write_bandwidth && c2->write_bandwidth)
535 out->write_bandwidth = min(c1->write_bandwidth,
536 c2->write_bandwidth);
537 out->write_latency = c1->write_latency + c2->write_latency;
538
539 if (c1->read_bandwidth && c2->read_bandwidth)
540 out->read_bandwidth = min(c1->read_bandwidth,
541 c2->read_bandwidth);
542 out->read_latency = c1->read_latency + c2->read_latency;
543 }
544
545 /**
546 * cxl_coordinates_combine - Combine the two input coordinates
547 *
548 * @out: Output coordinate of c1 and c2 combined
549 * @c1: input coordinates
550 * @c2: input coordinates
551 */
cxl_coordinates_combine(struct access_coordinate * out,struct access_coordinate * c1,struct access_coordinate * c2)552 void cxl_coordinates_combine(struct access_coordinate *out,
553 struct access_coordinate *c1,
554 struct access_coordinate *c2)
555 {
556 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++)
557 __cxl_coordinates_combine(&out[i], &c1[i], &c2[i]);
558 }
559
560 MODULE_IMPORT_NS("CXL");
561
cxl_bandwidth_add(struct access_coordinate * coord,struct access_coordinate * c1,struct access_coordinate * c2)562 static void cxl_bandwidth_add(struct access_coordinate *coord,
563 struct access_coordinate *c1,
564 struct access_coordinate *c2)
565 {
566 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
567 coord[i].read_bandwidth = c1[i].read_bandwidth +
568 c2[i].read_bandwidth;
569 coord[i].write_bandwidth = c1[i].write_bandwidth +
570 c2[i].write_bandwidth;
571 }
572 }
573
dpa_perf_contains(struct cxl_dpa_perf * perf,struct resource * dpa_res)574 static bool dpa_perf_contains(struct cxl_dpa_perf *perf,
575 struct resource *dpa_res)
576 {
577 struct range dpa = {
578 .start = dpa_res->start,
579 .end = dpa_res->end,
580 };
581
582 return range_contains(&perf->dpa_range, &dpa);
583 }
584
cxled_get_dpa_perf(struct cxl_endpoint_decoder * cxled)585 static struct cxl_dpa_perf *cxled_get_dpa_perf(struct cxl_endpoint_decoder *cxled)
586 {
587 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
588 struct cxl_dev_state *cxlds = cxlmd->cxlds;
589 struct cxl_dpa_perf *perf;
590
591 if (cxled->part < 0)
592 return ERR_PTR(-EINVAL);
593 perf = &cxlds->part[cxled->part].perf;
594
595 if (!perf)
596 return ERR_PTR(-EINVAL);
597
598 if (!dpa_perf_contains(perf, cxled->dpa_res))
599 return ERR_PTR(-EINVAL);
600
601 return perf;
602 }
603
604 /*
605 * Transient context for containing the current calculation of bandwidth when
606 * doing walking the port hierarchy to deal with shared upstream link.
607 */
608 struct cxl_perf_ctx {
609 struct access_coordinate coord[ACCESS_COORDINATE_MAX];
610 struct cxl_port *port;
611 };
612
613 /**
614 * cxl_endpoint_gather_bandwidth - collect all the endpoint bandwidth in an xarray
615 * @cxlr: CXL region for the bandwidth calculation
616 * @cxled: endpoint decoder to start on
617 * @usp_xa: (output) the xarray that collects all the bandwidth coordinates
618 * indexed by the upstream device with data of 'struct cxl_perf_ctx'.
619 * @gp_is_root: (output) bool of whether the grandparent is cxl root.
620 *
621 * Return: 0 for success or -errno
622 *
623 * Collects aggregated endpoint bandwidth and store the bandwidth in
624 * an xarray indexed by the upstream device of the switch or the RP
625 * device. Each endpoint consists the minimum of the bandwidth from DSLBIS
626 * from the endpoint CDAT, the endpoint upstream link bandwidth, and the
627 * bandwidth from the SSLBIS of the switch CDAT for the switch upstream port to
628 * the downstream port that's associated with the endpoint. If the
629 * device is directly connected to a RP, then no SSLBIS is involved.
630 */
cxl_endpoint_gather_bandwidth(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,struct xarray * usp_xa,bool * gp_is_root)631 static int cxl_endpoint_gather_bandwidth(struct cxl_region *cxlr,
632 struct cxl_endpoint_decoder *cxled,
633 struct xarray *usp_xa,
634 bool *gp_is_root)
635 {
636 struct cxl_port *endpoint = to_cxl_port(cxled->cxld.dev.parent);
637 struct cxl_port *parent_port = to_cxl_port(endpoint->dev.parent);
638 struct cxl_port *gp_port = to_cxl_port(parent_port->dev.parent);
639 struct access_coordinate pci_coord[ACCESS_COORDINATE_MAX];
640 struct access_coordinate sw_coord[ACCESS_COORDINATE_MAX];
641 struct access_coordinate ep_coord[ACCESS_COORDINATE_MAX];
642 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
643 struct cxl_dev_state *cxlds = cxlmd->cxlds;
644 struct pci_dev *pdev = to_pci_dev(cxlds->dev);
645 struct cxl_perf_ctx *perf_ctx;
646 struct cxl_dpa_perf *perf;
647 unsigned long index;
648 void *ptr;
649 int rc;
650
651 if (!dev_is_pci(cxlds->dev))
652 return -ENODEV;
653
654 if (cxlds->rcd)
655 return -ENODEV;
656
657 perf = cxled_get_dpa_perf(cxled);
658 if (IS_ERR(perf))
659 return PTR_ERR(perf);
660
661 *gp_is_root = is_cxl_root(gp_port);
662
663 /*
664 * If the grandparent is cxl root, then index is the root port,
665 * otherwise it's the parent switch upstream device.
666 */
667 if (*gp_is_root)
668 index = (unsigned long)endpoint->parent_dport->dport_dev;
669 else
670 index = (unsigned long)parent_port->uport_dev;
671
672 perf_ctx = xa_load(usp_xa, index);
673 if (!perf_ctx) {
674 struct cxl_perf_ctx *c __free(kfree) =
675 kzalloc(sizeof(*perf_ctx), GFP_KERNEL);
676
677 if (!c)
678 return -ENOMEM;
679 ptr = xa_store(usp_xa, index, c, GFP_KERNEL);
680 if (xa_is_err(ptr))
681 return xa_err(ptr);
682 perf_ctx = no_free_ptr(c);
683 perf_ctx->port = parent_port;
684 }
685
686 /* Direct upstream link from EP bandwidth */
687 rc = cxl_pci_get_bandwidth(pdev, pci_coord);
688 if (rc < 0)
689 return rc;
690
691 /*
692 * Min of upstream link bandwidth and Endpoint CDAT bandwidth from
693 * DSLBIS.
694 */
695 cxl_coordinates_combine(ep_coord, pci_coord, perf->cdat_coord);
696
697 /*
698 * If grandparent port is root, then there's no switch involved and
699 * the endpoint is connected to a root port.
700 */
701 if (!*gp_is_root) {
702 /*
703 * Retrieve the switch SSLBIS for switch downstream port
704 * associated with the endpoint bandwidth.
705 */
706 rc = cxl_port_get_switch_dport_bandwidth(endpoint, sw_coord);
707 if (rc)
708 return rc;
709
710 /*
711 * Min of the earlier coordinates with the switch SSLBIS
712 * bandwidth
713 */
714 cxl_coordinates_combine(ep_coord, ep_coord, sw_coord);
715 }
716
717 /*
718 * Aggregate the computed bandwidth with the current aggregated bandwidth
719 * of the endpoints with the same switch upstream device or RP.
720 */
721 cxl_bandwidth_add(perf_ctx->coord, perf_ctx->coord, ep_coord);
722
723 return 0;
724 }
725
free_perf_xa(struct xarray * xa)726 static void free_perf_xa(struct xarray *xa)
727 {
728 struct cxl_perf_ctx *ctx;
729 unsigned long index;
730
731 if (!xa)
732 return;
733
734 xa_for_each(xa, index, ctx)
735 kfree(ctx);
736 xa_destroy(xa);
737 kfree(xa);
738 }
DEFINE_FREE(free_perf_xa,struct xarray *,if (_T)free_perf_xa (_T))739 DEFINE_FREE(free_perf_xa, struct xarray *, if (_T) free_perf_xa(_T))
740
741 /**
742 * cxl_switch_gather_bandwidth - collect all the bandwidth at switch level in an xarray
743 * @cxlr: The region being operated on
744 * @input_xa: xarray indexed by upstream device of a switch with data of 'struct
745 * cxl_perf_ctx'
746 * @gp_is_root: (output) bool of whether the grandparent is cxl root.
747 *
748 * Return: a xarray of resulting cxl_perf_ctx per parent switch or root port
749 * or ERR_PTR(-errno)
750 *
751 * Iterate through the xarray. Take the minimum of the downstream calculated
752 * bandwidth, the upstream link bandwidth, and the SSLBIS of the upstream
753 * switch if exists. Sum the resulting bandwidth under the switch upstream
754 * device or a RP device. The function can be iterated over multiple switches
755 * if the switches are present.
756 */
757 static struct xarray *cxl_switch_gather_bandwidth(struct cxl_region *cxlr,
758 struct xarray *input_xa,
759 bool *gp_is_root)
760 {
761 struct xarray *res_xa __free(free_perf_xa) =
762 kzalloc(sizeof(*res_xa), GFP_KERNEL);
763 struct access_coordinate coords[ACCESS_COORDINATE_MAX];
764 struct cxl_perf_ctx *ctx, *us_ctx;
765 unsigned long index, us_index;
766 int dev_count = 0;
767 int gp_count = 0;
768 void *ptr;
769 int rc;
770
771 if (!res_xa)
772 return ERR_PTR(-ENOMEM);
773 xa_init(res_xa);
774
775 xa_for_each(input_xa, index, ctx) {
776 struct device *dev = (struct device *)index;
777 struct cxl_port *port = ctx->port;
778 struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
779 struct cxl_port *gp_port = to_cxl_port(parent_port->dev.parent);
780 struct cxl_dport *dport = port->parent_dport;
781 bool is_root = false;
782
783 dev_count++;
784 if (is_cxl_root(gp_port)) {
785 is_root = true;
786 gp_count++;
787 }
788
789 /*
790 * If the grandparent is cxl root, then index is the root port,
791 * otherwise it's the parent switch upstream device.
792 */
793 if (is_root)
794 us_index = (unsigned long)port->parent_dport->dport_dev;
795 else
796 us_index = (unsigned long)parent_port->uport_dev;
797
798 us_ctx = xa_load(res_xa, us_index);
799 if (!us_ctx) {
800 struct cxl_perf_ctx *n __free(kfree) =
801 kzalloc(sizeof(*n), GFP_KERNEL);
802
803 if (!n)
804 return ERR_PTR(-ENOMEM);
805
806 ptr = xa_store(res_xa, us_index, n, GFP_KERNEL);
807 if (xa_is_err(ptr))
808 return ERR_PTR(xa_err(ptr));
809 us_ctx = no_free_ptr(n);
810 us_ctx->port = parent_port;
811 }
812
813 /*
814 * If the device isn't an upstream PCIe port, there's something
815 * wrong with the topology.
816 */
817 if (!dev_is_pci(dev))
818 return ERR_PTR(-EINVAL);
819
820 /* Retrieve the upstream link bandwidth */
821 rc = cxl_pci_get_bandwidth(to_pci_dev(dev), coords);
822 if (rc)
823 return ERR_PTR(-ENXIO);
824
825 /*
826 * Take the min of downstream bandwidth and the upstream link
827 * bandwidth.
828 */
829 cxl_coordinates_combine(coords, coords, ctx->coord);
830
831 /*
832 * Take the min of the calculated bandwdith and the upstream
833 * switch SSLBIS bandwidth if there's a parent switch
834 */
835 if (!is_root)
836 cxl_coordinates_combine(coords, coords, dport->coord);
837
838 /*
839 * Aggregate the calculated bandwidth common to an upstream
840 * switch.
841 */
842 cxl_bandwidth_add(us_ctx->coord, us_ctx->coord, coords);
843 }
844
845 /* Asymmetric topology detected. */
846 if (gp_count) {
847 if (gp_count != dev_count) {
848 dev_dbg(&cxlr->dev,
849 "Asymmetric hierarchy detected, bandwidth not updated\n");
850 return ERR_PTR(-EOPNOTSUPP);
851 }
852 *gp_is_root = true;
853 }
854
855 return no_free_ptr(res_xa);
856 }
857
858 /**
859 * cxl_rp_gather_bandwidth - handle the root port level bandwidth collection
860 * @xa: the xarray that holds the cxl_perf_ctx that has the bandwidth calculated
861 * below each root port device.
862 *
863 * Return: xarray that holds cxl_perf_ctx per host bridge or ERR_PTR(-errno)
864 */
cxl_rp_gather_bandwidth(struct xarray * xa)865 static struct xarray *cxl_rp_gather_bandwidth(struct xarray *xa)
866 {
867 struct xarray *hb_xa __free(free_perf_xa) =
868 kzalloc(sizeof(*hb_xa), GFP_KERNEL);
869 struct cxl_perf_ctx *ctx;
870 unsigned long index;
871
872 if (!hb_xa)
873 return ERR_PTR(-ENOMEM);
874 xa_init(hb_xa);
875
876 xa_for_each(xa, index, ctx) {
877 struct cxl_port *port = ctx->port;
878 unsigned long hb_index = (unsigned long)port->uport_dev;
879 struct cxl_perf_ctx *hb_ctx;
880 void *ptr;
881
882 hb_ctx = xa_load(hb_xa, hb_index);
883 if (!hb_ctx) {
884 struct cxl_perf_ctx *n __free(kfree) =
885 kzalloc(sizeof(*n), GFP_KERNEL);
886
887 if (!n)
888 return ERR_PTR(-ENOMEM);
889 ptr = xa_store(hb_xa, hb_index, n, GFP_KERNEL);
890 if (xa_is_err(ptr))
891 return ERR_PTR(xa_err(ptr));
892 hb_ctx = no_free_ptr(n);
893 hb_ctx->port = port;
894 }
895
896 cxl_bandwidth_add(hb_ctx->coord, hb_ctx->coord, ctx->coord);
897 }
898
899 return no_free_ptr(hb_xa);
900 }
901
902 /**
903 * cxl_hb_gather_bandwidth - handle the host bridge level bandwidth collection
904 * @xa: the xarray that holds the cxl_perf_ctx that has the bandwidth calculated
905 * below each host bridge.
906 *
907 * Return: xarray that holds cxl_perf_ctx per ACPI0017 device or ERR_PTR(-errno)
908 */
cxl_hb_gather_bandwidth(struct xarray * xa)909 static struct xarray *cxl_hb_gather_bandwidth(struct xarray *xa)
910 {
911 struct xarray *mw_xa __free(free_perf_xa) =
912 kzalloc(sizeof(*mw_xa), GFP_KERNEL);
913 struct cxl_perf_ctx *ctx;
914 unsigned long index;
915
916 if (!mw_xa)
917 return ERR_PTR(-ENOMEM);
918 xa_init(mw_xa);
919
920 xa_for_each(xa, index, ctx) {
921 struct cxl_port *port = ctx->port;
922 struct cxl_port *parent_port;
923 struct cxl_perf_ctx *mw_ctx;
924 struct cxl_dport *dport;
925 unsigned long mw_index;
926 void *ptr;
927
928 parent_port = to_cxl_port(port->dev.parent);
929 mw_index = (unsigned long)parent_port->uport_dev;
930
931 mw_ctx = xa_load(mw_xa, mw_index);
932 if (!mw_ctx) {
933 struct cxl_perf_ctx *n __free(kfree) =
934 kzalloc(sizeof(*n), GFP_KERNEL);
935
936 if (!n)
937 return ERR_PTR(-ENOMEM);
938 ptr = xa_store(mw_xa, mw_index, n, GFP_KERNEL);
939 if (xa_is_err(ptr))
940 return ERR_PTR(xa_err(ptr));
941 mw_ctx = no_free_ptr(n);
942 }
943
944 dport = port->parent_dport;
945 cxl_coordinates_combine(ctx->coord, ctx->coord, dport->coord);
946 cxl_bandwidth_add(mw_ctx->coord, mw_ctx->coord, ctx->coord);
947 }
948
949 return no_free_ptr(mw_xa);
950 }
951
952 /**
953 * cxl_region_update_bandwidth - Update the bandwidth access coordinates of a region
954 * @cxlr: The region being operated on
955 * @input_xa: xarray holds cxl_perf_ctx wht calculated bandwidth per ACPI0017 instance
956 */
cxl_region_update_bandwidth(struct cxl_region * cxlr,struct xarray * input_xa)957 static void cxl_region_update_bandwidth(struct cxl_region *cxlr,
958 struct xarray *input_xa)
959 {
960 struct access_coordinate coord[ACCESS_COORDINATE_MAX];
961 struct cxl_perf_ctx *ctx;
962 unsigned long index;
963
964 memset(coord, 0, sizeof(coord));
965 xa_for_each(input_xa, index, ctx)
966 cxl_bandwidth_add(coord, coord, ctx->coord);
967
968 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
969 cxlr->coord[i].read_bandwidth = coord[i].read_bandwidth;
970 cxlr->coord[i].write_bandwidth = coord[i].write_bandwidth;
971 }
972 }
973
974 /**
975 * cxl_region_shared_upstream_bandwidth_update - Recalculate the bandwidth for
976 * the region
977 * @cxlr: the cxl region to recalculate
978 *
979 * The function walks the topology from bottom up and calculates the bandwidth. It
980 * starts at the endpoints, processes at the switches if any, processes at the rootport
981 * level, at the host bridge level, and finally aggregates at the region.
982 */
cxl_region_shared_upstream_bandwidth_update(struct cxl_region * cxlr)983 void cxl_region_shared_upstream_bandwidth_update(struct cxl_region *cxlr)
984 {
985 struct xarray *working_xa;
986 int root_count = 0;
987 bool is_root;
988 int rc;
989
990 lockdep_assert_held(&cxl_dpa_rwsem);
991
992 struct xarray *usp_xa __free(free_perf_xa) =
993 kzalloc(sizeof(*usp_xa), GFP_KERNEL);
994
995 if (!usp_xa)
996 return;
997
998 xa_init(usp_xa);
999
1000 /* Collect bandwidth data from all the endpoints. */
1001 for (int i = 0; i < cxlr->params.nr_targets; i++) {
1002 struct cxl_endpoint_decoder *cxled = cxlr->params.targets[i];
1003
1004 is_root = false;
1005 rc = cxl_endpoint_gather_bandwidth(cxlr, cxled, usp_xa, &is_root);
1006 if (rc)
1007 return;
1008 root_count += is_root;
1009 }
1010
1011 /* Detect asymmetric hierarchy with some direct attached endpoints. */
1012 if (root_count && root_count != cxlr->params.nr_targets) {
1013 dev_dbg(&cxlr->dev,
1014 "Asymmetric hierarchy detected, bandwidth not updated\n");
1015 return;
1016 }
1017
1018 /*
1019 * Walk up one or more switches to deal with the bandwidth of the
1020 * switches if they exist. Endpoints directly attached to RPs skip
1021 * over this part.
1022 */
1023 if (!root_count) {
1024 do {
1025 working_xa = cxl_switch_gather_bandwidth(cxlr, usp_xa,
1026 &is_root);
1027 if (IS_ERR(working_xa))
1028 return;
1029 free_perf_xa(usp_xa);
1030 usp_xa = working_xa;
1031 } while (!is_root);
1032 }
1033
1034 /* Handle the bandwidth at the root port of the hierarchy */
1035 working_xa = cxl_rp_gather_bandwidth(usp_xa);
1036 if (IS_ERR(working_xa))
1037 return;
1038 free_perf_xa(usp_xa);
1039 usp_xa = working_xa;
1040
1041 /* Handle the bandwidth at the host bridge of the hierarchy */
1042 working_xa = cxl_hb_gather_bandwidth(usp_xa);
1043 if (IS_ERR(working_xa))
1044 return;
1045 free_perf_xa(usp_xa);
1046 usp_xa = working_xa;
1047
1048 /*
1049 * Aggregate all the bandwidth collected per CFMWS (ACPI0017) and
1050 * update the region bandwidth with the final calculated values.
1051 */
1052 cxl_region_update_bandwidth(cxlr, usp_xa);
1053 }
1054
cxl_region_perf_data_calculate(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled)1055 void cxl_region_perf_data_calculate(struct cxl_region *cxlr,
1056 struct cxl_endpoint_decoder *cxled)
1057 {
1058 struct cxl_dpa_perf *perf;
1059
1060 lockdep_assert_held(&cxl_dpa_rwsem);
1061
1062 perf = cxled_get_dpa_perf(cxled);
1063 if (IS_ERR(perf))
1064 return;
1065
1066 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
1067 /* Get total bandwidth and the worst latency for the cxl region */
1068 cxlr->coord[i].read_latency = max_t(unsigned int,
1069 cxlr->coord[i].read_latency,
1070 perf->coord[i].read_latency);
1071 cxlr->coord[i].write_latency = max_t(unsigned int,
1072 cxlr->coord[i].write_latency,
1073 perf->coord[i].write_latency);
1074 cxlr->coord[i].read_bandwidth += perf->coord[i].read_bandwidth;
1075 cxlr->coord[i].write_bandwidth += perf->coord[i].write_bandwidth;
1076 }
1077 }
1078
cxl_update_hmat_access_coordinates(int nid,struct cxl_region * cxlr,enum access_coordinate_class access)1079 int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr,
1080 enum access_coordinate_class access)
1081 {
1082 return hmat_update_target_coordinates(nid, &cxlr->coord[access], access);
1083 }
1084
cxl_need_node_perf_attrs_update(int nid)1085 bool cxl_need_node_perf_attrs_update(int nid)
1086 {
1087 return !acpi_node_backed_by_real_pxm(nid);
1088 }
1089