1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2019, Intel Corporation.
4  *
5  * Heterogeneous Memory Attributes Table (HMAT) representation
6  *
7  * This program parses and reports the platform's HMAT tables, and registers
8  * the applicable attributes with the node's interfaces.
9  */
10 
11 #define pr_fmt(fmt) "acpi/hmat: " fmt
12 
13 #include <linux/acpi.h>
14 #include <linux/bitops.h>
15 #include <linux/device.h>
16 #include <linux/init.h>
17 #include <linux/list.h>
18 #include <linux/mm.h>
19 #include <linux/platform_device.h>
20 #include <linux/list_sort.h>
21 #include <linux/memregion.h>
22 #include <linux/memory.h>
23 #include <linux/mutex.h>
24 #include <linux/node.h>
25 #include <linux/sysfs.h>
26 #include <linux/dax.h>
27 #include <linux/memory-tiers.h>
28 
29 static u8 hmat_revision;
30 static int hmat_disable __initdata;
31 
32 void __init disable_hmat(void)
33 {
34 	hmat_disable = 1;
35 }
36 
37 static LIST_HEAD(targets);
38 static LIST_HEAD(initiators);
39 static LIST_HEAD(localities);
40 
41 static DEFINE_MUTEX(target_lock);
42 
43 /*
44  * The defined enum order is used to prioritize attributes to break ties when
45  * selecting the best performing node.
46  */
47 enum locality_types {
48 	WRITE_LATENCY,
49 	READ_LATENCY,
50 	WRITE_BANDWIDTH,
51 	READ_BANDWIDTH,
52 };
53 
54 static struct memory_locality *localities_types[4];
55 
56 struct target_cache {
57 	struct list_head node;
58 	struct node_cache_attrs cache_attrs;
59 };
60 
61 enum {
62 	NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL = ACCESS_COORDINATE_MAX,
63 	NODE_ACCESS_CLASS_GENPORT_SINK_CPU,
64 	NODE_ACCESS_CLASS_MAX,
65 };
66 
67 struct memory_target {
68 	struct list_head node;
69 	unsigned int memory_pxm;
70 	unsigned int processor_pxm;
71 	struct resource memregions;
72 	struct access_coordinate coord[NODE_ACCESS_CLASS_MAX];
73 	struct list_head caches;
74 	struct node_cache_attrs cache_attrs;
75 	u8 gen_port_device_handle[ACPI_SRAT_DEVICE_HANDLE_SIZE];
76 	bool registered;
77 	bool ext_updated;	/* externally updated */
78 };
79 
80 struct memory_initiator {
81 	struct list_head node;
82 	unsigned int processor_pxm;
83 	bool has_cpu;
84 };
85 
86 struct memory_locality {
87 	struct list_head node;
88 	struct acpi_hmat_locality *hmat_loc;
89 };
90 
91 static struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm)
92 {
93 	struct memory_initiator *initiator;
94 
95 	list_for_each_entry(initiator, &initiators, node)
96 		if (initiator->processor_pxm == cpu_pxm)
97 			return initiator;
98 	return NULL;
99 }
100 
101 static struct memory_target *find_mem_target(unsigned int mem_pxm)
102 {
103 	struct memory_target *target;
104 
105 	list_for_each_entry(target, &targets, node)
106 		if (target->memory_pxm == mem_pxm)
107 			return target;
108 	return NULL;
109 }
110 
111 /**
112  * hmat_get_extended_linear_cache_size - Retrieve the extended linear cache size
113  * @backing_res: resource from the backing media
114  * @nid: node id for the memory region
115  * @cache_size: (Output) size of extended linear cache.
116  *
117  * Return: 0 on success. Errno on failure.
118  *
119  */
120 int hmat_get_extended_linear_cache_size(struct resource *backing_res, int nid,
121 					resource_size_t *cache_size)
122 {
123 	unsigned int pxm = node_to_pxm(nid);
124 	struct memory_target *target;
125 	struct target_cache *tcache;
126 	struct resource *res;
127 
128 	target = find_mem_target(pxm);
129 	if (!target)
130 		return -ENOENT;
131 
132 	list_for_each_entry(tcache, &target->caches, node) {
133 		if (tcache->cache_attrs.address_mode !=
134 				NODE_CACHE_ADDR_MODE_EXTENDED_LINEAR)
135 			continue;
136 
137 		res = &target->memregions;
138 		if (!resource_contains(res, backing_res))
139 			continue;
140 
141 		*cache_size = tcache->cache_attrs.size;
142 		return 0;
143 	}
144 
145 	*cache_size = 0;
146 	return 0;
147 }
148 EXPORT_SYMBOL_NS_GPL(hmat_get_extended_linear_cache_size, "CXL");
149 
150 static struct memory_target *acpi_find_genport_target(u32 uid)
151 {
152 	struct memory_target *target;
153 	u32 target_uid;
154 	u8 *uid_ptr;
155 
156 	list_for_each_entry(target, &targets, node) {
157 		uid_ptr = target->gen_port_device_handle + 8;
158 		target_uid = *(u32 *)uid_ptr;
159 		if (uid == target_uid)
160 			return target;
161 	}
162 
163 	return NULL;
164 }
165 
166 /**
167  * acpi_get_genport_coordinates - Retrieve the access coordinates for a generic port
168  * @uid: ACPI unique id
169  * @coord: The access coordinates written back out for the generic port.
170  *	   Expect 2 levels array.
171  *
172  * Return: 0 on success. Errno on failure.
173  *
174  * Only supports device handles that are ACPI. Assume ACPI0016 HID for CXL.
175  */
176 int acpi_get_genport_coordinates(u32 uid,
177 				 struct access_coordinate *coord)
178 {
179 	struct memory_target *target;
180 
181 	guard(mutex)(&target_lock);
182 	target = acpi_find_genport_target(uid);
183 	if (!target)
184 		return -ENOENT;
185 
186 	coord[ACCESS_COORDINATE_LOCAL] =
187 		target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL];
188 	coord[ACCESS_COORDINATE_CPU] =
189 		target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_CPU];
190 
191 	return 0;
192 }
193 EXPORT_SYMBOL_NS_GPL(acpi_get_genport_coordinates, "CXL");
194 
195 static __init void alloc_memory_initiator(unsigned int cpu_pxm)
196 {
197 	struct memory_initiator *initiator;
198 
199 	if (pxm_to_node(cpu_pxm) == NUMA_NO_NODE)
200 		return;
201 
202 	initiator = find_mem_initiator(cpu_pxm);
203 	if (initiator)
204 		return;
205 
206 	initiator = kzalloc(sizeof(*initiator), GFP_KERNEL);
207 	if (!initiator)
208 		return;
209 
210 	initiator->processor_pxm = cpu_pxm;
211 	initiator->has_cpu = node_state(pxm_to_node(cpu_pxm), N_CPU);
212 	list_add_tail(&initiator->node, &initiators);
213 }
214 
215 static __init struct memory_target *alloc_target(unsigned int mem_pxm)
216 {
217 	struct memory_target *target;
218 
219 	target = find_mem_target(mem_pxm);
220 	if (!target) {
221 		target = kzalloc(sizeof(*target), GFP_KERNEL);
222 		if (!target)
223 			return NULL;
224 		target->memory_pxm = mem_pxm;
225 		target->processor_pxm = PXM_INVAL;
226 		target->memregions = (struct resource) {
227 			.name	= "ACPI mem",
228 			.start	= 0,
229 			.end	= -1,
230 			.flags	= IORESOURCE_MEM,
231 		};
232 		list_add_tail(&target->node, &targets);
233 		INIT_LIST_HEAD(&target->caches);
234 	}
235 
236 	return target;
237 }
238 
239 static __init void alloc_memory_target(unsigned int mem_pxm,
240 				       resource_size_t start,
241 				       resource_size_t len)
242 {
243 	struct memory_target *target;
244 
245 	target = alloc_target(mem_pxm);
246 	if (!target)
247 		return;
248 
249 	/*
250 	 * There are potentially multiple ranges per PXM, so record each
251 	 * in the per-target memregions resource tree.
252 	 */
253 	if (!__request_region(&target->memregions, start, len, "memory target",
254 				IORESOURCE_MEM))
255 		pr_warn("failed to reserve %#llx - %#llx in pxm: %d\n",
256 				start, start + len, mem_pxm);
257 }
258 
259 static __init void alloc_genport_target(unsigned int mem_pxm, u8 *handle)
260 {
261 	struct memory_target *target;
262 
263 	target = alloc_target(mem_pxm);
264 	if (!target)
265 		return;
266 
267 	memcpy(target->gen_port_device_handle, handle,
268 	       ACPI_SRAT_DEVICE_HANDLE_SIZE);
269 }
270 
271 static __init const char *hmat_data_type(u8 type)
272 {
273 	switch (type) {
274 	case ACPI_HMAT_ACCESS_LATENCY:
275 		return "Access Latency";
276 	case ACPI_HMAT_READ_LATENCY:
277 		return "Read Latency";
278 	case ACPI_HMAT_WRITE_LATENCY:
279 		return "Write Latency";
280 	case ACPI_HMAT_ACCESS_BANDWIDTH:
281 		return "Access Bandwidth";
282 	case ACPI_HMAT_READ_BANDWIDTH:
283 		return "Read Bandwidth";
284 	case ACPI_HMAT_WRITE_BANDWIDTH:
285 		return "Write Bandwidth";
286 	default:
287 		return "Reserved";
288 	}
289 }
290 
291 static __init const char *hmat_data_type_suffix(u8 type)
292 {
293 	switch (type) {
294 	case ACPI_HMAT_ACCESS_LATENCY:
295 	case ACPI_HMAT_READ_LATENCY:
296 	case ACPI_HMAT_WRITE_LATENCY:
297 		return " nsec";
298 	case ACPI_HMAT_ACCESS_BANDWIDTH:
299 	case ACPI_HMAT_READ_BANDWIDTH:
300 	case ACPI_HMAT_WRITE_BANDWIDTH:
301 		return " MB/s";
302 	default:
303 		return "";
304 	}
305 }
306 
307 static u32 hmat_normalize(u16 entry, u64 base, u8 type)
308 {
309 	u32 value;
310 
311 	/*
312 	 * Check for invalid and overflow values
313 	 */
314 	if (entry == 0xffff || !entry)
315 		return 0;
316 	else if (base > (UINT_MAX / (entry)))
317 		return 0;
318 
319 	/*
320 	 * Divide by the base unit for version 1, convert latency from
321 	 * picosenonds to nanoseconds if revision 2.
322 	 */
323 	value = entry * base;
324 	if (hmat_revision == 1) {
325 		if (value < 10)
326 			return 0;
327 		value = DIV_ROUND_UP(value, 10);
328 	} else if (hmat_revision == 2) {
329 		switch (type) {
330 		case ACPI_HMAT_ACCESS_LATENCY:
331 		case ACPI_HMAT_READ_LATENCY:
332 		case ACPI_HMAT_WRITE_LATENCY:
333 			value = DIV_ROUND_UP(value, 1000);
334 			break;
335 		default:
336 			break;
337 		}
338 	}
339 	return value;
340 }
341 
342 static void hmat_update_target_access(struct memory_target *target,
343 				      u8 type, u32 value, int access)
344 {
345 	switch (type) {
346 	case ACPI_HMAT_ACCESS_LATENCY:
347 		target->coord[access].read_latency = value;
348 		target->coord[access].write_latency = value;
349 		break;
350 	case ACPI_HMAT_READ_LATENCY:
351 		target->coord[access].read_latency = value;
352 		break;
353 	case ACPI_HMAT_WRITE_LATENCY:
354 		target->coord[access].write_latency = value;
355 		break;
356 	case ACPI_HMAT_ACCESS_BANDWIDTH:
357 		target->coord[access].read_bandwidth = value;
358 		target->coord[access].write_bandwidth = value;
359 		break;
360 	case ACPI_HMAT_READ_BANDWIDTH:
361 		target->coord[access].read_bandwidth = value;
362 		break;
363 	case ACPI_HMAT_WRITE_BANDWIDTH:
364 		target->coord[access].write_bandwidth = value;
365 		break;
366 	default:
367 		break;
368 	}
369 }
370 
371 int hmat_update_target_coordinates(int nid, struct access_coordinate *coord,
372 				   enum access_coordinate_class access)
373 {
374 	struct memory_target *target;
375 	int pxm;
376 
377 	if (nid == NUMA_NO_NODE)
378 		return -EINVAL;
379 
380 	pxm = node_to_pxm(nid);
381 	guard(mutex)(&target_lock);
382 	target = find_mem_target(pxm);
383 	if (!target)
384 		return -ENODEV;
385 
386 	hmat_update_target_access(target, ACPI_HMAT_READ_LATENCY,
387 				  coord->read_latency, access);
388 	hmat_update_target_access(target, ACPI_HMAT_WRITE_LATENCY,
389 				  coord->write_latency, access);
390 	hmat_update_target_access(target, ACPI_HMAT_READ_BANDWIDTH,
391 				  coord->read_bandwidth, access);
392 	hmat_update_target_access(target, ACPI_HMAT_WRITE_BANDWIDTH,
393 				  coord->write_bandwidth, access);
394 	target->ext_updated = true;
395 
396 	return 0;
397 }
398 EXPORT_SYMBOL_GPL(hmat_update_target_coordinates);
399 
400 static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc)
401 {
402 	struct memory_locality *loc;
403 
404 	loc = kzalloc(sizeof(*loc), GFP_KERNEL);
405 	if (!loc) {
406 		pr_notice_once("Failed to allocate HMAT locality\n");
407 		return;
408 	}
409 
410 	loc->hmat_loc = hmat_loc;
411 	list_add_tail(&loc->node, &localities);
412 
413 	switch (hmat_loc->data_type) {
414 	case ACPI_HMAT_ACCESS_LATENCY:
415 		localities_types[READ_LATENCY] = loc;
416 		localities_types[WRITE_LATENCY] = loc;
417 		break;
418 	case ACPI_HMAT_READ_LATENCY:
419 		localities_types[READ_LATENCY] = loc;
420 		break;
421 	case ACPI_HMAT_WRITE_LATENCY:
422 		localities_types[WRITE_LATENCY] = loc;
423 		break;
424 	case ACPI_HMAT_ACCESS_BANDWIDTH:
425 		localities_types[READ_BANDWIDTH] = loc;
426 		localities_types[WRITE_BANDWIDTH] = loc;
427 		break;
428 	case ACPI_HMAT_READ_BANDWIDTH:
429 		localities_types[READ_BANDWIDTH] = loc;
430 		break;
431 	case ACPI_HMAT_WRITE_BANDWIDTH:
432 		localities_types[WRITE_BANDWIDTH] = loc;
433 		break;
434 	default:
435 		break;
436 	}
437 }
438 
439 static __init void hmat_update_target(unsigned int tgt_pxm, unsigned int init_pxm,
440 				      u8 mem_hier, u8 type, u32 value)
441 {
442 	struct memory_target *target = find_mem_target(tgt_pxm);
443 
444 	if (mem_hier != ACPI_HMAT_MEMORY)
445 		return;
446 
447 	if (target && target->processor_pxm == init_pxm) {
448 		hmat_update_target_access(target, type, value,
449 					  ACCESS_COORDINATE_LOCAL);
450 		/* If the node has a CPU, update access ACCESS_COORDINATE_CPU */
451 		if (node_state(pxm_to_node(init_pxm), N_CPU))
452 			hmat_update_target_access(target, type, value,
453 						  ACCESS_COORDINATE_CPU);
454 	}
455 }
456 
457 static __init int hmat_parse_locality(union acpi_subtable_headers *header,
458 				      const unsigned long end)
459 {
460 	struct acpi_hmat_locality *hmat_loc = (void *)header;
461 	unsigned int init, targ, total_size, ipds, tpds;
462 	u32 *inits, *targs, value;
463 	u16 *entries;
464 	u8 type, mem_hier;
465 
466 	if (hmat_loc->header.length < sizeof(*hmat_loc)) {
467 		pr_notice("Unexpected locality header length: %u\n",
468 			 hmat_loc->header.length);
469 		return -EINVAL;
470 	}
471 
472 	type = hmat_loc->data_type;
473 	mem_hier = hmat_loc->flags & ACPI_HMAT_MEMORY_HIERARCHY;
474 	ipds = hmat_loc->number_of_initiator_Pds;
475 	tpds = hmat_loc->number_of_target_Pds;
476 	total_size = sizeof(*hmat_loc) + sizeof(*entries) * ipds * tpds +
477 		     sizeof(*inits) * ipds + sizeof(*targs) * tpds;
478 	if (hmat_loc->header.length < total_size) {
479 		pr_notice("Unexpected locality header length:%u, minimum required:%u\n",
480 			 hmat_loc->header.length, total_size);
481 		return -EINVAL;
482 	}
483 
484 	pr_debug("Locality: Flags:%02x Type:%s Initiator Domains:%u Target Domains:%u Base:%lld\n",
485 		 hmat_loc->flags, hmat_data_type(type), ipds, tpds,
486 		 hmat_loc->entry_base_unit);
487 
488 	inits = (u32 *)(hmat_loc + 1);
489 	targs = inits + ipds;
490 	entries = (u16 *)(targs + tpds);
491 	for (init = 0; init < ipds; init++) {
492 		alloc_memory_initiator(inits[init]);
493 		for (targ = 0; targ < tpds; targ++) {
494 			value = hmat_normalize(entries[init * tpds + targ],
495 					       hmat_loc->entry_base_unit,
496 					       type);
497 			pr_debug("  Initiator-Target[%u-%u]:%u%s\n",
498 				 inits[init], targs[targ], value,
499 				 hmat_data_type_suffix(type));
500 
501 			hmat_update_target(targs[targ], inits[init],
502 					   mem_hier, type, value);
503 		}
504 	}
505 
506 	if (mem_hier == ACPI_HMAT_MEMORY)
507 		hmat_add_locality(hmat_loc);
508 
509 	return 0;
510 }
511 
512 static __init int hmat_parse_cache(union acpi_subtable_headers *header,
513 				   const unsigned long end)
514 {
515 	struct acpi_hmat_cache *cache = (void *)header;
516 	struct memory_target *target;
517 	struct target_cache *tcache;
518 	u32 attrs;
519 
520 	if (cache->header.length < sizeof(*cache)) {
521 		pr_notice("Unexpected cache header length: %u\n",
522 			 cache->header.length);
523 		return -EINVAL;
524 	}
525 
526 	attrs = cache->cache_attributes;
527 	pr_debug("Cache: Domain:%u Size:%llu Attrs:%08x SMBIOS Handles:%d\n",
528 		 cache->memory_PD, cache->cache_size, attrs,
529 		 cache->number_of_SMBIOShandles);
530 
531 	target = find_mem_target(cache->memory_PD);
532 	if (!target)
533 		return 0;
534 
535 	tcache = kzalloc(sizeof(*tcache), GFP_KERNEL);
536 	if (!tcache) {
537 		pr_notice_once("Failed to allocate HMAT cache info\n");
538 		return 0;
539 	}
540 
541 	tcache->cache_attrs.size = cache->cache_size;
542 	tcache->cache_attrs.level = (attrs & ACPI_HMAT_CACHE_LEVEL) >> 4;
543 	tcache->cache_attrs.line_size = (attrs & ACPI_HMAT_CACHE_LINE_SIZE) >> 16;
544 
545 	switch ((attrs & ACPI_HMAT_CACHE_ASSOCIATIVITY) >> 8) {
546 	case ACPI_HMAT_CA_DIRECT_MAPPED:
547 		tcache->cache_attrs.indexing = NODE_CACHE_DIRECT_MAP;
548 		/* Extended Linear mode is only valid if cache is direct mapped */
549 		if (cache->address_mode == ACPI_HMAT_CACHE_MODE_EXTENDED_LINEAR) {
550 			tcache->cache_attrs.address_mode =
551 				NODE_CACHE_ADDR_MODE_EXTENDED_LINEAR;
552 		}
553 		break;
554 	case ACPI_HMAT_CA_COMPLEX_CACHE_INDEXING:
555 		tcache->cache_attrs.indexing = NODE_CACHE_INDEXED;
556 		break;
557 	case ACPI_HMAT_CA_NONE:
558 	default:
559 		tcache->cache_attrs.indexing = NODE_CACHE_OTHER;
560 		break;
561 	}
562 
563 	switch ((attrs & ACPI_HMAT_WRITE_POLICY) >> 12) {
564 	case ACPI_HMAT_CP_WB:
565 		tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_BACK;
566 		break;
567 	case ACPI_HMAT_CP_WT:
568 		tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_THROUGH;
569 		break;
570 	case ACPI_HMAT_CP_NONE:
571 	default:
572 		tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_OTHER;
573 		break;
574 	}
575 	list_add_tail(&tcache->node, &target->caches);
576 
577 	return 0;
578 }
579 
580 static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *header,
581 					      const unsigned long end)
582 {
583 	struct acpi_hmat_proximity_domain *p = (void *)header;
584 	struct memory_target *target = NULL;
585 
586 	if (p->header.length != sizeof(*p)) {
587 		pr_notice("Unexpected address range header length: %u\n",
588 			 p->header.length);
589 		return -EINVAL;
590 	}
591 
592 	if (hmat_revision == 1)
593 		pr_debug("Memory (%#llx length %#llx) Flags:%04x Processor Domain:%u Memory Domain:%u\n",
594 			 p->reserved3, p->reserved4, p->flags, p->processor_PD,
595 			 p->memory_PD);
596 	else
597 		pr_info("Memory Flags:%04x Processor Domain:%u Memory Domain:%u\n",
598 			p->flags, p->processor_PD, p->memory_PD);
599 
600 	if ((hmat_revision == 1 && p->flags & ACPI_HMAT_MEMORY_PD_VALID) ||
601 	    hmat_revision > 1) {
602 		target = find_mem_target(p->memory_PD);
603 		if (!target) {
604 			pr_debug("Memory Domain missing from SRAT\n");
605 			return -EINVAL;
606 		}
607 	}
608 	if (target && p->flags & ACPI_HMAT_PROCESSOR_PD_VALID) {
609 		int p_node = pxm_to_node(p->processor_PD);
610 
611 		if (p_node == NUMA_NO_NODE) {
612 			pr_debug("Invalid Processor Domain\n");
613 			return -EINVAL;
614 		}
615 		target->processor_pxm = p->processor_PD;
616 	}
617 
618 	return 0;
619 }
620 
621 static int __init hmat_parse_subtable(union acpi_subtable_headers *header,
622 				      const unsigned long end)
623 {
624 	struct acpi_hmat_structure *hdr = (void *)header;
625 
626 	if (!hdr)
627 		return -EINVAL;
628 
629 	switch (hdr->type) {
630 	case ACPI_HMAT_TYPE_PROXIMITY:
631 		return hmat_parse_proximity_domain(header, end);
632 	case ACPI_HMAT_TYPE_LOCALITY:
633 		return hmat_parse_locality(header, end);
634 	case ACPI_HMAT_TYPE_CACHE:
635 		return hmat_parse_cache(header, end);
636 	default:
637 		return -EINVAL;
638 	}
639 }
640 
641 static __init int srat_parse_mem_affinity(union acpi_subtable_headers *header,
642 					  const unsigned long end)
643 {
644 	struct acpi_srat_mem_affinity *ma = (void *)header;
645 
646 	if (!ma)
647 		return -EINVAL;
648 	if (!(ma->flags & ACPI_SRAT_MEM_ENABLED))
649 		return 0;
650 	alloc_memory_target(ma->proximity_domain, ma->base_address, ma->length);
651 	return 0;
652 }
653 
654 static __init int srat_parse_genport_affinity(union acpi_subtable_headers *header,
655 					      const unsigned long end)
656 {
657 	struct acpi_srat_generic_affinity *ga = (void *)header;
658 
659 	if (!ga)
660 		return -EINVAL;
661 
662 	if (!(ga->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED))
663 		return 0;
664 
665 	/* Skip PCI device_handle for now */
666 	if (ga->device_handle_type != 0)
667 		return 0;
668 
669 	alloc_genport_target(ga->proximity_domain,
670 			     (u8 *)ga->device_handle);
671 
672 	return 0;
673 }
674 
675 static u32 hmat_initiator_perf(struct memory_target *target,
676 			       struct memory_initiator *initiator,
677 			       struct acpi_hmat_locality *hmat_loc)
678 {
679 	unsigned int ipds, tpds, i, idx = 0, tdx = 0;
680 	u32 *inits, *targs;
681 	u16 *entries;
682 
683 	ipds = hmat_loc->number_of_initiator_Pds;
684 	tpds = hmat_loc->number_of_target_Pds;
685 	inits = (u32 *)(hmat_loc + 1);
686 	targs = inits + ipds;
687 	entries = (u16 *)(targs + tpds);
688 
689 	for (i = 0; i < ipds; i++) {
690 		if (inits[i] == initiator->processor_pxm) {
691 			idx = i;
692 			break;
693 		}
694 	}
695 
696 	if (i == ipds)
697 		return 0;
698 
699 	for (i = 0; i < tpds; i++) {
700 		if (targs[i] == target->memory_pxm) {
701 			tdx = i;
702 			break;
703 		}
704 	}
705 	if (i == tpds)
706 		return 0;
707 
708 	return hmat_normalize(entries[idx * tpds + tdx],
709 			      hmat_loc->entry_base_unit,
710 			      hmat_loc->data_type);
711 }
712 
713 static bool hmat_update_best(u8 type, u32 value, u32 *best)
714 {
715 	bool updated = false;
716 
717 	if (!value)
718 		return false;
719 
720 	switch (type) {
721 	case ACPI_HMAT_ACCESS_LATENCY:
722 	case ACPI_HMAT_READ_LATENCY:
723 	case ACPI_HMAT_WRITE_LATENCY:
724 		if (!*best || *best > value) {
725 			*best = value;
726 			updated = true;
727 		}
728 		break;
729 	case ACPI_HMAT_ACCESS_BANDWIDTH:
730 	case ACPI_HMAT_READ_BANDWIDTH:
731 	case ACPI_HMAT_WRITE_BANDWIDTH:
732 		if (!*best || *best < value) {
733 			*best = value;
734 			updated = true;
735 		}
736 		break;
737 	}
738 
739 	return updated;
740 }
741 
742 static int initiator_cmp(void *priv, const struct list_head *a,
743 			 const struct list_head *b)
744 {
745 	struct memory_initiator *ia;
746 	struct memory_initiator *ib;
747 
748 	ia = list_entry(a, struct memory_initiator, node);
749 	ib = list_entry(b, struct memory_initiator, node);
750 
751 	return ia->processor_pxm - ib->processor_pxm;
752 }
753 
754 static int initiators_to_nodemask(unsigned long *p_nodes)
755 {
756 	struct memory_initiator *initiator;
757 
758 	if (list_empty(&initiators))
759 		return -ENXIO;
760 
761 	list_for_each_entry(initiator, &initiators, node)
762 		set_bit(initiator->processor_pxm, p_nodes);
763 
764 	return 0;
765 }
766 
767 static void hmat_update_target_attrs(struct memory_target *target,
768 				     unsigned long *p_nodes, int access)
769 {
770 	struct memory_initiator *initiator;
771 	unsigned int cpu_nid;
772 	struct memory_locality *loc = NULL;
773 	u32 best = 0;
774 	int i;
775 
776 	/* Don't update if an external agent has changed the data.  */
777 	if (target->ext_updated)
778 		return;
779 
780 	/* Don't update for generic port if there's no device handle */
781 	if ((access == NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL ||
782 	     access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) &&
783 	    !(*(u16 *)target->gen_port_device_handle))
784 		return;
785 
786 	bitmap_zero(p_nodes, MAX_NUMNODES);
787 	/*
788 	 * If the Address Range Structure provides a local processor pxm, set
789 	 * only that one. Otherwise, find the best performance attributes and
790 	 * collect all initiators that match.
791 	 */
792 	if (target->processor_pxm != PXM_INVAL) {
793 		cpu_nid = pxm_to_node(target->processor_pxm);
794 		if (access == ACCESS_COORDINATE_LOCAL ||
795 		    node_state(cpu_nid, N_CPU)) {
796 			set_bit(target->processor_pxm, p_nodes);
797 			return;
798 		}
799 	}
800 
801 	if (list_empty(&localities))
802 		return;
803 
804 	/*
805 	 * We need the initiator list sorted so we can use bitmap_clear for
806 	 * previously set initiators when we find a better memory accessor.
807 	 * We'll also use the sorting to prime the candidate nodes with known
808 	 * initiators.
809 	 */
810 	list_sort(NULL, &initiators, initiator_cmp);
811 	if (initiators_to_nodemask(p_nodes) < 0)
812 		return;
813 
814 	for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) {
815 		loc = localities_types[i];
816 		if (!loc)
817 			continue;
818 
819 		best = 0;
820 		list_for_each_entry(initiator, &initiators, node) {
821 			u32 value;
822 
823 			if ((access == ACCESS_COORDINATE_CPU ||
824 			     access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) &&
825 			    !initiator->has_cpu) {
826 				clear_bit(initiator->processor_pxm, p_nodes);
827 				continue;
828 			}
829 			if (!test_bit(initiator->processor_pxm, p_nodes))
830 				continue;
831 
832 			value = hmat_initiator_perf(target, initiator, loc->hmat_loc);
833 			if (hmat_update_best(loc->hmat_loc->data_type, value, &best))
834 				bitmap_clear(p_nodes, 0, initiator->processor_pxm);
835 			if (value != best)
836 				clear_bit(initiator->processor_pxm, p_nodes);
837 		}
838 		if (best)
839 			hmat_update_target_access(target, loc->hmat_loc->data_type, best, access);
840 	}
841 }
842 
843 static void __hmat_register_target_initiators(struct memory_target *target,
844 					      unsigned long *p_nodes,
845 					      int access)
846 {
847 	unsigned int mem_nid, cpu_nid;
848 	int i;
849 
850 	mem_nid = pxm_to_node(target->memory_pxm);
851 	hmat_update_target_attrs(target, p_nodes, access);
852 	for_each_set_bit(i, p_nodes, MAX_NUMNODES) {
853 		cpu_nid = pxm_to_node(i);
854 		register_memory_node_under_compute_node(mem_nid, cpu_nid, access);
855 	}
856 }
857 
858 static void hmat_update_generic_target(struct memory_target *target)
859 {
860 	static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
861 
862 	hmat_update_target_attrs(target, p_nodes,
863 				 NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL);
864 	hmat_update_target_attrs(target, p_nodes,
865 				 NODE_ACCESS_CLASS_GENPORT_SINK_CPU);
866 }
867 
868 static void hmat_register_target_initiators(struct memory_target *target)
869 {
870 	static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
871 
872 	__hmat_register_target_initiators(target, p_nodes,
873 					  ACCESS_COORDINATE_LOCAL);
874 	__hmat_register_target_initiators(target, p_nodes,
875 					  ACCESS_COORDINATE_CPU);
876 }
877 
878 static void hmat_register_target_cache(struct memory_target *target)
879 {
880 	unsigned mem_nid = pxm_to_node(target->memory_pxm);
881 	struct target_cache *tcache;
882 
883 	list_for_each_entry(tcache, &target->caches, node)
884 		node_add_cache(mem_nid, &tcache->cache_attrs);
885 }
886 
887 static void hmat_register_target_perf(struct memory_target *target, int access)
888 {
889 	unsigned mem_nid = pxm_to_node(target->memory_pxm);
890 	node_set_perf_attrs(mem_nid, &target->coord[access], access);
891 }
892 
893 static void hmat_register_target_devices(struct memory_target *target)
894 {
895 	struct resource *res;
896 
897 	/*
898 	 * Do not bother creating devices if no driver is available to
899 	 * consume them.
900 	 */
901 	if (!IS_ENABLED(CONFIG_DEV_DAX_HMEM))
902 		return;
903 
904 	for (res = target->memregions.child; res; res = res->sibling) {
905 		int target_nid = pxm_to_node(target->memory_pxm);
906 
907 		hmem_register_resource(target_nid, res);
908 	}
909 }
910 
911 static void hmat_register_target(struct memory_target *target)
912 {
913 	int nid = pxm_to_node(target->memory_pxm);
914 
915 	/*
916 	 * Devices may belong to either an offline or online
917 	 * node, so unconditionally add them.
918 	 */
919 	hmat_register_target_devices(target);
920 
921 	/*
922 	 * Register generic port perf numbers. The nid may not be
923 	 * initialized and is still NUMA_NO_NODE.
924 	 */
925 	mutex_lock(&target_lock);
926 	if (*(u16 *)target->gen_port_device_handle) {
927 		hmat_update_generic_target(target);
928 		target->registered = true;
929 	}
930 	mutex_unlock(&target_lock);
931 
932 	/*
933 	 * Skip offline nodes. This can happen when memory
934 	 * marked EFI_MEMORY_SP, "specific purpose", is applied
935 	 * to all the memory in a proximity domain leading to
936 	 * the node being marked offline / unplugged, or if
937 	 * memory-only "hotplug" node is offline.
938 	 */
939 	if (nid == NUMA_NO_NODE || !node_online(nid))
940 		return;
941 
942 	mutex_lock(&target_lock);
943 	if (!target->registered) {
944 		hmat_register_target_initiators(target);
945 		hmat_register_target_cache(target);
946 		hmat_register_target_perf(target, ACCESS_COORDINATE_LOCAL);
947 		hmat_register_target_perf(target, ACCESS_COORDINATE_CPU);
948 		target->registered = true;
949 	}
950 	mutex_unlock(&target_lock);
951 }
952 
953 static void hmat_register_targets(void)
954 {
955 	struct memory_target *target;
956 
957 	list_for_each_entry(target, &targets, node)
958 		hmat_register_target(target);
959 }
960 
961 static int hmat_callback(struct notifier_block *self,
962 			 unsigned long action, void *arg)
963 {
964 	struct memory_target *target;
965 	struct memory_notify *mnb = arg;
966 	int pxm, nid = mnb->status_change_nid;
967 
968 	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
969 		return NOTIFY_OK;
970 
971 	pxm = node_to_pxm(nid);
972 	target = find_mem_target(pxm);
973 	if (!target)
974 		return NOTIFY_OK;
975 
976 	hmat_register_target(target);
977 	return NOTIFY_OK;
978 }
979 
980 static int __init hmat_set_default_dram_perf(void)
981 {
982 	int rc;
983 	int nid, pxm;
984 	struct memory_target *target;
985 	struct access_coordinate *attrs;
986 
987 	for_each_node_mask(nid, default_dram_nodes) {
988 		pxm = node_to_pxm(nid);
989 		target = find_mem_target(pxm);
990 		if (!target)
991 			continue;
992 		attrs = &target->coord[ACCESS_COORDINATE_CPU];
993 		rc = mt_set_default_dram_perf(nid, attrs, "ACPI HMAT");
994 		if (rc)
995 			return rc;
996 	}
997 
998 	return 0;
999 }
1000 
1001 static int hmat_calculate_adistance(struct notifier_block *self,
1002 				    unsigned long nid, void *data)
1003 {
1004 	static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
1005 	struct memory_target *target;
1006 	struct access_coordinate *perf;
1007 	int *adist = data;
1008 	int pxm;
1009 
1010 	pxm = node_to_pxm(nid);
1011 	target = find_mem_target(pxm);
1012 	if (!target)
1013 		return NOTIFY_OK;
1014 
1015 	mutex_lock(&target_lock);
1016 	hmat_update_target_attrs(target, p_nodes, ACCESS_COORDINATE_CPU);
1017 	mutex_unlock(&target_lock);
1018 
1019 	perf = &target->coord[ACCESS_COORDINATE_CPU];
1020 
1021 	if (mt_perf_to_adistance(perf, adist))
1022 		return NOTIFY_OK;
1023 
1024 	return NOTIFY_STOP;
1025 }
1026 
1027 static struct notifier_block hmat_adist_nb __meminitdata = {
1028 	.notifier_call = hmat_calculate_adistance,
1029 	.priority = 100,
1030 };
1031 
1032 static __init void hmat_free_structures(void)
1033 {
1034 	struct memory_target *target, *tnext;
1035 	struct memory_locality *loc, *lnext;
1036 	struct memory_initiator *initiator, *inext;
1037 	struct target_cache *tcache, *cnext;
1038 
1039 	list_for_each_entry_safe(target, tnext, &targets, node) {
1040 		struct resource *res, *res_next;
1041 
1042 		list_for_each_entry_safe(tcache, cnext, &target->caches, node) {
1043 			list_del(&tcache->node);
1044 			kfree(tcache);
1045 		}
1046 
1047 		list_del(&target->node);
1048 		res = target->memregions.child;
1049 		while (res) {
1050 			res_next = res->sibling;
1051 			__release_region(&target->memregions, res->start,
1052 					resource_size(res));
1053 			res = res_next;
1054 		}
1055 		kfree(target);
1056 	}
1057 
1058 	list_for_each_entry_safe(initiator, inext, &initiators, node) {
1059 		list_del(&initiator->node);
1060 		kfree(initiator);
1061 	}
1062 
1063 	list_for_each_entry_safe(loc, lnext, &localities, node) {
1064 		list_del(&loc->node);
1065 		kfree(loc);
1066 	}
1067 }
1068 
1069 static __init int hmat_init(void)
1070 {
1071 	struct acpi_table_header *tbl;
1072 	enum acpi_hmat_type i;
1073 	acpi_status status;
1074 
1075 	if (srat_disabled() || hmat_disable)
1076 		return 0;
1077 
1078 	status = acpi_get_table(ACPI_SIG_SRAT, 0, &tbl);
1079 	if (ACPI_FAILURE(status))
1080 		return 0;
1081 
1082 	if (acpi_table_parse_entries(ACPI_SIG_SRAT,
1083 				sizeof(struct acpi_table_srat),
1084 				ACPI_SRAT_TYPE_MEMORY_AFFINITY,
1085 				srat_parse_mem_affinity, 0) < 0)
1086 		goto out_put;
1087 
1088 	if (acpi_table_parse_entries(ACPI_SIG_SRAT,
1089 				     sizeof(struct acpi_table_srat),
1090 				     ACPI_SRAT_TYPE_GENERIC_PORT_AFFINITY,
1091 				     srat_parse_genport_affinity, 0) < 0)
1092 		goto out_put;
1093 
1094 	acpi_put_table(tbl);
1095 
1096 	status = acpi_get_table(ACPI_SIG_HMAT, 0, &tbl);
1097 	if (ACPI_FAILURE(status))
1098 		goto out_put;
1099 
1100 	hmat_revision = tbl->revision;
1101 	switch (hmat_revision) {
1102 	case 1:
1103 	case 2:
1104 		break;
1105 	default:
1106 		pr_notice("Ignoring: Unknown revision:%d\n", hmat_revision);
1107 		goto out_put;
1108 	}
1109 
1110 	for (i = ACPI_HMAT_TYPE_PROXIMITY; i < ACPI_HMAT_TYPE_RESERVED; i++) {
1111 		if (acpi_table_parse_entries(ACPI_SIG_HMAT,
1112 					     sizeof(struct acpi_table_hmat), i,
1113 					     hmat_parse_subtable, 0) < 0) {
1114 			pr_notice("Ignoring: Invalid table");
1115 			goto out_put;
1116 		}
1117 	}
1118 	hmat_register_targets();
1119 
1120 	/* Keep the table and structures if the notifier may use them */
1121 	if (hotplug_memory_notifier(hmat_callback, HMAT_CALLBACK_PRI))
1122 		goto out_put;
1123 
1124 	if (!hmat_set_default_dram_perf())
1125 		register_mt_adistance_algorithm(&hmat_adist_nb);
1126 
1127 	return 0;
1128 out_put:
1129 	hmat_free_structures();
1130 	acpi_put_table(tbl);
1131 	return 0;
1132 }
1133 subsys_initcall(hmat_init);
1134