Lines Matching +full:non +full:- +full:descriptive

1 // SPDX-License-Identifier: GPL-2.0
5 * Pseudo-locking support built on top of Cache Allocation Technology (CAT)
26 #include <asm/intel-family.h>
44 * pseudo-locked regions.
51 * get_prefetch_disable_bits - prefetch disable bits of supported platforms
54 * pseudo-locking. This includes testing to ensure pseudo-locked regions
56 * as well as that these pseudo-locked regions can maintain their low cache
59 * After a platform has been validated to support pseudo-locking its
105 * pseudo_lock_minor_get - Obtain available minor number
120 return -ENOSPC; in pseudo_lock_minor_get()
129 * pseudo_lock_minor_release - Return minor number to available
138 * region_find_by_minor - Locate a pseudo-lock region by inode minor number
139 * @minor: The minor number of the device representing pseudo-locked region
142 * pseudo-locked region it belongs to. This is done by matching the minor
143 * number of the device to the pseudo-locked region it belongs.
145 * Minor numbers are assigned at the time a pseudo-locked region is associated
148 * Return: On success return pointer to resource group owning the pseudo-locked
156 if (rdtgrp->plr && rdtgrp->plr->minor == minor) { in region_find_by_minor()
165 * pseudo_lock_pm_req - A power management QoS request list entry
166 * @list: Entry within the @pm_reqs list for a pseudo-locked region
178 list_for_each_entry_safe(pm_req, next, &plr->pm_reqs, list) { in pseudo_lock_cstates_relax()
179 dev_pm_qos_remove_request(&pm_req->req); in pseudo_lock_cstates_relax()
180 list_del(&pm_req->list); in pseudo_lock_cstates_relax()
186 * pseudo_lock_cstates_constrain - Restrict cores from entering C6
206 for_each_cpu(cpu, &plr->d->cpu_mask) { in pseudo_lock_cstates_constrain()
210 ret = -ENOMEM; in pseudo_lock_cstates_constrain()
214 &pm_req->req, in pseudo_lock_cstates_constrain()
221 ret = -1; in pseudo_lock_cstates_constrain()
224 list_add(&pm_req->list, &plr->pm_reqs); in pseudo_lock_cstates_constrain()
235 * pseudo_lock_region_clear - Reset pseudo-lock region data
236 * @plr: pseudo-lock region
238 * All content of the pseudo-locked region is reset - any memory allocated
245 plr->size = 0; in pseudo_lock_region_clear()
246 plr->line_size = 0; in pseudo_lock_region_clear()
247 kfree(plr->kmem); in pseudo_lock_region_clear()
248 plr->kmem = NULL; in pseudo_lock_region_clear()
249 plr->r = NULL; in pseudo_lock_region_clear()
250 if (plr->d) in pseudo_lock_region_clear()
251 plr->d->plr = NULL; in pseudo_lock_region_clear()
252 plr->d = NULL; in pseudo_lock_region_clear()
253 plr->cbm = 0; in pseudo_lock_region_clear()
254 plr->debugfs_dir = NULL; in pseudo_lock_region_clear()
258 * pseudo_lock_region_init - Initialize pseudo-lock region information
259 * @plr: pseudo-lock region
261 * Called after user provided a schemata to be pseudo-locked. From the
264 * required for pseudo-locking is deduced from this data and &struct
266 * - size in bytes of the region to be pseudo-locked
267 * - cache line size to know the stride with which data needs to be accessed
268 * to be pseudo-locked
269 * - a cpu associated with the cache instance on which the pseudo-locking
272 * Return: 0 on success, <0 on failure. Descriptive error will be written
282 plr->cpu = cpumask_first(&plr->d->cpu_mask); in pseudo_lock_region_init()
284 if (!cpu_online(plr->cpu)) { in pseudo_lock_region_init()
286 plr->cpu); in pseudo_lock_region_init()
287 ret = -ENODEV; in pseudo_lock_region_init()
291 ci = get_cpu_cacheinfo(plr->cpu); in pseudo_lock_region_init()
293 plr->size = rdtgroup_cbm_to_size(plr->r, plr->d, plr->cbm); in pseudo_lock_region_init()
295 for (i = 0; i < ci->num_leaves; i++) { in pseudo_lock_region_init()
296 if (ci->info_list[i].level == plr->r->cache_level) { in pseudo_lock_region_init()
297 plr->line_size = ci->info_list[i].coherency_line_size; in pseudo_lock_region_init()
302 ret = -1; in pseudo_lock_region_init()
310 * pseudo_lock_init - Initialize a pseudo-lock region
311 * @rdtgrp: resource group to which new pseudo-locked region will belong
313 * A pseudo-locked region is associated with a resource group. When this
314 * association is created the pseudo-locked region is initialized. The
315 * details of the pseudo-locked region are not known at this time so only
326 return -ENOMEM; in pseudo_lock_init()
328 init_waitqueue_head(&plr->lock_thread_wq); in pseudo_lock_init()
329 INIT_LIST_HEAD(&plr->pm_reqs); in pseudo_lock_init()
330 rdtgrp->plr = plr; in pseudo_lock_init()
335 * pseudo_lock_region_alloc - Allocate kernel memory that will be pseudo-locked
336 * @plr: pseudo-lock region
338 * Initialize the details required to set up the pseudo-locked region and
339 * allocate the contiguous memory that will be pseudo-locked to the cache.
341 * Return: 0 on success, <0 on failure. Descriptive error will be written
356 if (plr->size > KMALLOC_MAX_SIZE) { in pseudo_lock_region_alloc()
358 ret = -E2BIG; in pseudo_lock_region_alloc()
362 plr->kmem = kzalloc(plr->size, GFP_KERNEL); in pseudo_lock_region_alloc()
363 if (!plr->kmem) { in pseudo_lock_region_alloc()
365 ret = -ENOMEM; in pseudo_lock_region_alloc()
378 * pseudo_lock_free - Free a pseudo-locked region
379 * @rdtgrp: resource group to which pseudo-locked region belonged
381 * The pseudo-locked region's resources have already been released, or not
389 pseudo_lock_region_clear(rdtgrp->plr); in pseudo_lock_free()
390 kfree(rdtgrp->plr); in pseudo_lock_free()
391 rdtgrp->plr = NULL; in pseudo_lock_free()
395 * pseudo_lock_fn - Load kernel memory into cache
396 * @_rdtgrp: resource group to which pseudo-lock region belongs
398 * This is the core pseudo-locking flow.
403 * with class of service set to the bitmask of the pseudo-locked region.
416 struct pseudo_lock_region *plr = rdtgrp->plr; in pseudo_lock_fn()
426 * pseudo-locking success rate when KASAN is active. in pseudo_lock_fn()
439 * will get a cache hit in below loop from outside of pseudo-locked in pseudo_lock_fn()
458 * being pseudo-locked is reached the hardware will not read beyond in pseudo_lock_fn()
459 * the buffer and evict pseudo-locked memory read earlier from the in pseudo_lock_fn()
465 mem_r = plr->kmem; in pseudo_lock_fn()
466 size = plr->size; in pseudo_lock_fn()
467 line_size = plr->line_size; in pseudo_lock_fn()
471 * pseudo-locked followed by reading of kernel memory to load it in pseudo_lock_fn()
474 __wrmsr(IA32_PQR_ASSOC, rmid_p, rdtgrp->closid); in pseudo_lock_fn()
477 * into cache region associated with just activated plr->closid. in pseudo_lock_fn()
479 * - In first loop the cache region is shared with the page walker in pseudo_lock_fn()
481 * - In the second loop the paging structure caches are used and in pseudo_lock_fn()
508 * does not overlap with pseudo-locked region. in pseudo_lock_fn()
512 /* Re-enable the hardware prefetcher(s) */ in pseudo_lock_fn()
516 plr->thread_done = 1; in pseudo_lock_fn()
517 wake_up_interruptible(&plr->lock_thread_wq); in pseudo_lock_fn()
522 * rdtgroup_monitor_in_progress - Test if monitoring in progress
530 return !list_empty(&rdtgrp->mon.crdtgrp_list); in rdtgroup_monitor_in_progress()
534 * rdtgroup_locksetup_user_restrict - Restrict user access to group
537 * A resource group used for cache pseudo-locking cannot have cpus or tasks
584 * rdtgroup_locksetup_user_restore - Restore user access to group
631 * rdtgroup_locksetup_enter - Resource group enters locksetup mode
635 * to represent a pseudo-locked region and is in the process of being set
636 * up to do so. A resource group used for a pseudo-locked region would
639 * future. Monitoring of a pseudo-locked region is not allowed either.
641 * The above and more restrictions on a pseudo-locked region are checked
657 rdt_last_cmd_puts("Cannot pseudo-lock default group\n"); in rdtgroup_locksetup_enter()
658 return -EINVAL; in rdtgroup_locksetup_enter()
662 * Cache Pseudo-locking not supported when CDP is enabled. in rdtgroup_locksetup_enter()
666 * - When CDP is enabled two separate resources are exposed, in rdtgroup_locksetup_enter()
668 * The implication for pseudo-locking is that if a in rdtgroup_locksetup_enter()
669 * pseudo-locked region is created on a domain of one in rdtgroup_locksetup_enter()
670 * resource (eg. L3CODE), then a pseudo-locked region cannot in rdtgroup_locksetup_enter()
673 * pseudo-locked region involves a call to wbinvd that will in rdtgroup_locksetup_enter()
675 * - Considering the previous, it may be possible to only in rdtgroup_locksetup_enter()
676 * expose one of the CDP resources to pseudo-locking and in rdtgroup_locksetup_enter()
680 * - If only one region is exposed to pseudo-locking we should in rdtgroup_locksetup_enter()
682 * for pseudo-locking should take into account both resources. in rdtgroup_locksetup_enter()
683 * Similarly, if a pseudo-locked region is created in one in rdtgroup_locksetup_enter()
690 return -EINVAL; in rdtgroup_locksetup_enter()
695 * platform does not support Cache Pseudo-Locking. in rdtgroup_locksetup_enter()
699 rdt_last_cmd_puts("Pseudo-locking not supported\n"); in rdtgroup_locksetup_enter()
700 return -EINVAL; in rdtgroup_locksetup_enter()
705 return -EINVAL; in rdtgroup_locksetup_enter()
710 return -EINVAL; in rdtgroup_locksetup_enter()
713 if (!cpumask_empty(&rdtgrp->cpu_mask)) { in rdtgroup_locksetup_enter()
715 return -EINVAL; in rdtgroup_locksetup_enter()
720 return -EIO; in rdtgroup_locksetup_enter()
725 rdt_last_cmd_puts("Unable to init pseudo-lock region\n"); in rdtgroup_locksetup_enter()
732 * anymore when this group would be used for pseudo-locking. This in rdtgroup_locksetup_enter()
735 free_rmid(rdtgrp->mon.rmid); in rdtgroup_locksetup_enter()
747 * rdtgroup_locksetup_exit - resource group exist locksetup mode
765 rdtgrp->mon.rmid = ret; in rdtgroup_locksetup_exit()
770 free_rmid(rdtgrp->mon.rmid); in rdtgroup_locksetup_exit()
779 * rdtgroup_cbm_overlaps_pseudo_locked - Test if CBM or portion is pseudo-locked
785 * pseudo-locked region on @d.
790 * Return: true if @cbm overlaps with pseudo-locked region on @d, false
798 if (d->plr) { in rdtgroup_cbm_overlaps_pseudo_locked()
799 cbm_len = d->plr->r->cache.cbm_len; in rdtgroup_cbm_overlaps_pseudo_locked()
800 cbm_b = d->plr->cbm; in rdtgroup_cbm_overlaps_pseudo_locked()
808 * rdtgroup_pseudo_locked_in_hierarchy - Pseudo-locked region in cache hierarchy
811 * The setup of a pseudo-locked region affects all cache instances within
813 * pseudo-locked regions exist within a cache hierarchy to prevent any
814 * attempts to create new pseudo-locked regions in the same hierarchy.
816 * Return: true if a pseudo-locked region exists in the hierarchy of @d or
831 * First determine which cpus have pseudo-locked regions in rdtgroup_pseudo_locked_in_hierarchy()
835 list_for_each_entry(d_i, &r->domains, list) { in rdtgroup_pseudo_locked_in_hierarchy()
836 if (d_i->plr) in rdtgroup_pseudo_locked_in_hierarchy()
838 &d_i->cpu_mask); in rdtgroup_pseudo_locked_in_hierarchy()
843 * Next test if new pseudo-locked region would intersect with in rdtgroup_pseudo_locked_in_hierarchy()
846 if (cpumask_intersects(&d->cpu_mask, cpu_with_psl)) in rdtgroup_pseudo_locked_in_hierarchy()
854 * measure_cycles_lat_fn - Measure cycle latency to read pseudo-locked memory
855 * @_plr: pseudo-lock region to measure
879 mem_r = READ_ONCE(plr->kmem); in measure_cycles_lat_fn()
885 for (i = 0; i < plr->size; i += 32) { in measure_cycles_lat_fn()
892 trace_pseudo_lock_mem_latency((u32)(end - start)); in measure_cycles_lat_fn()
896 plr->thread_done = 1; in measure_cycles_lat_fn()
897 wake_up_interruptible(&plr->lock_thread_wq); in measure_cycles_lat_fn()
945 miss_event = perf_event_create_kernel_counter(miss_attr, plr->cpu, in measure_residency_fn()
950 hit_event = perf_event_create_kernel_counter(hit_attr, plr->cpu, in measure_residency_fn()
977 * interrupts disabled - it is thus safe to read the counter index. in measure_residency_fn()
981 line_size = READ_ONCE(plr->line_size); in measure_residency_fn()
982 mem_r = READ_ONCE(plr->kmem); in measure_residency_fn()
983 size = READ_ONCE(plr->size); in measure_residency_fn()
986 * Read counter variables twice - first to load the instructions in measure_residency_fn()
993 * From SDM: Performing back-to-back fast reads are not guaranteed in measure_residency_fn()
1029 /* Re-enable hardware prefetchers */ in measure_residency_fn()
1040 counts->miss_before = miss_before; in measure_residency_fn()
1041 counts->hits_before = hits_before; in measure_residency_fn()
1042 counts->miss_after = miss_after; in measure_residency_fn()
1043 counts->hits_after = hits_after; in measure_residency_fn()
1053 * Non-architectural event for the Goldmont Microarchitecture in measure_l2_residency()
1077 trace_pseudo_lock_l2(counts.hits_after - counts.hits_before, in measure_l2_residency()
1078 counts.miss_after - counts.miss_before); in measure_l2_residency()
1080 plr->thread_done = 1; in measure_l2_residency()
1081 wake_up_interruptible(&plr->lock_thread_wq); in measure_l2_residency()
1117 counts.miss_after -= counts.miss_before; in measure_l3_residency()
1127 counts.hits_after -= counts.hits_before; in measure_l3_residency()
1129 counts.hits_after -= min(counts.miss_after, counts.hits_after); in measure_l3_residency()
1131 counts.hits_after -= counts.hits_before; in measure_l3_residency()
1136 plr->thread_done = 1; in measure_l3_residency()
1137 wake_up_interruptible(&plr->lock_thread_wq); in measure_l3_residency()
1142 * pseudo_lock_measure_cycles - Trigger latency measure to pseudo-locked region
1144 * The measurement of latency to access a pseudo-locked region should be
1145 * done from a cpu that is associated with that pseudo-locked region.
1153 struct pseudo_lock_region *plr = rdtgrp->plr; in pseudo_lock_measure_cycles()
1156 int ret = -1; in pseudo_lock_measure_cycles()
1161 if (rdtgrp->flags & RDT_DELETED) { in pseudo_lock_measure_cycles()
1162 ret = -ENODEV; in pseudo_lock_measure_cycles()
1166 if (!plr->d) { in pseudo_lock_measure_cycles()
1167 ret = -ENODEV; in pseudo_lock_measure_cycles()
1171 plr->thread_done = 0; in pseudo_lock_measure_cycles()
1172 cpu = cpumask_first(&plr->d->cpu_mask); in pseudo_lock_measure_cycles()
1174 ret = -ENODEV; in pseudo_lock_measure_cycles()
1178 plr->cpu = cpu; in pseudo_lock_measure_cycles()
1205 ret = wait_event_interruptible(plr->lock_thread_wq, in pseudo_lock_measure_cycles()
1206 plr->thread_done == 1); in pseudo_lock_measure_cycles()
1222 struct rdtgroup *rdtgrp = file->private_data; in pseudo_lock_measure_trigger()
1228 buf_size = min(count, (sizeof(buf) - 1)); in pseudo_lock_measure_trigger()
1230 return -EFAULT; in pseudo_lock_measure_trigger()
1236 return -EINVAL; in pseudo_lock_measure_trigger()
1237 ret = debugfs_file_get(file->f_path.dentry); in pseudo_lock_measure_trigger()
1243 debugfs_file_put(file->f_path.dentry); in pseudo_lock_measure_trigger()
1256 * rdtgroup_pseudo_lock_create - Create a pseudo-locked region
1257 * @rdtgrp: resource group to which pseudo-lock region belongs
1259 * Called when a resource group in the pseudo-locksetup mode receives a
1260 * valid schemata that should be pseudo-locked. Since the resource group is
1261 * in pseudo-locksetup mode the &struct pseudo_lock_region has already been
1263 * occurs the resource group remains in the pseudo-locksetup mode with the
1265 * information and ready for the user to re-attempt pseudo-locking by
1268 * Return: 0 if the pseudo-locked region was successfully pseudo-locked, <0
1269 * on failure. Descriptive error will be written to last_cmd_status buffer.
1273 struct pseudo_lock_region *plr = rdtgrp->plr; in rdtgroup_pseudo_lock_create()
1285 ret = -EINVAL; in rdtgroup_pseudo_lock_create()
1289 plr->thread_done = 0; in rdtgroup_pseudo_lock_create()
1292 cpu_to_node(plr->cpu), in rdtgroup_pseudo_lock_create()
1293 "pseudo_lock/%u", plr->cpu); in rdtgroup_pseudo_lock_create()
1300 kthread_bind(thread, plr->cpu); in rdtgroup_pseudo_lock_create()
1303 ret = wait_event_interruptible(plr->lock_thread_wq, in rdtgroup_pseudo_lock_create()
1304 plr->thread_done == 1); in rdtgroup_pseudo_lock_create()
1312 * empty pseudo-locking loop. in rdtgroup_pseudo_lock_create()
1326 * pseudo-locked region will still be here on return. in rdtgroup_pseudo_lock_create()
1329 * deadlock with the mm->mmap_lock which is obtained in the in rdtgroup_pseudo_lock_create()
1336 plr->debugfs_dir = debugfs_create_dir(rdtgrp->kn->name, in rdtgroup_pseudo_lock_create()
1338 if (!IS_ERR_OR_NULL(plr->debugfs_dir)) in rdtgroup_pseudo_lock_create()
1340 plr->debugfs_dir, rdtgrp, in rdtgroup_pseudo_lock_create()
1346 rdtgrp, "%s", rdtgrp->kn->name); in rdtgroup_pseudo_lock_create()
1357 /* We released the mutex - check if group was removed while we did so */ in rdtgroup_pseudo_lock_create()
1358 if (rdtgrp->flags & RDT_DELETED) { in rdtgroup_pseudo_lock_create()
1359 ret = -ENODEV; in rdtgroup_pseudo_lock_create()
1363 plr->minor = new_minor; in rdtgroup_pseudo_lock_create()
1365 rdtgrp->mode = RDT_MODE_PSEUDO_LOCKED; in rdtgroup_pseudo_lock_create()
1366 closid_free(rdtgrp->closid); in rdtgroup_pseudo_lock_create()
1376 debugfs_remove_recursive(plr->debugfs_dir); in rdtgroup_pseudo_lock_create()
1387 * rdtgroup_pseudo_lock_remove - Remove a pseudo-locked region
1388 * @rdtgrp: resource group to which the pseudo-locked region belongs
1390 * The removal of a pseudo-locked region can be initiated when the resource
1393 * not go back to pseudo-locksetup mode before it is removed, instead it is
1402 struct pseudo_lock_region *plr = rdtgrp->plr; in rdtgroup_pseudo_lock_remove()
1404 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { in rdtgroup_pseudo_lock_remove()
1406 * Default group cannot be a pseudo-locked region so we can in rdtgroup_pseudo_lock_remove()
1409 closid_free(rdtgrp->closid); in rdtgroup_pseudo_lock_remove()
1414 debugfs_remove_recursive(rdtgrp->plr->debugfs_dir); in rdtgroup_pseudo_lock_remove()
1415 device_destroy(pseudo_lock_class, MKDEV(pseudo_lock_major, plr->minor)); in rdtgroup_pseudo_lock_remove()
1416 pseudo_lock_minor_release(plr->minor); in rdtgroup_pseudo_lock_remove()
1431 return -ENODEV; in pseudo_lock_dev_open()
1434 filp->private_data = rdtgrp; in pseudo_lock_dev_open()
1435 atomic_inc(&rdtgrp->waitcount); in pseudo_lock_dev_open()
1436 /* Perform a non-seekable open - llseek is not supported */ in pseudo_lock_dev_open()
1437 filp->f_mode &= ~(FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE); in pseudo_lock_dev_open()
1449 rdtgrp = filp->private_data; in pseudo_lock_dev_release()
1453 return -ENODEV; in pseudo_lock_dev_release()
1455 filp->private_data = NULL; in pseudo_lock_dev_release()
1456 atomic_dec(&rdtgrp->waitcount); in pseudo_lock_dev_release()
1464 return -EINVAL; in pseudo_lock_dev_mremap()
1473 unsigned long vsize = vma->vm_end - vma->vm_start; in pseudo_lock_dev_mmap()
1474 unsigned long off = vma->vm_pgoff << PAGE_SHIFT; in pseudo_lock_dev_mmap()
1482 rdtgrp = filp->private_data; in pseudo_lock_dev_mmap()
1486 return -ENODEV; in pseudo_lock_dev_mmap()
1489 plr = rdtgrp->plr; in pseudo_lock_dev_mmap()
1491 if (!plr->d) { in pseudo_lock_dev_mmap()
1493 return -ENODEV; in pseudo_lock_dev_mmap()
1498 * with the pseudo-locked region. If this is not the case the task in pseudo_lock_dev_mmap()
1500 * pseudo-locked region. in pseudo_lock_dev_mmap()
1502 if (!cpumask_subset(current->cpus_ptr, &plr->d->cpu_mask)) { in pseudo_lock_dev_mmap()
1504 return -EINVAL; in pseudo_lock_dev_mmap()
1507 physical = __pa(plr->kmem) >> PAGE_SHIFT; in pseudo_lock_dev_mmap()
1508 psize = plr->size - off; in pseudo_lock_dev_mmap()
1510 if (off > plr->size) { in pseudo_lock_dev_mmap()
1512 return -ENOSPC; in pseudo_lock_dev_mmap()
1517 * do not allow copy-on-write mapping. in pseudo_lock_dev_mmap()
1519 if (!(vma->vm_flags & VM_SHARED)) { in pseudo_lock_dev_mmap()
1521 return -EINVAL; in pseudo_lock_dev_mmap()
1526 return -ENOSPC; in pseudo_lock_dev_mmap()
1529 memset(plr->kmem + off, 0, vsize); in pseudo_lock_dev_mmap()
1531 if (remap_pfn_range(vma, vma->vm_start, physical + vma->vm_pgoff, in pseudo_lock_dev_mmap()
1532 vsize, vma->vm_page_prot)) { in pseudo_lock_dev_mmap()
1534 return -EAGAIN; in pseudo_lock_dev_mmap()
1536 vma->vm_ops = &pseudo_mmap_ops; in pseudo_lock_dev_mmap()
1558 return kasprintf(GFP_KERNEL, "pseudo_lock/%s", rdtgrp->kn->name); in pseudo_lock_devnode()
1578 pseudo_lock_class->devnode = pseudo_lock_devnode; in rdt_pseudo_lock_init()