1 // SPDX-License-Identifier: GPL-2.0
5  * Pseudo-locking support built on top of Cache Allocation Technology (CAT)
26 #include <asm/intel-family.h>
44  * pseudo-locked regions.
56 	return kasprintf(GFP_KERNEL, "pseudo_lock/%s", rdtgrp->kn->name);  in pseudo_lock_devnode()
65  * get_prefetch_disable_bits - prefetch disable bits of supported platforms
69  * pseudo-locking. This includes testing to ensure pseudo-locked regions
71  * as well as that these pseudo-locked regions can maintain their low cache
74  * After a platform has been validated to support pseudo-locking its
120  * pseudo_lock_minor_get - Obtain available minor number
135 		return -ENOSPC;  in pseudo_lock_minor_get()
144  * pseudo_lock_minor_release - Return minor number to available
153  * region_find_by_minor - Locate a pseudo-lock region by inode minor number
154  * @minor: The minor number of the device representing pseudo-locked region
157  * pseudo-locked region it belongs to. This is done by matching the minor
158  * number of the device to the pseudo-locked region it belongs.
160  * Minor numbers are assigned at the time a pseudo-locked region is associated
163  * Return: On success return pointer to resource group owning the pseudo-locked
171 		if (rdtgrp->plr && rdtgrp->plr->minor == minor) {  in region_find_by_minor()
180  * struct pseudo_lock_pm_req - A power management QoS request list entry
181  * @list:	Entry within the @pm_reqs list for a pseudo-locked region
193 	list_for_each_entry_safe(pm_req, next, &plr->pm_reqs, list) {  in pseudo_lock_cstates_relax()
194 		dev_pm_qos_remove_request(&pm_req->req);  in pseudo_lock_cstates_relax()
195 		list_del(&pm_req->list);  in pseudo_lock_cstates_relax()
201  * pseudo_lock_cstates_constrain - Restrict cores from entering C6
202  * @plr: Pseudo-locked region
224 	for_each_cpu(cpu, &plr->d->cpu_mask) {  in pseudo_lock_cstates_constrain()
228 			ret = -ENOMEM;  in pseudo_lock_cstates_constrain()
232 					     &pm_req->req,  in pseudo_lock_cstates_constrain()
239 			ret = -1;  in pseudo_lock_cstates_constrain()
242 		list_add(&pm_req->list, &plr->pm_reqs);  in pseudo_lock_cstates_constrain()
253  * pseudo_lock_region_clear - Reset pseudo-lock region data
254  * @plr: pseudo-lock region
256  * All content of the pseudo-locked region is reset - any memory allocated
263 	plr->size = 0;  in pseudo_lock_region_clear()
264 	plr->line_size = 0;  in pseudo_lock_region_clear()
265 	kfree(plr->kmem);  in pseudo_lock_region_clear()
266 	plr->kmem = NULL;  in pseudo_lock_region_clear()
267 	plr->s = NULL;  in pseudo_lock_region_clear()
268 	if (plr->d)  in pseudo_lock_region_clear()
269 		plr->d->plr = NULL;  in pseudo_lock_region_clear()
270 	plr->d = NULL;  in pseudo_lock_region_clear()
271 	plr->cbm = 0;  in pseudo_lock_region_clear()
272 	plr->debugfs_dir = NULL;  in pseudo_lock_region_clear()
276  * pseudo_lock_region_init - Initialize pseudo-lock region information
277  * @plr: pseudo-lock region
279  * Called after user provided a schemata to be pseudo-locked. From the
282  * required for pseudo-locking is deduced from this data and &struct
284  * - size in bytes of the region to be pseudo-locked
285  * - cache line size to know the stride with which data needs to be accessed
286  *   to be pseudo-locked
287  * - a cpu associated with the cache instance on which the pseudo-locking
290  * Return: 0 on success, <0 on failure. Descriptive error will be written
300 	plr->cpu = cpumask_first(&plr->d->cpu_mask);  in pseudo_lock_region_init()
302 	if (!cpu_online(plr->cpu)) {  in pseudo_lock_region_init()
304 				    plr->cpu);  in pseudo_lock_region_init()
305 		ret = -ENODEV;  in pseudo_lock_region_init()
309 	ci = get_cpu_cacheinfo(plr->cpu);  in pseudo_lock_region_init()
311 	plr->size = rdtgroup_cbm_to_size(plr->s->res, plr->d, plr->cbm);  in pseudo_lock_region_init()
313 	for (i = 0; i < ci->num_leaves; i++) {  in pseudo_lock_region_init()
314 		if (ci->info_list[i].level == plr->s->res->cache_level) {  in pseudo_lock_region_init()
315 			plr->line_size = ci->info_list[i].coherency_line_size;  in pseudo_lock_region_init()
320 	ret = -1;  in pseudo_lock_region_init()
328  * pseudo_lock_init - Initialize a pseudo-lock region
329  * @rdtgrp: resource group to which new pseudo-locked region will belong
331  * A pseudo-locked region is associated with a resource group. When this
332  * association is created the pseudo-locked region is initialized. The
333  * details of the pseudo-locked region are not known at this time so only
344 		return -ENOMEM;  in pseudo_lock_init()
346 	init_waitqueue_head(&plr->lock_thread_wq);  in pseudo_lock_init()
347 	INIT_LIST_HEAD(&plr->pm_reqs);  in pseudo_lock_init()
348 	rdtgrp->plr = plr;  in pseudo_lock_init()
353  * pseudo_lock_region_alloc - Allocate kernel memory that will be pseudo-locked
354  * @plr: pseudo-lock region
356  * Initialize the details required to set up the pseudo-locked region and
357  * allocate the contiguous memory that will be pseudo-locked to the cache.
359  * Return: 0 on success, <0 on failure.  Descriptive error will be written
374 	if (plr->size > KMALLOC_MAX_SIZE) {  in pseudo_lock_region_alloc()
376 		ret = -E2BIG;  in pseudo_lock_region_alloc()
380 	plr->kmem = kzalloc(plr->size, GFP_KERNEL);  in pseudo_lock_region_alloc()
381 	if (!plr->kmem) {  in pseudo_lock_region_alloc()
383 		ret = -ENOMEM;  in pseudo_lock_region_alloc()
396  * pseudo_lock_free - Free a pseudo-locked region
397  * @rdtgrp: resource group to which pseudo-locked region belonged
399  * The pseudo-locked region's resources have already been released, or not
407 	pseudo_lock_region_clear(rdtgrp->plr);  in pseudo_lock_free()
408 	kfree(rdtgrp->plr);  in pseudo_lock_free()
409 	rdtgrp->plr = NULL;  in pseudo_lock_free()
413  * pseudo_lock_fn - Load kernel memory into cache
414  * @_rdtgrp: resource group to which pseudo-lock region belongs
416  * This is the core pseudo-locking flow.
421  * with class of service set to the bitmask of the pseudo-locked region.
434 	struct pseudo_lock_region *plr = rdtgrp->plr;  in pseudo_lock_fn()
445 	 * pseudo-locking success rate when KASAN is active.  in pseudo_lock_fn()
458 	 * will get a cache hit in below loop from outside of pseudo-locked  in pseudo_lock_fn()
477 	 * being pseudo-locked is reached the hardware will not read beyond  in pseudo_lock_fn()
478 	 * the buffer and evict pseudo-locked memory read earlier from the  in pseudo_lock_fn()
485 	mem_r = plr->kmem;  in pseudo_lock_fn()
486 	size = plr->size;  in pseudo_lock_fn()
487 	line_size = plr->line_size;  in pseudo_lock_fn()
491 	 * pseudo-locked followed by reading of kernel memory to load it  in pseudo_lock_fn()
494 	__wrmsr(MSR_IA32_PQR_ASSOC, rmid_p, rdtgrp->closid);  in pseudo_lock_fn()
497 	 * into cache region associated with just activated plr->closid.  in pseudo_lock_fn()
499 	 * - In first loop the cache region is shared with the page walker  in pseudo_lock_fn()
501 	 * - In the second loop the paging structure caches are used and  in pseudo_lock_fn()
528 	 * does not overlap with pseudo-locked region.  in pseudo_lock_fn()
532 	/* Re-enable the hardware prefetcher(s) */  in pseudo_lock_fn()
536 	plr->thread_done = 1;  in pseudo_lock_fn()
537 	wake_up_interruptible(&plr->lock_thread_wq);  in pseudo_lock_fn()
542  * rdtgroup_monitor_in_progress - Test if monitoring in progress
550 	return !list_empty(&rdtgrp->mon.crdtgrp_list);  in rdtgroup_monitor_in_progress()
554  * rdtgroup_locksetup_user_restrict - Restrict user access to group
557  * A resource group used for cache pseudo-locking cannot have cpus or tasks
604  * rdtgroup_locksetup_user_restore - Restore user access to group
651  * rdtgroup_locksetup_enter - Resource group enters locksetup mode
655  * to represent a pseudo-locked region and is in the process of being set
656  * up to do so. A resource group used for a pseudo-locked region would
659  * future. Monitoring of a pseudo-locked region is not allowed either.
661  * The above and more restrictions on a pseudo-locked region are checked
677 		rdt_last_cmd_puts("Cannot pseudo-lock default group\n");  in rdtgroup_locksetup_enter()
678 		return -EINVAL;  in rdtgroup_locksetup_enter()
682 	 * Cache Pseudo-locking not supported when CDP is enabled.  in rdtgroup_locksetup_enter()
686 	 * - When CDP is enabled two separate resources are exposed,  in rdtgroup_locksetup_enter()
688 	 *   The implication for pseudo-locking is that if a  in rdtgroup_locksetup_enter()
689 	 *   pseudo-locked region is created on a domain of one  in rdtgroup_locksetup_enter()
690 	 *   resource (eg. L3CODE), then a pseudo-locked region cannot  in rdtgroup_locksetup_enter()
693 	 *   pseudo-locked region involves a call to wbinvd that will  in rdtgroup_locksetup_enter()
695 	 * - Considering the previous, it may be possible to only  in rdtgroup_locksetup_enter()
696 	 *   expose one of the CDP resources to pseudo-locking and  in rdtgroup_locksetup_enter()
700 	 * - If only one region is exposed to pseudo-locking we should  in rdtgroup_locksetup_enter()
702 	 *   for pseudo-locking should take into account both resources.  in rdtgroup_locksetup_enter()
703 	 *   Similarly, if a pseudo-locked region is created in one  in rdtgroup_locksetup_enter()
710 		return -EINVAL;  in rdtgroup_locksetup_enter()
715 	 * platform does not support Cache Pseudo-Locking.  in rdtgroup_locksetup_enter()
719 		rdt_last_cmd_puts("Pseudo-locking not supported\n");  in rdtgroup_locksetup_enter()
720 		return -EINVAL;  in rdtgroup_locksetup_enter()
725 		return -EINVAL;  in rdtgroup_locksetup_enter()
730 		return -EINVAL;  in rdtgroup_locksetup_enter()
733 	if (!cpumask_empty(&rdtgrp->cpu_mask)) {  in rdtgroup_locksetup_enter()
735 		return -EINVAL;  in rdtgroup_locksetup_enter()
740 		return -EIO;  in rdtgroup_locksetup_enter()
745 		rdt_last_cmd_puts("Unable to init pseudo-lock region\n");  in rdtgroup_locksetup_enter()
752 	 * anymore when this group would be used for pseudo-locking. This  in rdtgroup_locksetup_enter()
755 	free_rmid(rdtgrp->mon.rmid);  in rdtgroup_locksetup_enter()
767  * rdtgroup_locksetup_exit - resource group exist locksetup mode
785 		rdtgrp->mon.rmid = ret;  in rdtgroup_locksetup_exit()
790 		free_rmid(rdtgrp->mon.rmid);  in rdtgroup_locksetup_exit()
799  * rdtgroup_cbm_overlaps_pseudo_locked - Test if CBM or portion is pseudo-locked
805  * pseudo-locked region on @d.
810  * Return: true if @cbm overlaps with pseudo-locked region on @d, false
818 	if (d->plr) {  in rdtgroup_cbm_overlaps_pseudo_locked()
819 		cbm_len = d->plr->s->res->cache.cbm_len;  in rdtgroup_cbm_overlaps_pseudo_locked()
820 		cbm_b = d->plr->cbm;  in rdtgroup_cbm_overlaps_pseudo_locked()
828  * rdtgroup_pseudo_locked_in_hierarchy - Pseudo-locked region in cache hierarchy
831  * The setup of a pseudo-locked region affects all cache instances within
833  * pseudo-locked regions exist within a cache hierarchy to prevent any
834  * attempts to create new pseudo-locked regions in the same hierarchy.
836  * Return: true if a pseudo-locked region exists in the hierarchy of @d or
851 	 * First determine which cpus have pseudo-locked regions  in rdtgroup_pseudo_locked_in_hierarchy()
855 		list_for_each_entry(d_i, &r->domains, list) {  in rdtgroup_pseudo_locked_in_hierarchy()
856 			if (d_i->plr)  in rdtgroup_pseudo_locked_in_hierarchy()
858 					   &d_i->cpu_mask);  in rdtgroup_pseudo_locked_in_hierarchy()
863 	 * Next test if new pseudo-locked region would intersect with  in rdtgroup_pseudo_locked_in_hierarchy()
866 	if (cpumask_intersects(&d->cpu_mask, cpu_with_psl))  in rdtgroup_pseudo_locked_in_hierarchy()
874  * measure_cycles_lat_fn - Measure cycle latency to read pseudo-locked memory
875  * @_plr: pseudo-lock region to measure
901 	mem_r = READ_ONCE(plr->kmem);  in measure_cycles_lat_fn()
907 	for (i = 0; i < plr->size; i += 32) {  in measure_cycles_lat_fn()
914 		trace_pseudo_lock_mem_latency((u32)(end - start));  in measure_cycles_lat_fn()
918 	plr->thread_done = 1;  in measure_cycles_lat_fn()
919 	wake_up_interruptible(&plr->lock_thread_wq);  in measure_cycles_lat_fn()
968 	miss_event = perf_event_create_kernel_counter(miss_attr, plr->cpu,  in measure_residency_fn()
973 	hit_event = perf_event_create_kernel_counter(hit_attr, plr->cpu,  in measure_residency_fn()
1001 	 * interrupts disabled - it is thus safe to read the counter index.  in measure_residency_fn()
1005 	line_size = READ_ONCE(plr->line_size);  in measure_residency_fn()
1006 	mem_r = READ_ONCE(plr->kmem);  in measure_residency_fn()
1007 	size = READ_ONCE(plr->size);  in measure_residency_fn()
1010 	 * Read counter variables twice - first to load the instructions  in measure_residency_fn()
1017 	 * From SDM: Performing back-to-back fast reads are not guaranteed  in measure_residency_fn()
1053 	/* Re-enable hardware prefetchers */  in measure_residency_fn()
1064 	counts->miss_before = miss_before;  in measure_residency_fn()
1065 	counts->hits_before = hits_before;  in measure_residency_fn()
1066 	counts->miss_after  = miss_after;  in measure_residency_fn()
1067 	counts->hits_after  = hits_after;  in measure_residency_fn()
1077 	 * Non-architectural event for the Goldmont Microarchitecture  in measure_l2_residency()
1101 	trace_pseudo_lock_l2(counts.hits_after - counts.hits_before,  in measure_l2_residency()
1102 			     counts.miss_after - counts.miss_before);  in measure_l2_residency()
1104 	plr->thread_done = 1;  in measure_l2_residency()
1105 	wake_up_interruptible(&plr->lock_thread_wq);  in measure_l2_residency()
1141 	counts.miss_after -= counts.miss_before;  in measure_l3_residency()
1151 		counts.hits_after -= counts.hits_before;  in measure_l3_residency()
1153 		counts.hits_after -= min(counts.miss_after, counts.hits_after);  in measure_l3_residency()
1155 		counts.hits_after -= counts.hits_before;  in measure_l3_residency()
1160 	plr->thread_done = 1;  in measure_l3_residency()
1161 	wake_up_interruptible(&plr->lock_thread_wq);  in measure_l3_residency()
1166  * pseudo_lock_measure_cycles - Trigger latency measure to pseudo-locked region
1167  * @rdtgrp: Resource group to which the pseudo-locked region belongs.
1168  * @sel: Selector of which measurement to perform on a pseudo-locked region.
1170  * The measurement of latency to access a pseudo-locked region should be
1171  * done from a cpu that is associated with that pseudo-locked region.
1179 	struct pseudo_lock_region *plr = rdtgrp->plr;  in pseudo_lock_measure_cycles()
1182 	int ret = -1;  in pseudo_lock_measure_cycles()
1187 	if (rdtgrp->flags & RDT_DELETED) {  in pseudo_lock_measure_cycles()
1188 		ret = -ENODEV;  in pseudo_lock_measure_cycles()
1192 	if (!plr->d) {  in pseudo_lock_measure_cycles()
1193 		ret = -ENODEV;  in pseudo_lock_measure_cycles()
1197 	plr->thread_done = 0;  in pseudo_lock_measure_cycles()
1198 	cpu = cpumask_first(&plr->d->cpu_mask);  in pseudo_lock_measure_cycles()
1200 		ret = -ENODEV;  in pseudo_lock_measure_cycles()
1204 	plr->cpu = cpu;  in pseudo_lock_measure_cycles()
1231 	ret = wait_event_interruptible(plr->lock_thread_wq,  in pseudo_lock_measure_cycles()
1232 				       plr->thread_done == 1);  in pseudo_lock_measure_cycles()
1248 	struct rdtgroup *rdtgrp = file->private_data;  in pseudo_lock_measure_trigger()
1254 	buf_size = min(count, (sizeof(buf) - 1));  in pseudo_lock_measure_trigger()
1256 		return -EFAULT;  in pseudo_lock_measure_trigger()
1262 			return -EINVAL;  in pseudo_lock_measure_trigger()
1263 		ret = debugfs_file_get(file->f_path.dentry);  in pseudo_lock_measure_trigger()
1269 		debugfs_file_put(file->f_path.dentry);  in pseudo_lock_measure_trigger()
1282  * rdtgroup_pseudo_lock_create - Create a pseudo-locked region
1283  * @rdtgrp: resource group to which pseudo-lock region belongs
1285  * Called when a resource group in the pseudo-locksetup mode receives a
1286  * valid schemata that should be pseudo-locked. Since the resource group is
1287  * in pseudo-locksetup mode the &struct pseudo_lock_region has already been
1289  * occurs the resource group remains in the pseudo-locksetup mode with the
1291  * information and ready for the user to re-attempt pseudo-locking by
1294  * Return: 0 if the pseudo-locked region was successfully pseudo-locked, <0
1295  * on failure. Descriptive error will be written to last_cmd_status buffer.
1299 	struct pseudo_lock_region *plr = rdtgrp->plr;  in rdtgroup_pseudo_lock_create()
1311 		ret = -EINVAL;  in rdtgroup_pseudo_lock_create()
1315 	plr->thread_done = 0;  in rdtgroup_pseudo_lock_create()
1318 					cpu_to_node(plr->cpu),  in rdtgroup_pseudo_lock_create()
1319 					"pseudo_lock/%u", plr->cpu);  in rdtgroup_pseudo_lock_create()
1326 	kthread_bind(thread, plr->cpu);  in rdtgroup_pseudo_lock_create()
1329 	ret = wait_event_interruptible(plr->lock_thread_wq,  in rdtgroup_pseudo_lock_create()
1330 				       plr->thread_done == 1);  in rdtgroup_pseudo_lock_create()
1338 		 * empty pseudo-locking loop.  in rdtgroup_pseudo_lock_create()
1352 	 * pseudo-locked region will still be here on return.  in rdtgroup_pseudo_lock_create()
1355 	 * deadlock with the mm->mmap_lock which is obtained in the  in rdtgroup_pseudo_lock_create()
1362 		plr->debugfs_dir = debugfs_create_dir(rdtgrp->kn->name,  in rdtgroup_pseudo_lock_create()
1364 		if (!IS_ERR_OR_NULL(plr->debugfs_dir))  in rdtgroup_pseudo_lock_create()
1366 					    plr->debugfs_dir, rdtgrp,  in rdtgroup_pseudo_lock_create()
1372 			    rdtgrp, "%s", rdtgrp->kn->name);  in rdtgroup_pseudo_lock_create()
1383 	/* We released the mutex - check if group was removed while we did so */  in rdtgroup_pseudo_lock_create()
1384 	if (rdtgrp->flags & RDT_DELETED) {  in rdtgroup_pseudo_lock_create()
1385 		ret = -ENODEV;  in rdtgroup_pseudo_lock_create()
1389 	plr->minor = new_minor;  in rdtgroup_pseudo_lock_create()
1391 	rdtgrp->mode = RDT_MODE_PSEUDO_LOCKED;  in rdtgroup_pseudo_lock_create()
1392 	closid_free(rdtgrp->closid);  in rdtgroup_pseudo_lock_create()
1402 	debugfs_remove_recursive(plr->debugfs_dir);  in rdtgroup_pseudo_lock_create()
1413  * rdtgroup_pseudo_lock_remove - Remove a pseudo-locked region
1414  * @rdtgrp: resource group to which the pseudo-locked region belongs
1416  * The removal of a pseudo-locked region can be initiated when the resource
1419  * not go back to pseudo-locksetup mode before it is removed, instead it is
1428 	struct pseudo_lock_region *plr = rdtgrp->plr;  in rdtgroup_pseudo_lock_remove()
1430 	if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {  in rdtgroup_pseudo_lock_remove()
1432 		 * Default group cannot be a pseudo-locked region so we can  in rdtgroup_pseudo_lock_remove()
1435 		closid_free(rdtgrp->closid);  in rdtgroup_pseudo_lock_remove()
1440 	debugfs_remove_recursive(rdtgrp->plr->debugfs_dir);  in rdtgroup_pseudo_lock_remove()
1441 	device_destroy(&pseudo_lock_class, MKDEV(pseudo_lock_major, plr->minor));  in rdtgroup_pseudo_lock_remove()
1442 	pseudo_lock_minor_release(plr->minor);  in rdtgroup_pseudo_lock_remove()
1457 		return -ENODEV;  in pseudo_lock_dev_open()
1460 	filp->private_data = rdtgrp;  in pseudo_lock_dev_open()
1461 	atomic_inc(&rdtgrp->waitcount);  in pseudo_lock_dev_open()
1462 	/* Perform a non-seekable open - llseek is not supported */  in pseudo_lock_dev_open()
1463 	filp->f_mode &= ~(FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE);  in pseudo_lock_dev_open()
1475 	rdtgrp = filp->private_data;  in pseudo_lock_dev_release()
1479 		return -ENODEV;  in pseudo_lock_dev_release()
1481 	filp->private_data = NULL;  in pseudo_lock_dev_release()
1482 	atomic_dec(&rdtgrp->waitcount);  in pseudo_lock_dev_release()
1490 	return -EINVAL;  in pseudo_lock_dev_mremap()
1499 	unsigned long vsize = vma->vm_end - vma->vm_start;  in pseudo_lock_dev_mmap()
1500 	unsigned long off = vma->vm_pgoff << PAGE_SHIFT;  in pseudo_lock_dev_mmap()
1508 	rdtgrp = filp->private_data;  in pseudo_lock_dev_mmap()
1512 		return -ENODEV;  in pseudo_lock_dev_mmap()
1515 	plr = rdtgrp->plr;  in pseudo_lock_dev_mmap()
1517 	if (!plr->d) {  in pseudo_lock_dev_mmap()
1519 		return -ENODEV;  in pseudo_lock_dev_mmap()
1524 	 * with the pseudo-locked region. If this is not the case the task  in pseudo_lock_dev_mmap()
1526 	 * pseudo-locked region.  in pseudo_lock_dev_mmap()
1528 	if (!cpumask_subset(current->cpus_ptr, &plr->d->cpu_mask)) {  in pseudo_lock_dev_mmap()
1530 		return -EINVAL;  in pseudo_lock_dev_mmap()
1533 	physical = __pa(plr->kmem) >> PAGE_SHIFT;  in pseudo_lock_dev_mmap()
1534 	psize = plr->size - off;  in pseudo_lock_dev_mmap()
1536 	if (off > plr->size) {  in pseudo_lock_dev_mmap()
1538 		return -ENOSPC;  in pseudo_lock_dev_mmap()
1543 	 * do not allow copy-on-write mapping.  in pseudo_lock_dev_mmap()
1545 	if (!(vma->vm_flags & VM_SHARED)) {  in pseudo_lock_dev_mmap()
1547 		return -EINVAL;  in pseudo_lock_dev_mmap()
1552 		return -ENOSPC;  in pseudo_lock_dev_mmap()
1555 	memset(plr->kmem + off, 0, vsize);  in pseudo_lock_dev_mmap()
1557 	if (remap_pfn_range(vma, vma->vm_start, physical + vma->vm_pgoff,  in pseudo_lock_dev_mmap()
1558 			    vsize, vma->vm_page_prot)) {  in pseudo_lock_dev_mmap()
1560 		return -EAGAIN;  in pseudo_lock_dev_mmap()
1562 	vma->vm_ops = &pseudo_mmap_ops;  in pseudo_lock_dev_mmap()