16 static int __check_vma(struct dev_dax *dev_dax, vma_flags_t flags,  in __check_vma()  argument
20 	struct device *dev = &dev_dax->dev;  in __check_vma()
23 	if (!dax_alive(dev_dax->dax_dev))  in __check_vma()
34 	mask = dev_dax->align - 1;  in __check_vma()
53 static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma,  in check_vma()  argument
56 	return __check_vma(dev_dax, vma->flags, vma->vm_start, vma->vm_end,  in check_vma()
61 __weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,  in dax_pgoff_to_phys()  argument
66 	for (i = 0; i < dev_dax->nr_range; i++) {  in dax_pgoff_to_phys()
67 		struct dev_dax_range *dax_range = &dev_dax->ranges[i];  in dax_pgoff_to_phys()
88 	struct dev_dax *dev_dax = filp->private_data;  in dax_set_mapping()  local
92 	if (dev_dax->pgmap->vmemmap_shift)  in dax_set_mapping()
109 static vm_fault_t __dev_dax_pte_fault(struct dev_dax *dev_dax,  in __dev_dax_pte_fault()  argument
112 	struct device *dev = &dev_dax->dev;  in __dev_dax_pte_fault()
117 	if (check_vma(dev_dax, vmf->vma, __func__))  in __dev_dax_pte_fault()
120 	if (dev_dax->align > PAGE_SIZE) {  in __dev_dax_pte_fault()
122 			dev_dax->align, fault_size);  in __dev_dax_pte_fault()
126 	if (fault_size != dev_dax->align)  in __dev_dax_pte_fault()
129 	phys = dax_pgoff_to_phys(dev_dax, vmf->pgoff, PAGE_SIZE);  in __dev_dax_pte_fault()
143 static vm_fault_t __dev_dax_pmd_fault(struct dev_dax *dev_dax,  in __dev_dax_pmd_fault()  argument
147 	struct device *dev = &dev_dax->dev;  in __dev_dax_pmd_fault()
153 	if (check_vma(dev_dax, vmf->vma, __func__))  in __dev_dax_pmd_fault()
156 	if (dev_dax->align > PMD_SIZE) {  in __dev_dax_pmd_fault()
158 			dev_dax->align, fault_size);  in __dev_dax_pmd_fault()
162 	if (fault_size < dev_dax->align)  in __dev_dax_pmd_fault()
164 	else if (fault_size > dev_dax->align)  in __dev_dax_pmd_fault()
173 	phys = dax_pgoff_to_phys(dev_dax, pgoff, PMD_SIZE);  in __dev_dax_pmd_fault()
188 static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,  in __dev_dax_pud_fault()  argument
192 	struct device *dev = &dev_dax->dev;  in __dev_dax_pud_fault()
199 	if (check_vma(dev_dax, vmf->vma, __func__))  in __dev_dax_pud_fault()
202 	if (dev_dax->align > PUD_SIZE) {  in __dev_dax_pud_fault()
204 			dev_dax->align, fault_size);  in __dev_dax_pud_fault()
208 	if (fault_size < dev_dax->align)  in __dev_dax_pud_fault()
210 	else if (fault_size > dev_dax->align)  in __dev_dax_pud_fault()
219 	phys = dax_pgoff_to_phys(dev_dax, pgoff, PUD_SIZE);  in __dev_dax_pud_fault()
233 static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,  in __dev_dax_pud_fault()  argument
245 	struct dev_dax *dev_dax = filp->private_data;  in dev_dax_huge_fault()  local
247 	dev_dbg(&dev_dax->dev, "%s: op=%s addr=%#lx order=%d\n", current->comm,  in dev_dax_huge_fault()
253 		rc = __dev_dax_pte_fault(dev_dax, vmf);  in dev_dax_huge_fault()
255 		rc = __dev_dax_pmd_fault(dev_dax, vmf);  in dev_dax_huge_fault()
257 		rc = __dev_dax_pud_fault(dev_dax, vmf);  in dev_dax_huge_fault()
274 	struct dev_dax *dev_dax = filp->private_data;  in dev_dax_may_split()  local
276 	if (!IS_ALIGNED(addr, dev_dax->align))  in dev_dax_may_split()
284 	struct dev_dax *dev_dax = filp->private_data;  in dev_dax_pagesize()  local
286 	return dev_dax->align;  in dev_dax_pagesize()
299 	struct dev_dax *dev_dax = filp->private_data;  in dax_mmap_prepare()  local
302 	dev_dbg(&dev_dax->dev, "trace\n");  in dax_mmap_prepare()
309 	rc = __check_vma(dev_dax, desc->vma_flags, desc->start, desc->end, filp,  in dax_mmap_prepare()
326 	struct dev_dax *dev_dax = filp ? filp->private_data : NULL;  in dax_get_unmapped_area()  local
328 	if (!dev_dax || addr)  in dax_get_unmapped_area()
331 	align = dev_dax->align;  in dax_get_unmapped_area()
360 	struct dev_dax *dev_dax = dax_get_private(dax_dev);  in dax_open()  local
362 	dev_dbg(&dev_dax->dev, "trace\n");  in dax_open()
369 	filp->private_data = dev_dax;  in dax_open()
377 	struct dev_dax *dev_dax = filp->private_data;  in dax_release()  local
379 	dev_dbg(&dev_dax->dev, "trace\n");  in dax_release()
398 static void dev_dax_kill(void *dev_dax)  in dev_dax_kill()  argument
400 	kill_dev_dax(dev_dax);  in dev_dax_kill()
403 static int dev_dax_probe(struct dev_dax *dev_dax)  in dev_dax_probe()  argument
405 	struct dax_device *dax_dev = dev_dax->dax_dev;  in dev_dax_probe()
406 	struct device *dev = &dev_dax->dev;  in dev_dax_probe()
413 	if (static_dev_dax(dev_dax))  {  in dev_dax_probe()
414 		if (dev_dax->nr_range > 1) {  in dev_dax_probe()
420 		pgmap = dev_dax->pgmap;  in dev_dax_probe()
422 		if (dev_dax->pgmap) {  in dev_dax_probe()
429                        struct_size(pgmap, ranges, dev_dax->nr_range - 1),  in dev_dax_probe()
434 		pgmap->nr_range = dev_dax->nr_range;  in dev_dax_probe()
435 		dev_dax->pgmap = pgmap;  in dev_dax_probe()
437 		for (i = 0; i < dev_dax->nr_range; i++) {  in dev_dax_probe()
438 			struct range *range = &dev_dax->ranges[i].range;  in dev_dax_probe()
443 	for (i = 0; i < dev_dax->nr_range; i++) {  in dev_dax_probe()
444 		struct range *range = &dev_dax->ranges[i].range;  in dev_dax_probe()
455 	if (dev_dax->align > PAGE_SIZE)  in dev_dax_probe()
457 			order_base_2(dev_dax->align >> PAGE_SHIFT);  in dev_dax_probe()
476 	return devm_add_action_or_reset(dev, dev_dax_kill, dev_dax);  in dev_dax_probe()