| /linux/include/asm-generic/ |
| H A D | pgalloc.h | 21 struct ptdesc *ptdesc = pagetable_alloc_noprof(GFP_PGTABLE_KERNEL & in __pte_alloc_one_kernel_noprof() local
24 if (!ptdesc) in __pte_alloc_one_kernel_noprof()
26 if (!pagetable_pte_ctor(mm, ptdesc)) { in __pte_alloc_one_kernel_noprof()
27 pagetable_free(ptdesc); in __pte_alloc_one_kernel_noprof()
31 return ptdesc_address(ptdesc); in __pte_alloc_one_kernel_noprof()
73 struct ptdesc *ptdesc; in __pte_alloc_one_noprof() local
75 ptdesc = pagetable_alloc_noprof(gfp, 0); in __pte_alloc_one_noprof()
76 if (!ptdesc) in __pte_alloc_one_noprof()
78 if (!pagetable_pte_ctor(mm, ptdesc)) { in __pte_alloc_one_noprof()
79 pagetable_free(ptdesc); in __pte_alloc_one_noprof()
[all …]
|
| H A D | tlb.h | 218 struct ptdesc *ptdesc = (struct ptdesc *)table; in __tlb_remove_table() local
220 pagetable_dtor_free(ptdesc); in __tlb_remove_table()
235 struct ptdesc *ptdesc = (struct ptdesc *)table; in tlb_remove_table() local
237 pagetable_dtor(ptdesc); in tlb_remove_table()
238 tlb_remove_page(tlb, ptdesc_page(ptdesc)); in tlb_remove_table()
506 static inline void tlb_remove_ptdesc(struct mmu_gather *tlb, struct ptdesc *pt) in tlb_remove_ptdesc()
|
| /linux/arch/powerpc/mm/ |
| H A D | pgtable-frag.c | 21 struct ptdesc *ptdesc; in pte_frag_destroy() local
23 ptdesc = virt_to_ptdesc(pte_frag); in pte_frag_destroy()
27 if (atomic_sub_and_test(PTE_FRAG_NR - count, &ptdesc->pt_frag_refcount)) { in pte_frag_destroy()
28 pagetable_dtor(ptdesc); in pte_frag_destroy()
29 pagetable_free(ptdesc); in pte_frag_destroy()
58 struct ptdesc *ptdesc; in __alloc_for_ptecache() local
64 ptdesc = pagetable_alloc(gfp, 0); in __alloc_for_ptecache()
65 if (!ptdesc) in __alloc_for_ptecache()
67 if (!pagetable_pte_ctor(mm, ptdesc)) { in __alloc_for_ptecache()
68 pagetable_free(ptdesc); in __alloc_for_ptecache()
[all …]
|
| /linux/arch/m68k/include/asm/ |
| H A D | mcf_pgalloc.h | 17 struct ptdesc *ptdesc = pagetable_alloc((GFP_DMA | __GFP_ZERO) & in pte_alloc_one_kernel() local
20 if (!ptdesc) in pte_alloc_one_kernel()
22 if (!pagetable_pte_ctor(mm, ptdesc)) { in pte_alloc_one_kernel()
23 pagetable_free(ptdesc); in pte_alloc_one_kernel()
27 return ptdesc_address(ptdesc); in pte_alloc_one_kernel()
42 struct ptdesc *ptdesc = virt_to_ptdesc(pgtable); in __pte_free_tlb() local
44 pagetable_dtor(ptdesc); in __pte_free_tlb()
45 pagetable_free(ptdesc); in __pte_free_tlb()
50 struct ptdesc *ptdesc = pagetable_alloc(GFP_DMA | __GFP_ZERO, 0); in pte_alloc_one() local
53 if (!ptdesc) in pte_alloc_one()
[all …]
|
| /linux/arch/s390/mm/ |
| H A D | pgalloc.c | 20 struct ptdesc *ptdesc; in crst_table_alloc_noprof() local
25 ptdesc = pagetable_alloc_noprof(gfp, CRST_ALLOC_ORDER); in crst_table_alloc_noprof()
26 if (!ptdesc) in crst_table_alloc_noprof()
28 table = ptdesc_address(ptdesc); in crst_table_alloc_noprof()
119 struct ptdesc *page_table_alloc_pgste_noprof(struct mm_struct *mm) in page_table_alloc_pgste_noprof()
121 struct ptdesc *ptdesc; in page_table_alloc_pgste_noprof() local
124 ptdesc = pagetable_alloc_noprof(GFP_KERNEL_ACCOUNT, 0); in page_table_alloc_pgste_noprof()
125 if (ptdesc) { in page_table_alloc_pgste_noprof()
126 table = (u64 *)ptdesc_address(ptdesc); in page_table_alloc_pgste_noprof()
131 return ptdesc; in page_table_alloc_pgste_noprof()
[all …]
|
| H A D | gmap.c | 1200 struct ptdesc *ptdesc; in gmap_unshadow_pgt() local
1213 ptdesc = page_ptdesc(phys_to_page(pgt)); in gmap_unshadow_pgt()
1214 page_table_free_pgste(ptdesc); in gmap_unshadow_pgt()
1228 struct ptdesc *ptdesc; in __gmap_unshadow_sgt() local
1240 ptdesc = page_ptdesc(phys_to_page(pgt)); in __gmap_unshadow_sgt()
1241 page_table_free_pgste(ptdesc); in __gmap_unshadow_sgt()
1696 static void gmap_pgste_set_pgt_addr(struct ptdesc *ptdesc, unsigned long pgt_addr) in gmap_pgste_set_pgt_addr() argument
1698 unsigned long *pgstes = page_to_virt(ptdesc_page(ptdesc)); in gmap_pgste_set_pgt_addr()
1731 struct ptdesc *ptdesc; in gmap_shadow_pgt() local
1737 ptdesc = page_table_alloc_pgste(sg->mm); in gmap_shadow_pgt()
[all …]
|
| /linux/arch/mips/include/asm/ |
| H A D | pgalloc.h | 59 struct ptdesc *ptdesc; in pmd_alloc_one() local
61 ptdesc = pagetable_alloc(GFP_KERNEL_ACCOUNT, PMD_TABLE_ORDER); in pmd_alloc_one()
62 if (!ptdesc) in pmd_alloc_one()
65 if (!pagetable_pmd_ctor(mm, ptdesc)) { in pmd_alloc_one()
66 pagetable_free(ptdesc); in pmd_alloc_one()
70 pmd = ptdesc_address(ptdesc); in pmd_alloc_one()
84 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, in pud_alloc_one() local
87 if (!ptdesc) in pud_alloc_one()
89 pagetable_pud_ctor(ptdesc); in pud_alloc_one()
90 pud = ptdesc_address(ptdesc); in pud_alloc_one()
|
| /linux/arch/arm64/include/asm/ |
| H A D | tlb.h | 78 struct ptdesc *ptdesc = page_ptdesc(pte); in __pte_free_tlb() local
80 tlb_remove_ptdesc(tlb, ptdesc); in __pte_free_tlb()
87 struct ptdesc *ptdesc = virt_to_ptdesc(pmdp); in __pmd_free_tlb() local
89 tlb_remove_ptdesc(tlb, ptdesc); in __pmd_free_tlb()
97 struct ptdesc *ptdesc = virt_to_ptdesc(pudp); in __pud_free_tlb() local
102 tlb_remove_ptdesc(tlb, ptdesc); in __pud_free_tlb()
110 struct ptdesc *ptdesc = virt_to_ptdesc(p4dp); in __p4d_free_tlb() local
115 tlb_remove_ptdesc(tlb, ptdesc); in __p4d_free_tlb()
|
| /linux/arch/loongarch/include/asm/ |
| H A D | pgalloc.h | 66 struct ptdesc *ptdesc; in pmd_alloc_one() local
68 ptdesc = pagetable_alloc(GFP_KERNEL_ACCOUNT, 0); in pmd_alloc_one()
69 if (!ptdesc) in pmd_alloc_one()
72 if (!pagetable_pmd_ctor(mm, ptdesc)) { in pmd_alloc_one()
73 pagetable_free(ptdesc); in pmd_alloc_one()
77 pmd = ptdesc_address(ptdesc); in pmd_alloc_one()
91 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL, 0); in pud_alloc_one() local
93 if (!ptdesc) in pud_alloc_one()
95 pagetable_pud_ctor(ptdesc); in pud_alloc_one()
96 pud = ptdesc_address(ptdesc); in pud_alloc_one()
|
| /linux/arch/parisc/include/asm/ |
| H A D | pgalloc.h | 34 struct ptdesc *ptdesc; in pmd_alloc_one() local
39 ptdesc = pagetable_alloc(gfp, PMD_TABLE_ORDER); in pmd_alloc_one()
40 if (!ptdesc) in pmd_alloc_one()
42 if (!pagetable_pmd_ctor(mm, ptdesc)) { in pmd_alloc_one()
43 pagetable_free(ptdesc); in pmd_alloc_one()
46 return ptdesc_address(ptdesc); in pmd_alloc_one()
|
| /linux/arch/arm/include/asm/ |
| H A D | tlb.h | 35 struct ptdesc *ptdesc = page_ptdesc(pte); in __pte_free_tlb() local
46 tlb_remove_ptdesc(tlb, ptdesc); in __pte_free_tlb()
53 struct ptdesc *ptdesc = virt_to_ptdesc(pmdp); in __pmd_free_tlb() local
55 tlb_remove_ptdesc(tlb, ptdesc); in __pmd_free_tlb()
|
| /linux/include/linux/ |
| H A D | mm_types.h | 546 struct ptdesc { struct
584 static_assert(offsetof(struct page, pg) == offsetof(struct ptdesc, pt)) argument
597 static_assert(sizeof(struct ptdesc) <= sizeof(struct page));
600 const struct ptdesc *: (const struct page *)(pt), \
601 struct ptdesc *: (struct page *)(pt)))
604 const struct ptdesc *: (const struct folio *)(pt), \
605 struct ptdesc *: (struct folio *)(pt)))
608 const struct page *: (const struct ptdesc *)(p), \
609 struct page *: (struct ptdesc *)(p)))
612 static inline void ptdesc_pmd_pts_init(struct ptdesc *ptdesc) in ptdesc_pmd_pts_init() argument
[all …]
|
| H A D | mm.h | 2947 static inline struct ptdesc *virt_to_ptdesc(const void *x) in virt_to_ptdesc()
2958 static inline void *ptdesc_address(const struct ptdesc *pt) in ptdesc_address()
2963 static inline bool pagetable_is_reserved(struct ptdesc *pt) in pagetable_is_reserved()
2978 static inline struct ptdesc *pagetable_alloc_noprof(gfp_t gfp, unsigned int order) in pagetable_alloc_noprof()
2993 static inline void pagetable_free(struct ptdesc *pt) in pagetable_free()
3003 bool ptlock_alloc(struct ptdesc *ptdesc);
3004 void ptlock_free(struct ptdesc *ptdesc);
3006 static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc) in ptlock_ptr() argument
3008 return ptdesc->ptl; in ptlock_ptr()
3015 static inline bool ptlock_alloc(struct ptdesc *ptdesc) in ptlock_alloc() argument
[all …]
|
| /linux/arch/powerpc/mm/book3s64/ |
| H A D | pgtable.c | 410 struct ptdesc *ptdesc; in __alloc_for_pmdcache() local
415 ptdesc = pagetable_alloc(gfp, 0); in __alloc_for_pmdcache()
416 if (!ptdesc) in __alloc_for_pmdcache()
418 if (!pagetable_pmd_ctor(mm, ptdesc)) { in __alloc_for_pmdcache()
419 pagetable_free(ptdesc); in __alloc_for_pmdcache()
423 atomic_set(&ptdesc->pt_frag_refcount, 1); in __alloc_for_pmdcache()
425 ret = ptdesc_address(ptdesc); in __alloc_for_pmdcache()
440 atomic_set(&ptdesc->pt_frag_refcount, PMD_FRAG_NR); in __alloc_for_pmdcache()
461 struct ptdesc *ptdesc = virt_to_ptdesc(pmd); in pmd_fragment_free() local
463 if (pagetable_is_reserved(ptdesc)) in pmd_fragment_free()
[all …]
|
| H A D | mmu_context.c | 249 struct ptdesc *ptdesc; in pmd_frag_destroy() local
251 ptdesc = virt_to_ptdesc(pmd_frag); in pmd_frag_destroy()
255 if (atomic_sub_and_test(PMD_FRAG_NR - count, &ptdesc->pt_frag_refcount)) { in pmd_frag_destroy()
256 pagetable_dtor(ptdesc); in pmd_frag_destroy()
257 pagetable_free(ptdesc); in pmd_frag_destroy()
|
| /linux/arch/x86/mm/ |
| H A D | pgtable.c | 60 struct ptdesc *ptdesc = virt_to_ptdesc(pgd); in pgd_list_add() local
62 list_add(&ptdesc->pt_list, &pgd_list); in pgd_list_add()
67 struct ptdesc *ptdesc = virt_to_ptdesc(pgd); in pgd_list_del() local
69 list_del(&ptdesc->pt_list); in pgd_list_del()
164 struct ptdesc *ptdesc; in free_pmds() local
168 ptdesc = virt_to_ptdesc(pmds[i]); in free_pmds()
170 pagetable_dtor(ptdesc); in free_pmds()
171 pagetable_free(ptdesc); in free_pmds()
188 struct ptdesc *ptdesc = pagetable_alloc(gfp, 0); in preallocate_pmds() local
190 if (!ptdesc) in preallocate_pmds()
[all …]
|
| /linux/arch/m68k/mm/ |
| H A D | motorola.c | 106 #define PD_PTABLE(ptdesc) ((ptable_desc *)&(virt_to_ptdesc((void *)(ptdesc))->pt_list)) argument
107 #define PD_PTDESC(ptable) (list_entry(ptable, struct ptdesc, pt_list))
155 struct ptdesc *ptdesc; in get_pointer_table() local
159 ptdesc = pagetable_alloc(GFP_KERNEL | __GFP_ZERO, 0); in get_pointer_table()
160 if (!ptdesc) in get_pointer_table()
163 pt_addr = ptdesc_address(ptdesc); in get_pointer_table()
171 pagetable_pte_ctor(mm, ptdesc); in get_pointer_table()
174 pagetable_pmd_ctor(mm, ptdesc); in get_pointer_table()
177 pagetable_pgd_ctor(ptdesc); in get_pointer_table()
|
| /linux/mm/ |
| H A D | mmu_gather.c | 325 struct ptdesc *ptdesc; in __tlb_remove_table_one_rcu() local
327 ptdesc = container_of(head, struct ptdesc, pt_rcu_head); in __tlb_remove_table_one_rcu()
328 __tlb_remove_table(ptdesc); in __tlb_remove_table_one_rcu()
333 struct ptdesc *ptdesc; in __tlb_remove_table_one() local
335 ptdesc = table; in __tlb_remove_table_one()
336 call_rcu(&ptdesc->pt_rcu_head, __tlb_remove_table_one_rcu); in __tlb_remove_table_one()
|
| H A D | memory.c | 7298 bool ptlock_alloc(struct ptdesc *ptdesc) in ptlock_alloc() argument
7305 ptdesc->ptl = ptl; in ptlock_alloc()
7309 void ptlock_free(struct ptdesc *ptdesc) in ptlock_free() argument
7311 if (ptdesc->ptl) in ptlock_free()
7312 kmem_cache_free(page_ptl_cachep, ptdesc->ptl); in ptlock_free()
|
| /linux/arch/riscv/mm/ |
| H A D | init.c | 444 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, 0); in alloc_pte_late() local
451 BUG_ON(!ptdesc || !pagetable_pte_ctor(NULL, ptdesc)); in alloc_pte_late()
452 return __pa((pte_t *)ptdesc_address(ptdesc)); in alloc_pte_late()
529 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, 0); in alloc_pmd_late() local
532 BUG_ON(!ptdesc || !pagetable_pmd_ctor(NULL, ptdesc)); in alloc_pmd_late()
533 return __pa((pmd_t *)ptdesc_address(ptdesc)); in alloc_pmd_late()
594 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL, 0); in alloc_pud_late() local
596 BUG_ON(!ptdesc); in alloc_pud_late()
597 pagetable_pud_ctor(ptdesc); in alloc_pud_late()
598 return __pa((pud_t *)ptdesc_address(ptdesc)); in alloc_pud_late()
[all …]
|
| /linux/arch/s390/include/asm/ |
| H A D | pgalloc.h | 30 struct ptdesc *page_table_alloc_pgste_noprof(struct mm_struct *mm);
32 void page_table_free_pgste(struct ptdesc *ptdesc);
|
| /linux/arch/arm64/mm/ |
| H A D | mmu.c | 479 struct ptdesc *ptdesc = pagetable_alloc(gfp & ~__GFP_ZERO, 0); in __pgd_pgtable_alloc() local
482 if (!ptdesc) in __pgd_pgtable_alloc()
485 pa = page_to_phys(ptdesc_page(ptdesc)); in __pgd_pgtable_alloc()
489 BUG_ON(!pagetable_pte_ctor(mm, ptdesc)); in __pgd_pgtable_alloc()
492 BUG_ON(!pagetable_pmd_ctor(mm, ptdesc)); in __pgd_pgtable_alloc()
495 pagetable_pud_ctor(ptdesc); in __pgd_pgtable_alloc()
498 pagetable_p4d_ctor(ptdesc); in __pgd_pgtable_alloc()
|
| /linux/arch/sparc/mm/ |
| H A D | init_64.c | 2883 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL | __GFP_ZERO, 0); in __pte_alloc_one() local
2885 if (!ptdesc) in __pte_alloc_one()
2887 if (!pagetable_pte_ctor(mm, ptdesc)) { in __pte_alloc_one()
2888 pagetable_free(ptdesc); in __pte_alloc_one()
2891 return ptdesc_address(ptdesc); in __pte_alloc_one()
2906 struct ptdesc *ptdesc = virt_to_ptdesc(pte); in __pte_free() local
2908 pagetable_dtor(ptdesc); in __pte_free()
2909 pagetable_free(ptdesc); in __pte_free()
|
| /linux/arch/arm/mm/ |
| H A D | mmu.c | 735 void *ptdesc = pagetable_alloc(GFP_PGTABLE_KERNEL & ~__GFP_HIGHMEM, in late_alloc() local
738 if (!ptdesc || !pagetable_pte_ctor(NULL, ptdesc)) in late_alloc()
740 return ptdesc_address(ptdesc); in late_alloc()
|
| /linux/arch/x86/mm/pat/ |
| H A D | set_memory.c | 407 struct ptdesc *ptdesc, *tmp; in cpa_collapse_large_pages() local
430 list_for_each_entry_safe(ptdesc, tmp, &pgtables, pt_list) { in cpa_collapse_large_pages()
431 list_del(&ptdesc->pt_list); in cpa_collapse_large_pages()
432 __free_page(ptdesc_page(ptdesc)); in cpa_collapse_large_pages()
|