include/asm/pgtable.h

1 /* SPDX-License-Identifier: GPL-2.0 */
9  *  Derived from "include/asm-i386/pgtable.h"
18 #include <linux/page-flags.h>
19 #include <linux/radix-tree.h>
56  * for zero-mapped memory areas etc..
81  * The vmalloc and module area will always be on the topmost area of the
85  * happen without trampolines and in addition the placement within a
90 #define VMALLOC_DEFAULT_SIZE	((512UL << 30) - MODULES_LEN)
111 #define KMSAN_VMALLOC_SIZE (VMALLOC_END - VMALLOC_START)
136  * I Page-Invalid Bit:    Page is not available for address-translation
137  * P Page-Protection Bit: Store access not possible for page
138  * C Change-bit override: HW is not required to set change bit
141  * |        P-table origin                              |      TT
145  * I Segment-Invalid Bit:    Segment is not available for address-translation
146  * C Common-Segment Bit:     Segment is not private (PoP 3-30)
147  * P Page-Protection Bit: Store access not possible for page
151  * |        S-table origin                             |   TF  TTTL
155  * I Segment-Invalid Bit:    Segment is not available for address-translation
165  * X Space-Switch event:
166  * G Segment-Invalid Bit:
167  * P Private-Space Bit:
168  * S Storage-Alteration:
170  * TL Table-Length:
182 #define _PAGE_NOEXEC	0x100		/* HW no-execute bit  */
183 #define _PAGE_PROTECT	0x200		/* HW read-only bit  */
204 #define _PAGE_SWP_EXCLUSIVE _PAGE_LARGE	/* SW pte exclusive swap bit */
212  * HW bit and all SW bits.
217  * handle_pte_fault uses pte_present and pte_none to find out the pte type
219  * distinguish present from not-present ptes. It is changed only with the page
223  * the pte hardware and software bits in the last 12 bits of a pte
224  * (. unassigned bit, x don't care, t swap type):
231  * swap				.11..ttttt.0
232  * prot-none, clean, old	.11.xx0000.1
233  * prot-none, clean, young	.11.xx0001.1
234  * prot-none, dirty, old	.11.xx0010.1
235  * prot-none, dirty, young	.11.xx0011.1
236  * read-only, clean, old	.11.xx0100.1
237  * read-only, clean, young	.01.xx0101.1
238  * read-only, dirty, old	.11.xx0110.1
239  * read-only, dirty, young	.01.xx0111.1
240  * read-write, clean, old	.11.xx1100.1
241  * read-write, clean, young	.01.xx1101.1
242  * read-write, dirty, old	.10.xx1110.1
243  * read-write, dirty, young	.00.xx1111.1
244  * HW-bits: R read-only, I invalid
245  * SW-bits: p present, y young, d dirty, r read, w write, s special,
253 /* Bits in the segment/region table address-space-control-element */
269 #define _REGION_ENTRY_NOEXEC	0x100	/* region no-execute bit	    */
291 #define _REGION3_ENTRY_COMM	0x0010	/* Common-Region, marks swap entry */
292 #define _REGION3_ENTRY_LARGE	0x0400	/* RTTE-format control, large page  */
305  * SW region present bit. For non-leaf region-third-table entries, bits 62-63
306  * indicate the TABLE LENGTH and both must be set to 1. But such entries
319 #define _SEGMENT_ENTRY_NOEXEC	0x100	/* segment no-execute bit	    */
329 #define _SEGMENT_ENTRY_COMM	0x0010	/* Common-Segment, marks swap entry */
330 #define _SEGMENT_ENTRY_LARGE	0x0400	/* STE-format control, large page */
342 /* Common bits in region and segment table entries, for swap entries */
343 #define _RST_ENTRY_COMM		0x0010	/* Common-Region/Segment, marks swap entry */
368 #define _REGION1_MASK	(~(_REGION1_SIZE - 1))
369 #define _REGION2_MASK	(~(_REGION2_SIZE - 1))
370 #define _REGION3_MASK	(~(_REGION3_SIZE - 1))
371 #define _SEGMENT_MASK	(~(_SEGMENT_SIZE - 1))
395  * Segment table and region3 table entry encoding
396  * (R = read-only, I = invalid, y = young bit):
398  * prot-none, clean, old	00..1...1...00
399  * prot-none, clean, young	01..1...1...00
400  * prot-none, dirty, old	10..1...1...00
401  * prot-none, dirty, young	11..1...1...00
402  * read-only, clean, old	00..1...1...01
403  * read-only, clean, young	01..1...0...01
404  * read-only, dirty, old	10..1...1...01
405  * read-only, dirty, young	11..1...0...01
406  * read-write, clean, old	00..1...1...11
407  * read-write, clean, young	01..1...0...11
408  * read-write, dirty, old	10..0...1...11
409  * read-write, dirty, young	11..0...0...11
411  * read-write, old segment table entries (origin!=0)
412  * HW-bits: R read-only, I invalid
413  * SW-bits: y young, d dirty, r read, w write
439  * A user page table pointer has the space-switch-event bit, the
440  * private-space-control bit and the storage-alteration-event-control
553 	return mm->context.asce_limit <= _REGION1_SIZE;  in mm_p4d_folded()
559 	return mm->context.asce_limit <= _REGION2_SIZE;  in mm_pud_folded()
565 	return mm->context.asce_limit <= _REGION3_SIZE;  in mm_pmd_folded()
572 	if (unlikely(mm->context.has_pgste))  in mm_has_pgste()
581 	if (unlikely(atomic_read(&mm->context.protected_count)))  in mm_is_protected()
629  * mapped shared zeropages and prevent new shared zeropages from getting
636 	if (!mm->context.allow_cow_sharing)  in mm_forbids_zeropage()
645 	if (mm->context.uses_skeys)  in mm_uses_skeys()
664  * cspg() - Compare and Swap and Purge (CSPG)
669  * Return: True if compare and swap was successful, otherwise false.
691  * crdte() - Compare and Replace DAT Table Entry
699  * Return: True if compare and replace was successful, otherwise false.
993  * usable for kernel address space mappings where fault driven dirty and
994  * young/old accounting is not supported, i.e _PAGE_PROTECT and _PAGE_INVALID
1077 	 * newprot for PAGE_NONE, PAGE_RO, PAGE_RX, PAGE_RW and PAGE_RWX  in pte_modify()
1083 	 * newprot for PAGE_RO, PAGE_RX, PAGE_RW and PAGE_RWX has the page  in pte_modify()
1157 	pto = __pa(ptep) & ~(PTRS_PER_PTE * sizeof(pte_t) - 1);  in __ptep_rdp()
1202  * This is hard to understand. ptep_get_and_clear and ptep_clear_flush
1223 	pte = ptep_xchg_direct(vma->vm_mm, addr, ptep, pte_mkold(pte));  in ptep_test_and_clear_young()
1258 	res = ptep_xchg_direct(vma->vm_mm, addr, ptep, __pte(_PAGE_INVALID));  in ptep_clear_flush()
1260 	if (mm_is_protected(vma->vm_mm) && pte_present(res))  in ptep_clear_flush()
1270  * full==1 and a simple pte_clear is enough. See tlb.h.
1296 	 * If something went wrong and the page could not be destroyed, or  in ptep_get_and_clear_full()
1316  * bits in the comparison. Those might change e.g. because of dirty and young
1340 	 * Therefore, only do the local TLB flush when RDP can be used, and the  in flush_tlb_fix_spurious_fault()
1359 	if (cpu_has_rdp() && !mm_has_pgste(vma->vm_mm) && pte_allow_rdp(*ptep, entry))  in ptep_set_access_flags()
1360 		ptep_reset_dat_prot(vma->vm_mm, addr, ptep, entry);  in ptep_set_access_flags()
1362 		ptep_xchg_direct(vma->vm_mm, addr, ptep, entry);  in ptep_set_access_flags()
1411  * are within the same folio, PMD and VMA.
1421 			if (--nr == 0)  in set_ptes()
1430 			if (--nr == 0)  in set_ptes()
1440  * Conversion functions: convert a page and protection to a page entry,
1441  * and a page entry and page directory to the page they refer to.
1461 #define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
1462 #define p4d_index(address) (((address) >> P4D_SHIFT) & (PTRS_PER_P4D-1))
1463 #define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
1464 #define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))
1501  * The pgd_offset function *always* adds the index for the top-level
1504  *	pgdp = pgd_offset(current->mm, addr);
1508  * The subsequent p4d_offset, pud_offset and pmd_offset functions
1520 	return pgd + ((address >> shift) & (PTRS_PER_PGD - 1));  in pgd_offset_raw()
1523 #define pgd_offset(mm, address) pgd_offset_raw(READ_ONCE((mm)->pgd), address)
1573 	return end <= current->mm->context.asce_limit;  in gup_fast_permitted()
1718 	sto = __pa(pmdp) - pmd_index(addr) * sizeof(pmd_t);  in __pmdp_idte()
1744 	r3o = __pa(pudp) - pud_index(addr) * sizeof(pud_t);  in __pudp_idte()
1790 	pmdp_xchg_direct(vma->vm_mm, addr, pmdp, entry);  in pmdp_set_access_flags()
1800 	pmd = pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd_mkold(pmd));  in pmdp_test_and_clear_young()
1842 	return pmdp_xchg_lazy(vma->vm_mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));  in pmdp_huge_get_and_clear_full()
1849 	return pmdp_huge_get_and_clear(vma->vm_mm, addr, pmdp);  in pmdp_huge_clear_flush()
1860 	return pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd);  in pmdp_invalidate()
1877 	return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);  in pmdp_collapse_flush()
1897  * 64 bit swap entry format:
1898  * A page-table entry has some bits we have to treat in a special way.
1899  * Bits 54 and 63 are used to indicate the page type. Bit 53 marks the pte
1901  * A swap pte is indicated by bit pattern (pte & 0x201) == 0x200
1906  * Bits 0-51 store the offset.
1908  * Bits 57-61 store the type.
1910  * Bits 55 and 56 (X) are unused.
1913 #define __SWP_OFFSET_MASK	((1UL << 52) - 1)
1915 #define __SWP_TYPE_MASK		((1UL << 5) - 1)
1947  * 64 bit swap entry format for REGION3 and SEGMENT table entries (RSTE)
1948  * Bits 59 and 63 are used to indicate the swap entry. Bit 58 marks the rste
1950  * A swap entry is indicated by bit pattern (rste & 0x011) == 0x010
1955  * Bits 0-51 store the offset.
1956  * Bits 53-57 store the type.
1958  * Bits 60-61 (TT) indicate the table type: 0x01 for REGION3 and 0x00 for SEGMENT.
1962 #define __SWP_OFFSET_MASK_RSTE	((1UL << 52) - 1)
1964 #define __SWP_TYPE_MASK_RSTE		((1UL << 5) - 1)
2008 	((pgtable_t)__va(pmd_val(pmd) & -sizeof(pte_t)*PTRS_PER_PTE))