3  *  Page table allocation functions
64 void crst_table_free(struct mm_struct *mm, unsigned long *table)  in crst_table_free()  argument
66 	free_pages((unsigned long) table, 2);  in crst_table_free()
169 	u64 *table;  in page_table_alloc_pgste()  local
173 		table = (u64 *)page_to_phys(page);  in page_table_alloc_pgste()
174 		memset64(table, _PAGE_INVALID, PTRS_PER_PTE);  in page_table_alloc_pgste()
175 		memset64(table + PTRS_PER_PTE, 0, PTRS_PER_PTE);  in page_table_alloc_pgste()
188  * page table entry allocation/free routines.
192 	unsigned long *table;  in page_table_alloc()  local
196 	/* Try to get a fragment of a 4K page as a 2K page table */  in page_table_alloc()
198 		table = NULL;  in page_table_alloc()
206 				table = (unsigned long *) page_to_phys(page);  in page_table_alloc()
209 					table += PTRS_PER_PTE;  in page_table_alloc()
216 		if (table)  in page_table_alloc()
217 			return table;  in page_table_alloc()
228 	/* Initialize page table */  in page_table_alloc()
229 	table = (unsigned long *) page_to_phys(page);  in page_table_alloc()
231 		/* Return 4K page table with PGSTEs */  in page_table_alloc()
233 		memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE);  in page_table_alloc()
234 		memset64((u64 *)table + PTRS_PER_PTE, 0, PTRS_PER_PTE);  in page_table_alloc()
238 		memset64((u64 *)table, _PAGE_INVALID, 2 * PTRS_PER_PTE);  in page_table_alloc()
243 	return table;  in page_table_alloc()
246 void page_table_free(struct mm_struct *mm, unsigned long *table)  in page_table_free()  argument
251 	page = pfn_to_page(__pa(table) >> PAGE_SHIFT);  in page_table_free()
253 		/* Free 2K page table fragment of a 4K page */  in page_table_free()
254 		bit = (__pa(table) & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t));  in page_table_free()
273 void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table,  in page_table_free_rcu()  argument
281 	page = pfn_to_page(__pa(table) >> PAGE_SHIFT);  in page_table_free_rcu()
283 		gmap_unlink(mm, table, vmaddr);  in page_table_free_rcu()
284 		table = (unsigned long *) (__pa(table) | 3);  in page_table_free_rcu()
285 		tlb_remove_table(tlb, table);  in page_table_free_rcu()
288 	bit = (__pa(table) & ~PAGE_MASK) / (PTRS_PER_PTE*sizeof(pte_t));  in page_table_free_rcu()
297 	table = (unsigned long *) (__pa(table) | (1U << bit));  in page_table_free_rcu()
298 	tlb_remove_table(tlb, table);  in page_table_free_rcu()
304 	void *table = (void *)((unsigned long) _table ^ mask);  in __tlb_remove_table()  local
305 	struct page *page = pfn_to_page(__pa(table) >> PAGE_SHIFT);  in __tlb_remove_table()
309 		free_pages((unsigned long) table, 2);  in __tlb_remove_table()
311 	case 1:		/* lower 2K of a 4K page table */  in __tlb_remove_table()
312 	case 2:		/* higher 2K of a 4K page table */  in __tlb_remove_table()
318 	case 3:		/* 4K page table with pgstes */  in __tlb_remove_table()
336 	u64 *table;  in base_pgt_alloc()  local
338 	table = kmem_cache_alloc(base_pgt_cache, GFP_KERNEL);  in base_pgt_alloc()
339 	if (table)  in base_pgt_alloc()
340 		memset64(table, _PAGE_INVALID, PTRS_PER_PTE);  in base_pgt_alloc()
341 	return (unsigned long) table;  in base_pgt_alloc()
344 static void base_pgt_free(unsigned long table)  in base_pgt_free()  argument
346 	kmem_cache_free(base_pgt_cache, (void *) table);  in base_pgt_free()
351 	unsigned long table;  in base_crst_alloc()  local
353 	table =	 __get_free_pages(GFP_KERNEL, CRST_ALLOC_ORDER);  in base_crst_alloc()
354 	if (table)  in base_crst_alloc()
355 		crst_table_init((unsigned long *)table, val);  in base_crst_alloc()
356 	return table;  in base_crst_alloc()
359 static void base_crst_free(unsigned long table)  in base_crst_free()  argument
361 	free_pages(table, CRST_ALLOC_ORDER);  in base_crst_free()
408 	unsigned long *ste, next, table;  in base_segment_walk()  local
418 			table = base_pgt_alloc();  in base_segment_walk()
419 			if (!table)  in base_segment_walk()
421 			*ste = table | _SEGMENT_ENTRY;  in base_segment_walk()
423 		table = *ste & _SEGMENT_ENTRY_ORIGIN;  in base_segment_walk()
424 		rc = base_page_walk(table, addr, next, alloc);  in base_segment_walk()
428 			base_pgt_free(table);  in base_segment_walk()
437 	unsigned long *rtte, next, table;  in base_region3_walk()  local
447 			table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY);  in base_region3_walk()
448 			if (!table)  in base_region3_walk()
450 			*rtte = table | _REGION3_ENTRY;  in base_region3_walk()
452 		table = *rtte & _REGION_ENTRY_ORIGIN;  in base_region3_walk()
453 		rc = base_segment_walk(table, addr, next, alloc);  in base_region3_walk()
457 			base_crst_free(table);  in base_region3_walk()
465 	unsigned long *rste, next, table;  in base_region2_walk()  local
475 			table = base_crst_alloc(_REGION3_ENTRY_EMPTY);  in base_region2_walk()
476 			if (!table)  in base_region2_walk()
478 			*rste = table | _REGION2_ENTRY;  in base_region2_walk()
480 		table = *rste & _REGION_ENTRY_ORIGIN;  in base_region2_walk()
481 		rc = base_region3_walk(table, addr, next, alloc);  in base_region2_walk()
485 			base_crst_free(table);  in base_region2_walk()
493 	unsigned long *rfte, next, table;  in base_region1_walk()  local
503 			table = base_crst_alloc(_REGION2_ENTRY_EMPTY);  in base_region1_walk()
504 			if (!table)  in base_region1_walk()
506 			*rfte = table | _REGION1_ENTRY;  in base_region1_walk()
508 		table = *rfte & _REGION_ENTRY_ORIGIN;  in base_region1_walk()
509 		rc = base_region2_walk(table, addr, next, alloc);  in base_region1_walk()
513 			base_crst_free(table);  in base_region1_walk()
527 	unsigned long table = asce & _ASCE_ORIGIN;  in base_asce_free()  local
533 		base_segment_walk(table, 0, _REGION3_SIZE, 0);  in base_asce_free()
536 		base_region3_walk(table, 0, _REGION2_SIZE, 0);  in base_asce_free()
539 		base_region2_walk(table, 0, _REGION1_SIZE, 0);  in base_asce_free()
542 		base_region1_walk(table, 0, TASK_SIZE_MAX, 0);  in base_asce_free()
545 	base_crst_free(table);  in base_asce_free()
579 	unsigned long asce, table, end;  in base_asce_alloc()  local
586 		table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY);  in base_asce_alloc()
587 		if (!table)  in base_asce_alloc()
589 		rc = base_segment_walk(table, addr, end, 1);  in base_asce_alloc()
590 		asce = table | _ASCE_TYPE_SEGMENT | _ASCE_TABLE_LENGTH;  in base_asce_alloc()
592 		table = base_crst_alloc(_REGION3_ENTRY_EMPTY);  in base_asce_alloc()
593 		if (!table)  in base_asce_alloc()
595 		rc = base_region3_walk(table, addr, end, 1);  in base_asce_alloc()
596 		asce = table | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH;  in base_asce_alloc()
598 		table = base_crst_alloc(_REGION2_ENTRY_EMPTY);  in base_asce_alloc()
599 		if (!table)  in base_asce_alloc()
601 		rc = base_region2_walk(table, addr, end, 1);  in base_asce_alloc()
602 		asce = table | _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH;  in base_asce_alloc()
604 		table = base_crst_alloc(_REGION1_ENTRY_EMPTY);  in base_asce_alloc()
605 		if (!table)  in base_asce_alloc()
607 		rc = base_region1_walk(table, addr, end, 1);  in base_asce_alloc()
608 		asce = table | _ASCE_TYPE_REGION1 | _ASCE_TABLE_LENGTH;  in base_asce_alloc()