1 /* 2 * Copyright (C) 2012, 2017, Red Hat Inc. 3 * 4 * This allocator provides contiguous physical addresses with page 5 * granularity. 6 */ 7 8 #include "libcflat.h" 9 #include "asm/spinlock.h" 10 #include "asm/page.h" 11 #include "asm/io.h" 12 #include "alloc.h" 13 #include "alloc_phys.h" 14 #include "alloc_page.h" 15 #include <bitops.h> 16 #include "vmalloc.h" 17 18 #define VM_MAGIC 0x7E57C0DE 19 20 #define GET_METADATA(x) (((struct metadata *)(x)) - 1) 21 #define GET_MAGIC(x) (*((unsigned long *)(x) - 1)) 22 23 struct metadata { 24 unsigned long npages; 25 unsigned long magic; 26 }; 27 28 static struct spinlock lock; 29 static void *vfree_top = 0; 30 static void *page_root; 31 32 /* 33 * Allocate a certain number of pages from the virtual address space (without 34 * physical backing). 35 * 36 * nr is the number of pages to allocate 37 * alignment_pages is the alignment of the allocation *in pages* 38 * metadata indicates whether an extra (unaligned) page needs to be allocated 39 * right before the main (aligned) allocation. 40 * 41 * The return value points to the first allocated virtual page, which will 42 * be the (potentially unaligned) metadata page if the metadata flag is 43 * specified. 44 */ 45 static void *do_alloc_vpages(ulong nr, unsigned int align_order, bool metadata) 46 { 47 uintptr_t ptr; 48 49 spin_lock(&lock); 50 ptr = (uintptr_t)vfree_top; 51 ptr -= PAGE_SIZE * nr; 52 ptr &= GENMASK_ULL(63, PAGE_SHIFT + align_order); 53 if (metadata) 54 ptr -= PAGE_SIZE; 55 vfree_top = (void *)ptr; 56 spin_unlock(&lock); 57 58 /* Cannot return vfree_top here, we are outside the lock! */ 59 return (void *)ptr; 60 } 61 62 void *alloc_vpages_aligned(ulong nr, unsigned int align_order) 63 { 64 return do_alloc_vpages(nr, align_order, false); 65 } 66 67 void *alloc_vpages(ulong nr) 68 { 69 return alloc_vpages_aligned(nr, 0); 70 } 71 72 void *alloc_vpage(void) 73 { 74 return alloc_vpages(1); 75 } 76 77 void *vmap(phys_addr_t phys, size_t size) 78 { 79 void *mem, *p; 80 size_t pages; 81 82 size = PAGE_ALIGN(size); 83 pages = size / PAGE_SIZE; 84 mem = p = alloc_vpages(pages); 85 86 phys &= ~(unsigned long long)(PAGE_SIZE - 1); 87 while (pages--) { 88 install_page(page_root, phys, p); 89 phys += PAGE_SIZE; 90 p += PAGE_SIZE; 91 } 92 return mem; 93 } 94 95 /* 96 * Allocate one page, for an object with specified alignment. 97 * The resulting pointer will be aligned to the required alignment, but 98 * intentionally not page-aligned. 99 * The metadata for single pages allocation is just the magic value, 100 * which is placed right before the pointer, like for bigger allocations. 101 */ 102 static void *vm_alloc_one_page(size_t alignment) 103 { 104 void *p; 105 106 /* this guarantees that there will be space for the magic value */ 107 assert(alignment >= sizeof(uintptr_t)); 108 assert(alignment < PAGE_SIZE); 109 p = alloc_vpage(); 110 install_page(page_root, virt_to_phys(alloc_page()), p); 111 p = (void *)((uintptr_t)p + alignment); 112 /* write the magic value right before the returned address */ 113 GET_MAGIC(p) = VM_MAGIC; 114 return p; 115 } 116 117 /* 118 * Allocate virtual memory, with the specified minimum alignment. 119 * If the allocation fits in one page, only one page is allocated. Otherwise 120 * enough pages are allocated for the object, plus one to keep metadata 121 * information about the allocation. 122 */ 123 static void *vm_memalign(size_t alignment, size_t size) 124 { 125 struct metadata *m; 126 phys_addr_t pa; 127 uintptr_t p; 128 void *mem; 129 size_t i; 130 131 if (!size) 132 return NULL; 133 assert(is_power_of_2(alignment)); 134 135 if (alignment < sizeof(uintptr_t)) 136 alignment = sizeof(uintptr_t); 137 /* it fits in one page, allocate only one page */ 138 if (alignment + size <= PAGE_SIZE) 139 return vm_alloc_one_page(alignment); 140 size = PAGE_ALIGN(size) / PAGE_SIZE; 141 alignment = get_order(PAGE_ALIGN(alignment) / PAGE_SIZE); 142 mem = do_alloc_vpages(size, alignment, true); 143 p = (uintptr_t)mem; 144 /* skip the metadata page */ 145 mem = (void *)(p + PAGE_SIZE); 146 /* 147 * time to actually allocate the physical pages to back our virtual 148 * allocation; note that we need to allocate one extra page (for the 149 * metadata), hence the <= 150 */ 151 for (i = 0; i <= size; i++, p += PAGE_SIZE) { 152 pa = virt_to_phys(alloc_page()); 153 assert(pa); 154 install_page(page_root, pa, (void *)p); 155 } 156 m = GET_METADATA(mem); 157 m->npages = size; 158 m->magic = VM_MAGIC; 159 return mem; 160 } 161 162 static void vm_free(void *mem) 163 { 164 struct metadata *m; 165 uintptr_t ptr, page, i; 166 167 if (!mem) 168 return; 169 /* the pointer is not page-aligned, it was a single-page allocation */ 170 if (!IS_ALIGNED((uintptr_t)mem, PAGE_SIZE)) { 171 assert(GET_MAGIC(mem) == VM_MAGIC); 172 page = virt_to_pte_phys(page_root, mem) & PAGE_MASK; 173 assert(page); 174 free_page(phys_to_virt(page)); 175 return; 176 } 177 178 /* the pointer is page-aligned, it was a multi-page allocation */ 179 m = GET_METADATA(mem); 180 assert(m->magic == VM_MAGIC); 181 assert(m->npages > 0); 182 assert(m->npages < BIT_ULL(BITS_PER_LONG - PAGE_SHIFT)); 183 /* free all the pages including the metadata page */ 184 ptr = (uintptr_t)m & PAGE_MASK; 185 for (i = 0 ; i < m->npages + 1; i++, ptr += PAGE_SIZE) { 186 page = virt_to_pte_phys(page_root, (void *)ptr) & PAGE_MASK; 187 assert(page); 188 free_page(phys_to_virt(page)); 189 } 190 } 191 192 static struct alloc_ops vmalloc_ops = { 193 .memalign = vm_memalign, 194 .free = vm_free, 195 }; 196 197 void __attribute__((__weak__)) find_highmem(void) 198 { 199 } 200 201 void init_alloc_vpage(void *top) 202 { 203 spin_lock(&lock); 204 assert(alloc_ops != &vmalloc_ops); 205 vfree_top = top; 206 spin_unlock(&lock); 207 } 208 209 void __setup_vm(void *opaque) 210 { 211 phys_addr_t base, top; 212 213 if (alloc_ops == &vmalloc_ops) 214 return; 215 216 phys_alloc_get_unused(&base, &top); 217 assert(base != top || page_alloc_initialized()); 218 /* 219 * Give low memory immediately to the page allocator, 220 * so that it can be used to allocate page tables. 221 */ 222 if (!page_alloc_initialized()) { 223 base = PAGE_ALIGN(base) >> PAGE_SHIFT; 224 top = top >> PAGE_SHIFT; 225 page_alloc_init_area(AREA_ANY_NUMBER, base, top); 226 page_alloc_ops_enable(); 227 } 228 229 find_highmem(); 230 phys_alloc_get_unused(&base, &top); 231 page_root = setup_mmu(top, opaque); 232 if (base != top) { 233 base = PAGE_ALIGN(base) >> PAGE_SHIFT; 234 top = top >> PAGE_SHIFT; 235 page_alloc_init_area(AREA_ANY_NUMBER, base, top); 236 } 237 238 spin_lock(&lock); 239 assert(alloc_ops != &vmalloc_ops); 240 alloc_ops = &vmalloc_ops; 241 spin_unlock(&lock); 242 } 243 244 void setup_vm(void) 245 { 246 __setup_vm(NULL); 247 } 248