1 #include "fwcfg.h" 2 #include "vm.h" 3 #include "libcflat.h" 4 5 static void *free = 0; 6 static void *vfree_top = 0; 7 8 static void free_memory(void *mem, unsigned long size) 9 { 10 void *end; 11 12 assert_msg((unsigned long) mem % PAGE_SIZE == 0, 13 "mem not page aligned: %p", mem); 14 15 assert_msg(size % PAGE_SIZE == 0, "size not page aligned: 0x%lx", size); 16 17 assert_msg(size == 0 || mem + size > mem, 18 "mem + size overflow: %p + 0x%lx", mem, size); 19 20 if (size == 0) { 21 free = NULL; 22 return; 23 } 24 25 free = mem; 26 end = mem + size; 27 while (mem + PAGE_SIZE != end) { 28 *(void **)mem = (mem + PAGE_SIZE); 29 mem += PAGE_SIZE; 30 } 31 32 *(void **)mem = NULL; 33 } 34 35 void *alloc_page() 36 { 37 void *p; 38 39 if (!free) 40 return 0; 41 42 p = free; 43 free = *(void **)free; 44 45 return p; 46 } 47 48 void free_page(void *page) 49 { 50 *(void **)page = free; 51 free = page; 52 } 53 54 extern char edata; 55 static unsigned long end_of_memory; 56 57 unsigned long *install_pte(unsigned long *cr3, 58 int pte_level, 59 void *virt, 60 unsigned long pte, 61 unsigned long *pt_page) 62 { 63 int level; 64 unsigned long *pt = cr3; 65 unsigned offset; 66 67 for (level = PAGE_LEVEL; level > pte_level; --level) { 68 offset = PGDIR_OFFSET((unsigned long)virt, level); 69 if (!(pt[offset] & PT_PRESENT_MASK)) { 70 unsigned long *new_pt = pt_page; 71 if (!new_pt) 72 new_pt = alloc_page(); 73 else 74 pt_page = 0; 75 memset(new_pt, 0, PAGE_SIZE); 76 pt[offset] = virt_to_phys(new_pt) | PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK; 77 } 78 pt = phys_to_virt(pt[offset] & PT_ADDR_MASK); 79 } 80 offset = PGDIR_OFFSET((unsigned long)virt, level); 81 pt[offset] = pte; 82 return &pt[offset]; 83 } 84 85 unsigned long *get_pte(unsigned long *cr3, void *virt) 86 { 87 int level; 88 unsigned long *pt = cr3, pte; 89 unsigned offset; 90 91 for (level = PAGE_LEVEL; level > 1; --level) { 92 offset = ((unsigned long)virt >> (((level-1) * PGDIR_WIDTH) + 12)) & PGDIR_MASK; 93 pte = pt[offset]; 94 if (!(pte & PT_PRESENT_MASK)) 95 return NULL; 96 if (level == 2 && (pte & PT_PAGE_SIZE_MASK)) 97 return &pt[offset]; 98 pt = phys_to_virt(pte & PT_ADDR_MASK); 99 } 100 offset = ((unsigned long)virt >> (((level-1) * PGDIR_WIDTH) + 12)) & PGDIR_MASK; 101 return &pt[offset]; 102 } 103 104 unsigned long *install_large_page(unsigned long *cr3, 105 unsigned long phys, 106 void *virt) 107 { 108 return install_pte(cr3, 2, virt, 109 phys | PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK | PT_PAGE_SIZE_MASK, 0); 110 } 111 112 unsigned long *install_page(unsigned long *cr3, 113 unsigned long phys, 114 void *virt) 115 { 116 return install_pte(cr3, 1, virt, phys | PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK, 0); 117 } 118 119 120 static void setup_mmu_range(unsigned long *cr3, unsigned long start, 121 unsigned long len) 122 { 123 u64 max = (u64)len + (u64)start; 124 u64 phys = start; 125 126 while (phys + LARGE_PAGE_SIZE <= max) { 127 install_large_page(cr3, phys, (void *)(ulong)phys); 128 phys += LARGE_PAGE_SIZE; 129 } 130 while (phys + PAGE_SIZE <= max) { 131 install_page(cr3, phys, (void *)(ulong)phys); 132 phys += PAGE_SIZE; 133 } 134 } 135 136 static void setup_mmu(unsigned long len) 137 { 138 unsigned long *cr3 = alloc_page(); 139 140 memset(cr3, 0, PAGE_SIZE); 141 142 #ifdef __x86_64__ 143 if (len < (1ul << 32)) 144 len = (1ul << 32); /* map mmio 1:1 */ 145 146 setup_mmu_range(cr3, 0, len); 147 #else 148 if (len > (1ul << 31)) 149 len = (1ul << 31); 150 151 /* 0 - 2G memory, 2G-3G valloc area, 3G-4G mmio */ 152 setup_mmu_range(cr3, 0, len); 153 setup_mmu_range(cr3, 3ul << 30, (1ul << 30)); 154 vfree_top = (void*)(3ul << 30); 155 #endif 156 157 write_cr3(virt_to_phys(cr3)); 158 #ifndef __x86_64__ 159 write_cr4(X86_CR4_PSE); 160 #endif 161 write_cr0(X86_CR0_PG |X86_CR0_PE | X86_CR0_WP); 162 163 printf("paging enabled\n"); 164 printf("cr0 = %lx\n", read_cr0()); 165 printf("cr3 = %lx\n", read_cr3()); 166 printf("cr4 = %lx\n", read_cr4()); 167 } 168 169 void setup_vm() 170 { 171 assert(!end_of_memory); 172 end_of_memory = fwcfg_get_u64(FW_CFG_RAM_SIZE); 173 free_memory(&edata, end_of_memory - (unsigned long)&edata); 174 setup_mmu(end_of_memory); 175 } 176 177 void *vmalloc(unsigned long size) 178 { 179 void *mem, *p; 180 unsigned pages; 181 182 size += sizeof(unsigned long); 183 184 size = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); 185 vfree_top -= size; 186 mem = p = vfree_top; 187 pages = size / PAGE_SIZE; 188 while (pages--) { 189 install_page(phys_to_virt(read_cr3()), virt_to_phys(alloc_page()), p); 190 p += PAGE_SIZE; 191 } 192 *(unsigned long *)mem = size; 193 mem += sizeof(unsigned long); 194 return mem; 195 } 196 197 uint64_t virt_to_phys_cr3(void *mem) 198 { 199 return (*get_pte(phys_to_virt(read_cr3()), mem) & PT_ADDR_MASK) + ((ulong)mem & (PAGE_SIZE - 1)); 200 } 201 202 void vfree(void *mem) 203 { 204 unsigned long size = ((unsigned long *)mem)[-1]; 205 206 while (size) { 207 free_page(phys_to_virt(*get_pte(phys_to_virt(read_cr3()), mem) & PT_ADDR_MASK)); 208 mem += PAGE_SIZE; 209 size -= PAGE_SIZE; 210 } 211 } 212 213 void *vmap(unsigned long long phys, unsigned long size) 214 { 215 void *mem, *p; 216 unsigned pages; 217 218 size = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); 219 vfree_top -= size; 220 phys &= ~(unsigned long long)(PAGE_SIZE - 1); 221 222 mem = p = vfree_top; 223 pages = size / PAGE_SIZE; 224 while (pages--) { 225 install_page(phys_to_virt(read_cr3()), phys, p); 226 phys += PAGE_SIZE; 227 p += PAGE_SIZE; 228 } 229 return mem; 230 } 231 232 void *alloc_vpages(ulong nr) 233 { 234 vfree_top -= PAGE_SIZE * nr; 235 return vfree_top; 236 } 237 238 void *alloc_vpage(void) 239 { 240 return alloc_vpages(1); 241 } 242