1 #include "fwcfg.h" 2 #include "vm.h" 3 #include "libcflat.h" 4 5 static void *free = 0; 6 static void *vfree_top = 0; 7 8 static void free_memory(void *mem, unsigned long size) 9 { 10 while (size >= PAGE_SIZE) { 11 *(void **)mem = free; 12 free = mem; 13 mem += PAGE_SIZE; 14 size -= PAGE_SIZE; 15 } 16 } 17 18 void *alloc_page() 19 { 20 void *p; 21 22 if (!free) 23 return 0; 24 25 p = free; 26 free = *(void **)free; 27 28 return p; 29 } 30 31 void free_page(void *page) 32 { 33 *(void **)page = free; 34 free = page; 35 } 36 37 extern char edata; 38 static unsigned long end_of_memory; 39 40 unsigned long *install_pte(unsigned long *cr3, 41 int pte_level, 42 void *virt, 43 unsigned long pte, 44 unsigned long *pt_page) 45 { 46 int level; 47 unsigned long *pt = cr3; 48 unsigned offset; 49 50 for (level = PAGE_LEVEL; level > pte_level; --level) { 51 offset = ((unsigned long)virt >> ((level-1) * PGDIR_WIDTH + 12)) & PGDIR_MASK; 52 if (!(pt[offset] & PT_PRESENT_MASK)) { 53 unsigned long *new_pt = pt_page; 54 if (!new_pt) 55 new_pt = alloc_page(); 56 else 57 pt_page = 0; 58 memset(new_pt, 0, PAGE_SIZE); 59 pt[offset] = virt_to_phys(new_pt) | PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK; 60 } 61 pt = phys_to_virt(pt[offset] & PT_ADDR_MASK); 62 } 63 offset = ((unsigned long)virt >> ((level-1) * PGDIR_WIDTH + 12)) & PGDIR_MASK; 64 pt[offset] = pte; 65 return &pt[offset]; 66 } 67 68 unsigned long *get_pte(unsigned long *cr3, void *virt) 69 { 70 int level; 71 unsigned long *pt = cr3, pte; 72 unsigned offset; 73 74 for (level = PAGE_LEVEL; level > 1; --level) { 75 offset = ((unsigned long)virt >> (((level-1) * PGDIR_WIDTH) + 12)) & PGDIR_MASK; 76 pte = pt[offset]; 77 if (!(pte & PT_PRESENT_MASK)) 78 return NULL; 79 if (level == 2 && (pte & PT_PAGE_SIZE_MASK)) 80 return &pt[offset]; 81 pt = phys_to_virt(pte & PT_ADDR_MASK); 82 } 83 offset = ((unsigned long)virt >> (((level-1) * PGDIR_WIDTH) + 12)) & PGDIR_MASK; 84 return &pt[offset]; 85 } 86 87 unsigned long *install_large_page(unsigned long *cr3, 88 unsigned long phys, 89 void *virt) 90 { 91 return install_pte(cr3, 2, virt, 92 phys | PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK | PT_PAGE_SIZE_MASK, 0); 93 } 94 95 unsigned long *install_page(unsigned long *cr3, 96 unsigned long phys, 97 void *virt) 98 { 99 return install_pte(cr3, 1, virt, phys | PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK, 0); 100 } 101 102 103 static void setup_mmu_range(unsigned long *cr3, unsigned long start, 104 unsigned long len) 105 { 106 u64 max = (u64)len + (u64)start; 107 u64 phys = start; 108 109 while (phys + LARGE_PAGE_SIZE <= max) { 110 install_large_page(cr3, phys, (void *)(ulong)phys); 111 phys += LARGE_PAGE_SIZE; 112 } 113 while (phys + PAGE_SIZE <= max) { 114 install_page(cr3, phys, (void *)(ulong)phys); 115 phys += PAGE_SIZE; 116 } 117 } 118 119 static void setup_mmu(unsigned long len) 120 { 121 unsigned long *cr3 = alloc_page(); 122 123 memset(cr3, 0, PAGE_SIZE); 124 125 #ifdef __x86_64__ 126 if (len < (1ul << 32)) 127 len = (1ul << 32); /* map mmio 1:1 */ 128 129 setup_mmu_range(cr3, 0, len); 130 #else 131 if (len > (1ul << 31)) 132 len = (1ul << 31); 133 134 /* 0 - 2G memory, 2G-3G valloc area, 3G-4G mmio */ 135 setup_mmu_range(cr3, 0, len); 136 setup_mmu_range(cr3, 3ul << 30, (1ul << 30)); 137 vfree_top = (void*)(3ul << 30); 138 #endif 139 140 write_cr3(virt_to_phys(cr3)); 141 #ifndef __x86_64__ 142 write_cr4(X86_CR4_PSE); 143 #endif 144 write_cr0(X86_CR0_PG |X86_CR0_PE | X86_CR0_WP); 145 146 printf("paging enabled\n"); 147 printf("cr0 = %lx\n", read_cr0()); 148 printf("cr3 = %lx\n", read_cr3()); 149 printf("cr4 = %lx\n", read_cr4()); 150 } 151 152 void setup_vm() 153 { 154 assert(!end_of_memory); 155 end_of_memory = fwcfg_get_u64(FW_CFG_RAM_SIZE); 156 free_memory(&edata, end_of_memory - (unsigned long)&edata); 157 setup_mmu(end_of_memory); 158 } 159 160 void *vmalloc(unsigned long size) 161 { 162 void *mem, *p; 163 unsigned pages; 164 165 size += sizeof(unsigned long); 166 167 size = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); 168 vfree_top -= size; 169 mem = p = vfree_top; 170 pages = size / PAGE_SIZE; 171 while (pages--) { 172 install_page(phys_to_virt(read_cr3()), virt_to_phys(alloc_page()), p); 173 p += PAGE_SIZE; 174 } 175 *(unsigned long *)mem = size; 176 mem += sizeof(unsigned long); 177 return mem; 178 } 179 180 uint64_t virt_to_phys_cr3(void *mem) 181 { 182 return (*get_pte(phys_to_virt(read_cr3()), mem) & PT_ADDR_MASK) + ((ulong)mem & (PAGE_SIZE - 1)); 183 } 184 185 void vfree(void *mem) 186 { 187 unsigned long size = ((unsigned long *)mem)[-1]; 188 189 while (size) { 190 free_page(phys_to_virt(*get_pte(phys_to_virt(read_cr3()), mem) & PT_ADDR_MASK)); 191 mem += PAGE_SIZE; 192 size -= PAGE_SIZE; 193 } 194 } 195 196 void *vmap(unsigned long long phys, unsigned long size) 197 { 198 void *mem, *p; 199 unsigned pages; 200 201 size = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); 202 vfree_top -= size; 203 phys &= ~(unsigned long long)(PAGE_SIZE - 1); 204 205 mem = p = vfree_top; 206 pages = size / PAGE_SIZE; 207 while (pages--) { 208 install_page(phys_to_virt(read_cr3()), phys, p); 209 phys += PAGE_SIZE; 210 p += PAGE_SIZE; 211 } 212 return mem; 213 } 214 215 void *alloc_vpages(ulong nr) 216 { 217 vfree_top -= PAGE_SIZE * nr; 218 return vfree_top; 219 } 220 221 void *alloc_vpage(void) 222 { 223 return alloc_vpages(1); 224 } 225