1 #include "fwcfg.h" 2 #include "vm.h" 3 #include "libcflat.h" 4 5 static void *free = 0; 6 static void *vfree_top = 0; 7 8 static void free_memory(void *mem, unsigned long size) 9 { 10 while (size >= PAGE_SIZE) { 11 *(void **)mem = free; 12 free = mem; 13 mem += PAGE_SIZE; 14 size -= PAGE_SIZE; 15 } 16 } 17 18 void *alloc_page() 19 { 20 void *p; 21 22 if (!free) 23 return 0; 24 25 p = free; 26 free = *(void **)free; 27 28 return p; 29 } 30 31 void free_page(void *page) 32 { 33 *(void **)page = free; 34 free = page; 35 } 36 37 extern char edata; 38 static unsigned long end_of_memory; 39 40 unsigned long *install_pte(unsigned long *cr3, 41 int pte_level, 42 void *virt, 43 unsigned long pte, 44 unsigned long *pt_page) 45 { 46 int level; 47 unsigned long *pt = cr3; 48 unsigned offset; 49 50 for (level = PAGE_LEVEL; level > pte_level; --level) { 51 offset = ((unsigned long)virt >> ((level-1) * PGDIR_WIDTH + 12)) & PGDIR_MASK; 52 if (!(pt[offset] & PT_PRESENT_MASK)) { 53 unsigned long *new_pt = pt_page; 54 if (!new_pt) 55 new_pt = alloc_page(); 56 else 57 pt_page = 0; 58 memset(new_pt, 0, PAGE_SIZE); 59 pt[offset] = virt_to_phys(new_pt) | PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK; 60 } 61 pt = phys_to_virt(pt[offset] & PT_ADDR_MASK); 62 } 63 offset = ((unsigned long)virt >> ((level-1) * PGDIR_WIDTH + 12)) & PGDIR_MASK; 64 pt[offset] = pte; 65 return &pt[offset]; 66 } 67 68 unsigned long *get_pte(unsigned long *cr3, void *virt) 69 { 70 int level; 71 unsigned long *pt = cr3, pte; 72 unsigned offset; 73 74 for (level = PAGE_LEVEL; level > 1; --level) { 75 offset = ((unsigned long)virt >> (((level-1) * PGDIR_WIDTH) + 12)) & PGDIR_MASK; 76 pte = pt[offset]; 77 if (!(pte & PT_PRESENT_MASK)) 78 return NULL; 79 if (level == 2 && (pte & PT_PAGE_SIZE_MASK)) 80 return &pt[offset]; 81 pt = phys_to_virt(pte & PT_ADDR_MASK); 82 } 83 offset = ((unsigned long)virt >> (((level-1) * PGDIR_WIDTH) + 12)) & PGDIR_MASK; 84 return &pt[offset]; 85 } 86 87 unsigned long *install_large_page(unsigned long *cr3, 88 unsigned long phys, 89 void *virt) 90 { 91 return install_pte(cr3, 2, virt, 92 phys | PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK | PT_PAGE_SIZE_MASK, 0); 93 } 94 95 unsigned long *install_page(unsigned long *cr3, 96 unsigned long phys, 97 void *virt) 98 { 99 return install_pte(cr3, 1, virt, phys | PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK, 0); 100 } 101 102 103 static void setup_mmu_range(unsigned long *cr3, unsigned long start, 104 unsigned long len) 105 { 106 u64 max = (u64)len + (u64)start; 107 u64 phys = start; 108 109 while (phys + LARGE_PAGE_SIZE <= max) { 110 install_large_page(cr3, phys, (void *)(ulong)phys); 111 phys += LARGE_PAGE_SIZE; 112 } 113 while (phys + PAGE_SIZE <= max) { 114 install_page(cr3, phys, (void *)(ulong)phys); 115 phys += PAGE_SIZE; 116 } 117 } 118 119 static void setup_mmu(unsigned long len) 120 { 121 unsigned long *cr3 = alloc_page(); 122 123 memset(cr3, 0, PAGE_SIZE); 124 125 #ifdef __x86_64__ 126 if (len < (1ul << 32)) 127 len = (1ul << 32); /* map mmio 1:1 */ 128 129 setup_mmu_range(cr3, 0, len); 130 #else 131 if (len > (1ul << 31)) 132 len = (1ul << 31); 133 134 /* 0 - 2G memory, 2G-3G valloc area, 3G-4G mmio */ 135 setup_mmu_range(cr3, 0, len); 136 setup_mmu_range(cr3, 3ul << 30, (1ul << 30)); 137 vfree_top = (void*)(3ul << 30); 138 #endif 139 140 write_cr3(virt_to_phys(cr3)); 141 #ifndef __x86_64__ 142 write_cr4(X86_CR4_PSE); 143 #endif 144 write_cr0(X86_CR0_PG |X86_CR0_PE | X86_CR0_WP); 145 146 printf("paging enabled\n"); 147 printf("cr0 = %lx\n", read_cr0()); 148 printf("cr3 = %lx\n", read_cr3()); 149 printf("cr4 = %lx\n", read_cr4()); 150 } 151 152 void setup_vm() 153 { 154 end_of_memory = fwcfg_get_u64(FW_CFG_RAM_SIZE); 155 free_memory(&edata, end_of_memory - (unsigned long)&edata); 156 setup_mmu(end_of_memory); 157 } 158 159 void *vmalloc(unsigned long size) 160 { 161 void *mem, *p; 162 unsigned pages; 163 164 size += sizeof(unsigned long); 165 166 size = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); 167 vfree_top -= size; 168 mem = p = vfree_top; 169 pages = size / PAGE_SIZE; 170 while (pages--) { 171 install_page(phys_to_virt(read_cr3()), virt_to_phys(alloc_page()), p); 172 p += PAGE_SIZE; 173 } 174 *(unsigned long *)mem = size; 175 mem += sizeof(unsigned long); 176 return mem; 177 } 178 179 uint64_t virt_to_phys_cr3(void *mem) 180 { 181 return (*get_pte(phys_to_virt(read_cr3()), mem) & PT_ADDR_MASK) + ((ulong)mem & (PAGE_SIZE - 1)); 182 } 183 184 void vfree(void *mem) 185 { 186 unsigned long size = ((unsigned long *)mem)[-1]; 187 188 while (size) { 189 free_page(phys_to_virt(*get_pte(phys_to_virt(read_cr3()), mem) & PT_ADDR_MASK)); 190 mem += PAGE_SIZE; 191 size -= PAGE_SIZE; 192 } 193 } 194 195 void *vmap(unsigned long long phys, unsigned long size) 196 { 197 void *mem, *p; 198 unsigned pages; 199 200 size = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); 201 vfree_top -= size; 202 phys &= ~(unsigned long long)(PAGE_SIZE - 1); 203 204 mem = p = vfree_top; 205 pages = size / PAGE_SIZE; 206 while (pages--) { 207 install_page(phys_to_virt(read_cr3()), phys, p); 208 phys += PAGE_SIZE; 209 p += PAGE_SIZE; 210 } 211 return mem; 212 } 213 214 void *alloc_vpages(ulong nr) 215 { 216 vfree_top -= PAGE_SIZE * nr; 217 return vfree_top; 218 } 219 220 void *alloc_vpage(void) 221 { 222 return alloc_vpages(1); 223 } 224