1 #include "libcflat.h" 2 #include "desc.h" 3 #include "processor.h" 4 #include <setjmp.h> 5 #include "apic-defs.h" 6 7 /* Boot-related data structures */ 8 9 /* IDT and IDT descriptor */ 10 idt_entry_t boot_idt[256] = {0}; 11 12 struct descriptor_table_ptr idt_descr = { 13 .limit = sizeof(boot_idt) - 1, 14 .base = (unsigned long)boot_idt, 15 }; 16 17 #ifdef __x86_64__ 18 /* GDT, TSS and descriptors */ 19 gdt_entry_t gdt[TSS_MAIN / 8 + MAX_TEST_CPUS * 2] = { 20 { 0, 0, 0, .type_limit_flags = 0x0000}, /* 0x00 null */ 21 {0xffff, 0, 0, .type_limit_flags = 0xaf9b}, /* 0x08 64-bit code segment */ 22 {0xffff, 0, 0, .type_limit_flags = 0xcf93}, /* 0x10 32/64-bit data segment */ 23 {0xffff, 0, 0, .type_limit_flags = 0xaf1b}, /* 0x18 64-bit code segment, not present */ 24 {0xffff, 0, 0, .type_limit_flags = 0xcf9b}, /* 0x20 32-bit code segment */ 25 {0xffff, 0, 0, .type_limit_flags = 0x8f9b}, /* 0x28 16-bit code segment */ 26 {0xffff, 0, 0, .type_limit_flags = 0x8f93}, /* 0x30 16-bit data segment */ 27 {0xffff, 0, 0, .type_limit_flags = 0xcffb}, /* 0x38 32-bit code segment (user) */ 28 {0xffff, 0, 0, .type_limit_flags = 0xcff3}, /* 0x40 32/64-bit data segment (user) */ 29 {0xffff, 0, 0, .type_limit_flags = 0xaffb}, /* 0x48 64-bit code segment (user) */ 30 }; 31 32 tss64_t tss[MAX_TEST_CPUS] = {0}; 33 34 struct descriptor_table_ptr gdt_descr = { 35 .limit = sizeof(gdt) - 1, 36 .base = (unsigned long)gdt, 37 }; 38 #endif 39 40 #ifndef __x86_64__ 41 __attribute__((regparm(1))) 42 #endif 43 void do_handle_exception(struct ex_regs *regs); 44 45 void set_idt_entry(int vec, void *addr, int dpl) 46 { 47 idt_entry_t *e = &boot_idt[vec]; 48 memset(e, 0, sizeof *e); 49 e->offset0 = (unsigned long)addr; 50 e->selector = read_cs(); 51 e->ist = 0; 52 e->type = 14; 53 e->dpl = dpl; 54 e->p = 1; 55 e->offset1 = (unsigned long)addr >> 16; 56 #ifdef __x86_64__ 57 e->offset2 = (unsigned long)addr >> 32; 58 #endif 59 } 60 61 void set_idt_dpl(int vec, u16 dpl) 62 { 63 idt_entry_t *e = &boot_idt[vec]; 64 e->dpl = dpl; 65 } 66 67 void set_idt_sel(int vec, u16 sel) 68 { 69 idt_entry_t *e = &boot_idt[vec]; 70 e->selector = sel; 71 } 72 73 struct ex_record { 74 unsigned long rip; 75 unsigned long handler; 76 }; 77 78 extern struct ex_record exception_table_start, exception_table_end; 79 80 static const char* exception_mnemonic(int vector) 81 { 82 switch(vector) { 83 case 0: return "#DE"; 84 case 1: return "#DB"; 85 case 2: return "#NMI"; 86 case 3: return "#BP"; 87 case 4: return "#OF"; 88 case 5: return "#BR"; 89 case 6: return "#UD"; 90 case 7: return "#NM"; 91 case 8: return "#DF"; 92 case 10: return "#TS"; 93 case 11: return "#NP"; 94 case 12: return "#SS"; 95 case 13: return "#GP"; 96 case 14: return "#PF"; 97 case 16: return "#MF"; 98 case 17: return "#AC"; 99 case 18: return "#MC"; 100 case 19: return "#XM"; 101 default: return "#??"; 102 } 103 } 104 105 void unhandled_exception(struct ex_regs *regs, bool cpu) 106 { 107 printf("Unhandled %sexception %ld %s at ip %016lx\n", 108 cpu ? "cpu " : "", regs->vector, 109 exception_mnemonic(regs->vector), regs->rip); 110 if (regs->vector == 14) 111 printf("PF at %#lx addr %#lx\n", regs->rip, read_cr2()); 112 113 printf("error_code=%04lx rflags=%08lx cs=%08lx\n" 114 "rax=%016lx rcx=%016lx rdx=%016lx rbx=%016lx\n" 115 "rbp=%016lx rsi=%016lx rdi=%016lx\n" 116 #ifdef __x86_64__ 117 " r8=%016lx r9=%016lx r10=%016lx r11=%016lx\n" 118 "r12=%016lx r13=%016lx r14=%016lx r15=%016lx\n" 119 #endif 120 "cr0=%016lx cr2=%016lx cr3=%016lx cr4=%016lx\n" 121 #ifdef __x86_64__ 122 "cr8=%016lx\n" 123 #endif 124 , 125 regs->error_code, regs->rflags, regs->cs, 126 regs->rax, regs->rcx, regs->rdx, regs->rbx, 127 regs->rbp, regs->rsi, regs->rdi, 128 #ifdef __x86_64__ 129 regs->r8, regs->r9, regs->r10, regs->r11, 130 regs->r12, regs->r13, regs->r14, regs->r15, 131 #endif 132 read_cr0(), read_cr2(), read_cr3(), read_cr4() 133 #ifdef __x86_64__ 134 , read_cr8() 135 #endif 136 ); 137 dump_frame_stack((void*) regs->rip, (void*) regs->rbp); 138 abort(); 139 } 140 141 static void check_exception_table(struct ex_regs *regs) 142 { 143 struct ex_record *ex; 144 unsigned ex_val; 145 146 ex_val = regs->vector | (regs->error_code << 16) | 147 (((regs->rflags >> 16) & 1) << 8); 148 asm("mov %0, %%gs:4" : : "r"(ex_val)); 149 150 for (ex = &exception_table_start; ex != &exception_table_end; ++ex) { 151 if (ex->rip == regs->rip) { 152 regs->rip = ex->handler; 153 return; 154 } 155 } 156 unhandled_exception(regs, false); 157 } 158 159 static handler exception_handlers[32]; 160 161 handler handle_exception(u8 v, handler fn) 162 { 163 handler old; 164 165 old = exception_handlers[v]; 166 if (v < 32) 167 exception_handlers[v] = fn; 168 return old; 169 } 170 171 #ifndef __x86_64__ 172 __attribute__((regparm(1))) 173 #endif 174 void do_handle_exception(struct ex_regs *regs) 175 { 176 if (regs->vector < 32 && exception_handlers[regs->vector]) { 177 exception_handlers[regs->vector](regs); 178 return; 179 } 180 unhandled_exception(regs, true); 181 } 182 183 #define EX(NAME, N) extern char NAME##_fault; \ 184 asm (".pushsection .text \n\t" \ 185 #NAME"_fault: \n\t" \ 186 "push"W" $0 \n\t" \ 187 "push"W" $"#N" \n\t" \ 188 "jmp __handle_exception \n\t" \ 189 ".popsection") 190 191 #define EX_E(NAME, N) extern char NAME##_fault; \ 192 asm (".pushsection .text \n\t" \ 193 #NAME"_fault: \n\t" \ 194 "push"W" $"#N" \n\t" \ 195 "jmp __handle_exception \n\t" \ 196 ".popsection") 197 198 EX(de, 0); 199 EX(db, 1); 200 EX(nmi, 2); 201 EX(bp, 3); 202 EX(of, 4); 203 EX(br, 5); 204 EX(ud, 6); 205 EX(nm, 7); 206 EX_E(df, 8); 207 EX_E(ts, 10); 208 EX_E(np, 11); 209 EX_E(ss, 12); 210 EX_E(gp, 13); 211 EX_E(pf, 14); 212 EX(mf, 16); 213 EX_E(ac, 17); 214 EX(mc, 18); 215 EX(xm, 19); 216 EX_E(cp, 21); 217 218 asm (".pushsection .text \n\t" 219 "__handle_exception: \n\t" 220 #ifdef __x86_64__ 221 "push %r15; push %r14; push %r13; push %r12 \n\t" 222 "push %r11; push %r10; push %r9; push %r8 \n\t" 223 #endif 224 "push %"R "di; push %"R "si; push %"R "bp; sub $"S", %"R "sp \n\t" 225 "push %"R "bx; push %"R "dx; push %"R "cx; push %"R "ax \n\t" 226 #ifdef __x86_64__ 227 "mov %"R "sp, %"R "di \n\t" 228 #else 229 "mov %"R "sp, %"R "ax \n\t" 230 #endif 231 "call do_handle_exception \n\t" 232 "pop %"R "ax; pop %"R "cx; pop %"R "dx; pop %"R "bx \n\t" 233 "add $"S", %"R "sp; pop %"R "bp; pop %"R "si; pop %"R "di \n\t" 234 #ifdef __x86_64__ 235 "pop %r8; pop %r9; pop %r10; pop %r11 \n\t" 236 "pop %r12; pop %r13; pop %r14; pop %r15 \n\t" 237 #endif 238 "add $"S", %"R "sp \n\t" 239 "add $"S", %"R "sp \n\t" 240 "iret"W" \n\t" 241 ".popsection"); 242 243 static void *idt_handlers[32] = { 244 [0] = &de_fault, 245 [1] = &db_fault, 246 [2] = &nmi_fault, 247 [3] = &bp_fault, 248 [4] = &of_fault, 249 [5] = &br_fault, 250 [6] = &ud_fault, 251 [7] = &nm_fault, 252 [8] = &df_fault, 253 [10] = &ts_fault, 254 [11] = &np_fault, 255 [12] = &ss_fault, 256 [13] = &gp_fault, 257 [14] = &pf_fault, 258 [16] = &mf_fault, 259 [17] = &ac_fault, 260 [18] = &mc_fault, 261 [19] = &xm_fault, 262 [21] = &cp_fault, 263 }; 264 265 void setup_idt(void) 266 { 267 int i; 268 static bool idt_initialized = false; 269 270 if (idt_initialized) { 271 return; 272 } 273 idt_initialized = true; 274 for (i = 0; i < 32; i++) 275 if (idt_handlers[i]) 276 set_idt_entry(i, idt_handlers[i], 0); 277 handle_exception(0, check_exception_table); 278 handle_exception(6, check_exception_table); 279 handle_exception(13, check_exception_table); 280 } 281 282 unsigned exception_vector(void) 283 { 284 unsigned char vector; 285 286 asm volatile("movb %%gs:4, %0" : "=q"(vector)); 287 return vector; 288 } 289 290 int write_cr4_checking(unsigned long val) 291 { 292 asm volatile(ASM_TRY("1f") 293 "mov %0,%%cr4\n\t" 294 "1:": : "r" (val)); 295 return exception_vector(); 296 } 297 298 unsigned exception_error_code(void) 299 { 300 unsigned short error_code; 301 302 asm volatile("mov %%gs:6, %0" : "=r"(error_code)); 303 return error_code; 304 } 305 306 bool exception_rflags_rf(void) 307 { 308 unsigned char rf_flag; 309 310 asm volatile("movb %%gs:5, %b0" : "=q"(rf_flag)); 311 return rf_flag & 1; 312 } 313 314 static char intr_alt_stack[4096]; 315 316 void set_gdt_entry(int sel, unsigned long base, u32 limit, u8 type, u8 flags) 317 { 318 gdt_entry_t *entry = &gdt[sel >> 3]; 319 320 /* Setup the descriptor base address */ 321 entry->base1 = (base & 0xFFFF); 322 entry->base2 = (base >> 16) & 0xFF; 323 entry->base3 = (base >> 24) & 0xFF; 324 325 /* Setup the descriptor limits, type and flags */ 326 entry->limit1 = (limit & 0xFFFF); 327 entry->type_limit_flags = ((limit & 0xF0000) >> 8) | ((flags & 0xF0) << 8) | type; 328 329 #ifdef __x86_64__ 330 if (!entry->s) { 331 struct system_desc64 *entry16 = (struct system_desc64 *)entry; 332 entry16->zero = 0; 333 entry16->base4 = base >> 32; 334 } 335 #endif 336 } 337 338 #ifndef __x86_64__ 339 void set_gdt_task_gate(u16 sel, u16 tss_sel) 340 { 341 set_gdt_entry(sel, tss_sel, 0, 0x85, 0); // task, present 342 } 343 344 void set_idt_task_gate(int vec, u16 sel) 345 { 346 idt_entry_t *e = &boot_idt[vec]; 347 348 memset(e, 0, sizeof *e); 349 350 e->selector = sel; 351 e->ist = 0; 352 e->type = 5; 353 e->dpl = 0; 354 e->p = 1; 355 } 356 357 /* 358 * 0 - main task 359 * 1 - interrupt task 360 */ 361 362 tss32_t tss_intr; 363 364 void setup_tss32(void) 365 { 366 u16 desc_size = sizeof(tss32_t); 367 368 tss.cr3 = read_cr3(); 369 tss_intr.cr3 = read_cr3(); 370 tss_intr.ss0 = tss_intr.ss1 = tss_intr.ss2 = 0x10; 371 tss_intr.esp = tss_intr.esp0 = tss_intr.esp1 = tss_intr.esp2 = 372 (u32)intr_alt_stack + 4096; 373 tss_intr.cs = 0x08; 374 tss_intr.ds = tss_intr.es = tss_intr.fs = tss_intr.ss = 0x10; 375 tss_intr.gs = read_gs(); 376 tss_intr.iomap_base = (u16)desc_size; 377 set_gdt_entry(TSS_INTR, (u32)&tss_intr, desc_size - 1, 0x89, 0); 378 } 379 380 void set_intr_task_gate(int e, void *fn) 381 { 382 tss_intr.eip = (u32)fn; 383 set_idt_task_gate(e, TSS_INTR); 384 } 385 386 void setup_alt_stack(void) 387 { 388 setup_tss32(); 389 } 390 391 void set_intr_alt_stack(int e, void *fn) 392 { 393 set_intr_task_gate(e, fn); 394 } 395 396 void print_current_tss_info(void) 397 { 398 u16 tr = str(); 399 400 if (tr != TSS_MAIN && tr != TSS_INTR) 401 printf("Unknown TSS %x\n", tr); 402 else 403 printf("TR=%x (%s) Main TSS back link %x. Intr TSS back link %x\n", 404 tr, tr ? "interrupt" : "main", tss.prev, tss_intr.prev); 405 } 406 #else 407 void set_intr_alt_stack(int e, void *addr) 408 { 409 set_idt_entry(e, addr, 0); 410 boot_idt[e].ist = 1; 411 } 412 413 void setup_alt_stack(void) 414 { 415 tss[0].ist1 = (u64)intr_alt_stack + 4096; 416 } 417 #endif 418 419 static bool exception; 420 static jmp_buf *exception_jmpbuf; 421 422 static void exception_handler_longjmp(void) 423 { 424 longjmp(*exception_jmpbuf, 1); 425 } 426 427 static void exception_handler(struct ex_regs *regs) 428 { 429 /* longjmp must happen after iret, so do not do it now. */ 430 exception = true; 431 regs->rip = (unsigned long)&exception_handler_longjmp; 432 regs->cs = read_cs(); 433 } 434 435 bool test_for_exception(unsigned int ex, void (*trigger_func)(void *data), 436 void *data) 437 { 438 handler old; 439 jmp_buf jmpbuf; 440 int ret; 441 442 old = handle_exception(ex, exception_handler); 443 ret = set_exception_jmpbuf(jmpbuf); 444 if (ret == 0) 445 trigger_func(data); 446 handle_exception(ex, old); 447 return ret; 448 } 449 450 void __set_exception_jmpbuf(jmp_buf *addr) 451 { 452 exception_jmpbuf = addr; 453 } 454 455 gdt_entry_t *get_tss_descr(void) 456 { 457 struct descriptor_table_ptr gdt_ptr; 458 gdt_entry_t *gdt; 459 460 sgdt(&gdt_ptr); 461 gdt = (gdt_entry_t *)gdt_ptr.base; 462 return &gdt[str() / 8]; 463 } 464 465 unsigned long get_gdt_entry_base(gdt_entry_t *entry) 466 { 467 unsigned long base; 468 base = entry->base1 | ((u32)entry->base2 << 16) | ((u32)entry->base3 << 24); 469 #ifdef __x86_64__ 470 if (!entry->s) { 471 base |= (u64)((struct system_desc64 *)entry)->base4 << 32; 472 } 473 #endif 474 return base; 475 } 476 477 unsigned long get_gdt_entry_limit(gdt_entry_t *entry) 478 { 479 unsigned long limit; 480 limit = entry->limit1 | ((u32)entry->limit2 << 16); 481 if (entry->g) { 482 limit = (limit << 12) | 0xFFF; 483 } 484 return limit; 485 } 486