1 #include "libcflat.h" 2 #include "desc.h" 3 #include "processor.h" 4 #include "smp.h" 5 #include <setjmp.h> 6 #include "apic-defs.h" 7 8 /* Boot-related data structures */ 9 10 /* IDT and IDT descriptor */ 11 idt_entry_t boot_idt[256] = {0}; 12 13 struct descriptor_table_ptr idt_descr = { 14 .limit = sizeof(boot_idt) - 1, 15 .base = (unsigned long)boot_idt, 16 }; 17 18 #ifndef __x86_64__ 19 /* GDT, TSS and descriptors */ 20 gdt_entry_t gdt[TSS_MAIN / 8 + MAX_TEST_CPUS * 2] = { 21 { 0, 0, 0, .type_limit_flags = 0x0000}, /* 0x00 null */ 22 {0xffff, 0, 0, .type_limit_flags = 0xcf9b}, /* flat 32-bit code segment */ 23 {0xffff, 0, 0, .type_limit_flags = 0xcf93}, /* flat 32-bit data segment */ 24 {0xffff, 0, 0, .type_limit_flags = 0xcf1b}, /* flat 32-bit code segment, not present */ 25 { 0, 0, 0, .type_limit_flags = 0x0000}, /* TSS for task gates */ 26 {0xffff, 0, 0, .type_limit_flags = 0x8f9b}, /* 16-bit code segment */ 27 {0xffff, 0, 0, .type_limit_flags = 0x8f93}, /* 16-bit data segment */ 28 {0xffff, 0, 0, .type_limit_flags = 0xcffb}, /* 32-bit code segment (user) */ 29 {0xffff, 0, 0, .type_limit_flags = 0xcff3}, /* 32-bit data segment (user) */ 30 }; 31 32 tss32_t tss[MAX_TEST_CPUS] = {0}; 33 #else 34 gdt_entry_t gdt[TSS_MAIN / 8 + MAX_TEST_CPUS * 2] = { 35 { 0, 0, 0, .type_limit_flags = 0x0000}, /* 0x00 null */ 36 {0xffff, 0, 0, .type_limit_flags = 0xaf9b}, /* 0x08 64-bit code segment */ 37 {0xffff, 0, 0, .type_limit_flags = 0xcf93}, /* 0x10 32/64-bit data segment */ 38 {0xffff, 0, 0, .type_limit_flags = 0xaf1b}, /* 0x18 64-bit code segment, not present */ 39 {0xffff, 0, 0, .type_limit_flags = 0xcf9b}, /* 0x20 32-bit code segment */ 40 {0xffff, 0, 0, .type_limit_flags = 0x8f9b}, /* 0x28 16-bit code segment */ 41 {0xffff, 0, 0, .type_limit_flags = 0x8f93}, /* 0x30 16-bit data segment */ 42 {0xffff, 0, 0, .type_limit_flags = 0xcffb}, /* 0x38 32-bit code segment (user) */ 43 {0xffff, 0, 0, .type_limit_flags = 0xcff3}, /* 0x40 32/64-bit data segment (user) */ 44 {0xffff, 0, 0, .type_limit_flags = 0xaffb}, /* 0x48 64-bit code segment (user) */ 45 }; 46 47 tss64_t tss[MAX_TEST_CPUS] = {0}; 48 #endif 49 50 struct descriptor_table_ptr gdt_descr = { 51 .limit = sizeof(gdt) - 1, 52 .base = (unsigned long)gdt, 53 }; 54 55 #ifndef __x86_64__ 56 __attribute__((regparm(1))) 57 #endif 58 void do_handle_exception(struct ex_regs *regs); 59 60 /* 61 * Fill an idt_entry_t or call gate entry, clearing e_sz bytes first. 62 * 63 * This can be used for both IDT entries and call gate entries, since the gate 64 * descriptor layout is identical to idt_entry_t, except for the absence of 65 * .offset2 and .reserved fields. To do so, pass in e_sz according to the gate 66 * descriptor size. 67 */ 68 void set_desc_entry(idt_entry_t *e, size_t e_sz, void *addr, 69 u16 sel, u16 type, u16 dpl) 70 { 71 memset(e, 0, e_sz); 72 e->offset0 = (unsigned long)addr; 73 e->selector = sel; 74 e->ist = 0; 75 e->type = type; 76 e->dpl = dpl; 77 e->p = 1; 78 e->offset1 = (unsigned long)addr >> 16; 79 #ifdef __x86_64__ 80 if (e_sz == sizeof(*e)) 81 e->offset2 = (unsigned long)addr >> 32; 82 #endif 83 } 84 85 void set_idt_entry(int vec, void *addr, int dpl) 86 { 87 idt_entry_t *e = &boot_idt[vec]; 88 set_desc_entry(e, sizeof *e, addr, read_cs(), 14, dpl); 89 } 90 91 void set_idt_dpl(int vec, u16 dpl) 92 { 93 idt_entry_t *e = &boot_idt[vec]; 94 e->dpl = dpl; 95 } 96 97 void set_idt_sel(int vec, u16 sel) 98 { 99 idt_entry_t *e = &boot_idt[vec]; 100 e->selector = sel; 101 } 102 103 struct ex_record { 104 unsigned long rip; 105 unsigned long handler; 106 }; 107 108 extern struct ex_record exception_table_start, exception_table_end; 109 110 const char* exception_mnemonic(int vector) 111 { 112 switch(vector) { 113 case 0: return "#DE"; 114 case 1: return "#DB"; 115 case 2: return "#NMI"; 116 case 3: return "#BP"; 117 case 4: return "#OF"; 118 case 5: return "#BR"; 119 case 6: return "#UD"; 120 case 7: return "#NM"; 121 case 8: return "#DF"; 122 case 10: return "#TS"; 123 case 11: return "#NP"; 124 case 12: return "#SS"; 125 case 13: return "#GP"; 126 case 14: return "#PF"; 127 case 16: return "#MF"; 128 case 17: return "#AC"; 129 case 18: return "#MC"; 130 case 19: return "#XM"; 131 default: return "#??"; 132 } 133 } 134 135 void unhandled_exception(struct ex_regs *regs, bool cpu) 136 { 137 printf("Unhandled %sexception %ld %s at ip %016lx\n", 138 cpu ? "cpu " : "", regs->vector, 139 exception_mnemonic(regs->vector), regs->rip); 140 if (regs->vector == 14) 141 printf("PF at %#lx addr %#lx\n", regs->rip, read_cr2()); 142 143 printf("error_code=%04lx rflags=%08lx cs=%08lx\n" 144 "rax=%016lx rcx=%016lx rdx=%016lx rbx=%016lx\n" 145 "rbp=%016lx rsi=%016lx rdi=%016lx\n" 146 #ifdef __x86_64__ 147 " r8=%016lx r9=%016lx r10=%016lx r11=%016lx\n" 148 "r12=%016lx r13=%016lx r14=%016lx r15=%016lx\n" 149 #endif 150 "cr0=%016lx cr2=%016lx cr3=%016lx cr4=%016lx\n" 151 #ifdef __x86_64__ 152 "cr8=%016lx\n" 153 #endif 154 , 155 regs->error_code, regs->rflags, regs->cs, 156 regs->rax, regs->rcx, regs->rdx, regs->rbx, 157 regs->rbp, regs->rsi, regs->rdi, 158 #ifdef __x86_64__ 159 regs->r8, regs->r9, regs->r10, regs->r11, 160 regs->r12, regs->r13, regs->r14, regs->r15, 161 #endif 162 read_cr0(), read_cr2(), read_cr3(), read_cr4() 163 #ifdef __x86_64__ 164 , read_cr8() 165 #endif 166 ); 167 dump_frame_stack((void*) regs->rip, (void*) regs->rbp); 168 abort(); 169 } 170 171 static void check_exception_table(struct ex_regs *regs) 172 { 173 struct ex_record *ex; 174 175 this_cpu_write_exception_vector(regs->vector); 176 this_cpu_write_exception_rflags_rf((regs->rflags >> 16) & 1); 177 this_cpu_write_exception_error_code(regs->error_code); 178 179 for (ex = &exception_table_start; ex != &exception_table_end; ++ex) { 180 if (ex->rip == regs->rip) { 181 regs->rip = ex->handler; 182 return; 183 } 184 } 185 unhandled_exception(regs, false); 186 } 187 188 static handler exception_handlers[32]; 189 190 handler handle_exception(u8 v, handler fn) 191 { 192 handler old; 193 194 old = exception_handlers[v]; 195 if (v < 32) 196 exception_handlers[v] = fn; 197 return old; 198 } 199 200 #ifndef __x86_64__ 201 __attribute__((regparm(1))) 202 #endif 203 void do_handle_exception(struct ex_regs *regs) 204 { 205 if (regs->vector < 32 && exception_handlers[regs->vector]) { 206 exception_handlers[regs->vector](regs); 207 return; 208 } 209 unhandled_exception(regs, true); 210 } 211 212 #define EX(NAME, N) extern char NAME##_fault; \ 213 asm (".pushsection .text \n\t" \ 214 #NAME"_fault: \n\t" \ 215 "push"W" $0 \n\t" \ 216 "push"W" $"#N" \n\t" \ 217 "jmp __handle_exception \n\t" \ 218 ".popsection") 219 220 #define EX_E(NAME, N) extern char NAME##_fault; \ 221 asm (".pushsection .text \n\t" \ 222 #NAME"_fault: \n\t" \ 223 "push"W" $"#N" \n\t" \ 224 "jmp __handle_exception \n\t" \ 225 ".popsection") 226 227 EX(de, 0); 228 EX(db, 1); 229 EX(nmi, 2); 230 EX(bp, 3); 231 EX(of, 4); 232 EX(br, 5); 233 EX(ud, 6); 234 EX(nm, 7); 235 EX_E(df, 8); 236 EX_E(ts, 10); 237 EX_E(np, 11); 238 EX_E(ss, 12); 239 EX_E(gp, 13); 240 EX_E(pf, 14); 241 EX(mf, 16); 242 EX_E(ac, 17); 243 EX(mc, 18); 244 EX(xm, 19); 245 EX_E(cp, 21); 246 247 asm (".pushsection .text \n\t" 248 "__handle_exception: \n\t" 249 #ifdef __x86_64__ 250 "push %r15; push %r14; push %r13; push %r12 \n\t" 251 "push %r11; push %r10; push %r9; push %r8 \n\t" 252 #endif 253 "push %"R "di; push %"R "si; push %"R "bp; sub $"S", %"R "sp \n\t" 254 "push %"R "bx; push %"R "dx; push %"R "cx; push %"R "ax \n\t" 255 #ifdef __x86_64__ 256 "mov %"R "sp, %"R "di \n\t" 257 #else 258 "mov %"R "sp, %"R "ax \n\t" 259 #endif 260 "call do_handle_exception \n\t" 261 "pop %"R "ax; pop %"R "cx; pop %"R "dx; pop %"R "bx \n\t" 262 "add $"S", %"R "sp; pop %"R "bp; pop %"R "si; pop %"R "di \n\t" 263 #ifdef __x86_64__ 264 "pop %r8; pop %r9; pop %r10; pop %r11 \n\t" 265 "pop %r12; pop %r13; pop %r14; pop %r15 \n\t" 266 #endif 267 "add $"S", %"R "sp \n\t" 268 "add $"S", %"R "sp \n\t" 269 "iret"W" \n\t" 270 ".popsection"); 271 272 static void *idt_handlers[32] = { 273 [0] = &de_fault, 274 [1] = &db_fault, 275 [2] = &nmi_fault, 276 [3] = &bp_fault, 277 [4] = &of_fault, 278 [5] = &br_fault, 279 [6] = &ud_fault, 280 [7] = &nm_fault, 281 [8] = &df_fault, 282 [10] = &ts_fault, 283 [11] = &np_fault, 284 [12] = &ss_fault, 285 [13] = &gp_fault, 286 [14] = &pf_fault, 287 [16] = &mf_fault, 288 [17] = &ac_fault, 289 [18] = &mc_fault, 290 [19] = &xm_fault, 291 [21] = &cp_fault, 292 }; 293 294 void setup_idt(void) 295 { 296 int i; 297 static bool idt_initialized = false; 298 299 if (idt_initialized) 300 return; 301 302 idt_initialized = true; 303 for (i = 0; i < 32; i++) { 304 if (idt_handlers[i]) 305 set_idt_entry(i, idt_handlers[i], 0); 306 } 307 handle_exception(0, check_exception_table); 308 handle_exception(6, check_exception_table); 309 handle_exception(13, check_exception_table); 310 } 311 312 void load_idt(void) 313 { 314 lidt(&idt_descr); 315 } 316 317 unsigned exception_vector(void) 318 { 319 return this_cpu_read_exception_vector(); 320 } 321 322 int write_cr4_checking(unsigned long val) 323 { 324 asm volatile(ASM_TRY("1f") 325 "mov %0,%%cr4\n\t" 326 "1:": : "r" (val)); 327 return exception_vector(); 328 } 329 330 unsigned exception_error_code(void) 331 { 332 return this_cpu_read_exception_error_code(); 333 } 334 335 bool exception_rflags_rf(void) 336 { 337 return this_cpu_read_exception_rflags_rf() & 1; 338 } 339 340 static char intr_alt_stack[4096]; 341 342 void set_gdt_entry(int sel, unsigned long base, u32 limit, u8 type, u8 flags) 343 { 344 gdt_entry_t *entry = &gdt[sel >> 3]; 345 346 /* Setup the descriptor base address */ 347 entry->base1 = (base & 0xFFFF); 348 entry->base2 = (base >> 16) & 0xFF; 349 entry->base3 = (base >> 24) & 0xFF; 350 351 /* Setup the descriptor limits, type and flags */ 352 entry->limit1 = (limit & 0xFFFF); 353 entry->type_limit_flags = ((limit & 0xF0000) >> 8) | ((flags & 0xF0) << 8) | type; 354 355 #ifdef __x86_64__ 356 if (!entry->s) { 357 struct system_desc64 *entry16 = (struct system_desc64 *)entry; 358 entry16->zero = 0; 359 entry16->base4 = base >> 32; 360 } 361 #endif 362 } 363 364 #ifndef __x86_64__ 365 void set_gdt_task_gate(u16 sel, u16 tss_sel) 366 { 367 set_gdt_entry(sel, tss_sel, 0, 0x85, 0); // task, present 368 } 369 370 void set_idt_task_gate(int vec, u16 sel) 371 { 372 idt_entry_t *e = &boot_idt[vec]; 373 374 memset(e, 0, sizeof *e); 375 376 e->selector = sel; 377 e->ist = 0; 378 e->type = 5; 379 e->dpl = 0; 380 e->p = 1; 381 } 382 383 /* 384 * 0 - main task 385 * 1 - interrupt task 386 */ 387 388 tss32_t tss_intr; 389 390 void setup_tss32(void) 391 { 392 u16 desc_size = sizeof(tss32_t); 393 394 tss[0].cr3 = read_cr3(); 395 tss_intr.cr3 = read_cr3(); 396 tss_intr.ss0 = tss_intr.ss1 = tss_intr.ss2 = 0x10; 397 tss_intr.esp = tss_intr.esp0 = tss_intr.esp1 = tss_intr.esp2 = 398 (u32)intr_alt_stack + 4096; 399 tss_intr.cs = 0x08; 400 tss_intr.ds = tss_intr.es = tss_intr.fs = tss_intr.ss = 0x10; 401 tss_intr.gs = read_gs(); 402 tss_intr.iomap_base = (u16)desc_size; 403 set_gdt_entry(TSS_INTR, (u32)&tss_intr, desc_size - 1, 0x89, 0); 404 } 405 406 void set_intr_task_gate(int e, void *fn) 407 { 408 tss_intr.eip = (u32)fn; 409 set_idt_task_gate(e, TSS_INTR); 410 } 411 412 void setup_alt_stack(void) 413 { 414 setup_tss32(); 415 } 416 417 void set_intr_alt_stack(int e, void *fn) 418 { 419 set_intr_task_gate(e, fn); 420 } 421 422 void print_current_tss_info(void) 423 { 424 u16 tr = str(); 425 426 if (tr != TSS_MAIN && tr != TSS_INTR) 427 printf("Unknown TSS %x\n", tr); 428 else 429 printf("TR=%x (%s) Main TSS back link %x. Intr TSS back link %x\n", 430 tr, tr ? "interrupt" : "main", tss[0].prev, tss_intr.prev); 431 } 432 #else 433 void set_intr_alt_stack(int e, void *addr) 434 { 435 set_idt_entry(e, addr, 0); 436 boot_idt[e].ist = 1; 437 } 438 439 void setup_alt_stack(void) 440 { 441 tss[0].ist1 = (u64)intr_alt_stack + 4096; 442 } 443 #endif 444 445 static bool exception; 446 static jmp_buf *exception_jmpbuf; 447 448 static void exception_handler_longjmp(void) 449 { 450 longjmp(*exception_jmpbuf, 1); 451 } 452 453 static void exception_handler(struct ex_regs *regs) 454 { 455 /* longjmp must happen after iret, so do not do it now. */ 456 exception = true; 457 regs->rip = (unsigned long)&exception_handler_longjmp; 458 regs->cs = read_cs(); 459 } 460 461 bool test_for_exception(unsigned int ex, void (*trigger_func)(void *data), 462 void *data) 463 { 464 handler old; 465 jmp_buf jmpbuf; 466 int ret; 467 468 old = handle_exception(ex, exception_handler); 469 ret = set_exception_jmpbuf(jmpbuf); 470 if (ret == 0) 471 trigger_func(data); 472 handle_exception(ex, old); 473 return ret; 474 } 475 476 void __set_exception_jmpbuf(jmp_buf *addr) 477 { 478 exception_jmpbuf = addr; 479 } 480 481 gdt_entry_t *get_tss_descr(void) 482 { 483 struct descriptor_table_ptr gdt_ptr; 484 gdt_entry_t *gdt; 485 486 sgdt(&gdt_ptr); 487 gdt = (gdt_entry_t *)gdt_ptr.base; 488 return &gdt[str() / 8]; 489 } 490 491 unsigned long get_gdt_entry_base(gdt_entry_t *entry) 492 { 493 unsigned long base; 494 base = entry->base1 | ((u32)entry->base2 << 16) | ((u32)entry->base3 << 24); 495 #ifdef __x86_64__ 496 if (!entry->s) { 497 base |= (u64)((struct system_desc64 *)entry)->base4 << 32; 498 } 499 #endif 500 return base; 501 } 502 503 unsigned long get_gdt_entry_limit(gdt_entry_t *entry) 504 { 505 unsigned long limit; 506 limit = entry->limit1 | ((u32)entry->limit2 << 16); 507 if (entry->g) { 508 limit = (limit << 12) | 0xFFF; 509 } 510 return limit; 511 } 512