1 #include "libcflat.h" 2 #include "desc.h" 3 #include "processor.h" 4 #include "smp.h" 5 #include <setjmp.h> 6 #include "apic-defs.h" 7 8 /* Boot-related data structures */ 9 10 /* IDT and IDT descriptor */ 11 idt_entry_t boot_idt[256] = {0}; 12 13 struct descriptor_table_ptr idt_descr = { 14 .limit = sizeof(boot_idt) - 1, 15 .base = (unsigned long)boot_idt, 16 }; 17 18 #ifndef __x86_64__ 19 /* GDT, TSS and descriptors */ 20 gdt_entry_t gdt[TSS_MAIN / 8 + MAX_TEST_CPUS * 2] = { 21 { 0, 0, 0, .type_limit_flags = 0x0000}, /* 0x00 null */ 22 {0xffff, 0, 0, .type_limit_flags = 0xcf9b}, /* flat 32-bit code segment */ 23 {0xffff, 0, 0, .type_limit_flags = 0xcf93}, /* flat 32-bit data segment */ 24 {0xffff, 0, 0, .type_limit_flags = 0xcf1b}, /* flat 32-bit code segment, not present */ 25 { 0, 0, 0, .type_limit_flags = 0x0000}, /* TSS for task gates */ 26 {0xffff, 0, 0, .type_limit_flags = 0x8f9b}, /* 16-bit code segment */ 27 {0xffff, 0, 0, .type_limit_flags = 0x8f93}, /* 16-bit data segment */ 28 {0xffff, 0, 0, .type_limit_flags = 0xcffb}, /* 32-bit code segment (user) */ 29 {0xffff, 0, 0, .type_limit_flags = 0xcff3}, /* 32-bit data segment (user) */ 30 }; 31 32 tss32_t tss[MAX_TEST_CPUS] = {0}; 33 #else 34 gdt_entry_t gdt[TSS_MAIN / 8 + MAX_TEST_CPUS * 2] = { 35 { 0, 0, 0, .type_limit_flags = 0x0000}, /* 0x00 null */ 36 {0xffff, 0, 0, .type_limit_flags = 0xaf9b}, /* 0x08 64-bit code segment */ 37 {0xffff, 0, 0, .type_limit_flags = 0xcf93}, /* 0x10 32/64-bit data segment */ 38 {0xffff, 0, 0, .type_limit_flags = 0xaf1b}, /* 0x18 64-bit code segment, not present */ 39 {0xffff, 0, 0, .type_limit_flags = 0xcf9b}, /* 0x20 32-bit code segment */ 40 {0xffff, 0, 0, .type_limit_flags = 0x8f9b}, /* 0x28 16-bit code segment */ 41 {0xffff, 0, 0, .type_limit_flags = 0x8f93}, /* 0x30 16-bit data segment */ 42 {0xffff, 0, 0, .type_limit_flags = 0xcffb}, /* 0x38 32-bit code segment (user) */ 43 {0xffff, 0, 0, .type_limit_flags = 0xcff3}, /* 0x40 32/64-bit data segment (user) */ 44 {0xffff, 0, 0, .type_limit_flags = 0xaffb}, /* 0x48 64-bit code segment (user) */ 45 }; 46 47 tss64_t tss[MAX_TEST_CPUS] = {0}; 48 #endif 49 50 struct descriptor_table_ptr gdt_descr = { 51 .limit = sizeof(gdt) - 1, 52 .base = (unsigned long)gdt, 53 }; 54 55 #ifndef __x86_64__ 56 __attribute__((regparm(1))) 57 #endif 58 void do_handle_exception(struct ex_regs *regs); 59 60 void set_idt_entry(int vec, void *addr, int dpl) 61 { 62 idt_entry_t *e = &boot_idt[vec]; 63 memset(e, 0, sizeof *e); 64 e->offset0 = (unsigned long)addr; 65 e->selector = read_cs(); 66 e->ist = 0; 67 e->type = 14; 68 e->dpl = dpl; 69 e->p = 1; 70 e->offset1 = (unsigned long)addr >> 16; 71 #ifdef __x86_64__ 72 e->offset2 = (unsigned long)addr >> 32; 73 #endif 74 } 75 76 void set_idt_dpl(int vec, u16 dpl) 77 { 78 idt_entry_t *e = &boot_idt[vec]; 79 e->dpl = dpl; 80 } 81 82 void set_idt_sel(int vec, u16 sel) 83 { 84 idt_entry_t *e = &boot_idt[vec]; 85 e->selector = sel; 86 } 87 88 struct ex_record { 89 unsigned long rip; 90 unsigned long handler; 91 }; 92 93 extern struct ex_record exception_table_start, exception_table_end; 94 95 const char* exception_mnemonic(int vector) 96 { 97 switch(vector) { 98 case 0: return "#DE"; 99 case 1: return "#DB"; 100 case 2: return "#NMI"; 101 case 3: return "#BP"; 102 case 4: return "#OF"; 103 case 5: return "#BR"; 104 case 6: return "#UD"; 105 case 7: return "#NM"; 106 case 8: return "#DF"; 107 case 10: return "#TS"; 108 case 11: return "#NP"; 109 case 12: return "#SS"; 110 case 13: return "#GP"; 111 case 14: return "#PF"; 112 case 16: return "#MF"; 113 case 17: return "#AC"; 114 case 18: return "#MC"; 115 case 19: return "#XM"; 116 default: return "#??"; 117 } 118 } 119 120 void unhandled_exception(struct ex_regs *regs, bool cpu) 121 { 122 printf("Unhandled %sexception %ld %s at ip %016lx\n", 123 cpu ? "cpu " : "", regs->vector, 124 exception_mnemonic(regs->vector), regs->rip); 125 if (regs->vector == 14) 126 printf("PF at %#lx addr %#lx\n", regs->rip, read_cr2()); 127 128 printf("error_code=%04lx rflags=%08lx cs=%08lx\n" 129 "rax=%016lx rcx=%016lx rdx=%016lx rbx=%016lx\n" 130 "rbp=%016lx rsi=%016lx rdi=%016lx\n" 131 #ifdef __x86_64__ 132 " r8=%016lx r9=%016lx r10=%016lx r11=%016lx\n" 133 "r12=%016lx r13=%016lx r14=%016lx r15=%016lx\n" 134 #endif 135 "cr0=%016lx cr2=%016lx cr3=%016lx cr4=%016lx\n" 136 #ifdef __x86_64__ 137 "cr8=%016lx\n" 138 #endif 139 , 140 regs->error_code, regs->rflags, regs->cs, 141 regs->rax, regs->rcx, regs->rdx, regs->rbx, 142 regs->rbp, regs->rsi, regs->rdi, 143 #ifdef __x86_64__ 144 regs->r8, regs->r9, regs->r10, regs->r11, 145 regs->r12, regs->r13, regs->r14, regs->r15, 146 #endif 147 read_cr0(), read_cr2(), read_cr3(), read_cr4() 148 #ifdef __x86_64__ 149 , read_cr8() 150 #endif 151 ); 152 dump_frame_stack((void*) regs->rip, (void*) regs->rbp); 153 abort(); 154 } 155 156 static void check_exception_table(struct ex_regs *regs) 157 { 158 struct ex_record *ex; 159 160 this_cpu_write_exception_vector(regs->vector); 161 this_cpu_write_exception_rflags_rf((regs->rflags >> 16) & 1); 162 this_cpu_write_exception_error_code(regs->error_code); 163 164 for (ex = &exception_table_start; ex != &exception_table_end; ++ex) { 165 if (ex->rip == regs->rip) { 166 regs->rip = ex->handler; 167 return; 168 } 169 } 170 unhandled_exception(regs, false); 171 } 172 173 static handler exception_handlers[32]; 174 175 handler handle_exception(u8 v, handler fn) 176 { 177 handler old; 178 179 old = exception_handlers[v]; 180 if (v < 32) 181 exception_handlers[v] = fn; 182 return old; 183 } 184 185 #ifndef __x86_64__ 186 __attribute__((regparm(1))) 187 #endif 188 void do_handle_exception(struct ex_regs *regs) 189 { 190 if (regs->vector < 32 && exception_handlers[regs->vector]) { 191 exception_handlers[regs->vector](regs); 192 return; 193 } 194 unhandled_exception(regs, true); 195 } 196 197 #define EX(NAME, N) extern char NAME##_fault; \ 198 asm (".pushsection .text \n\t" \ 199 #NAME"_fault: \n\t" \ 200 "push"W" $0 \n\t" \ 201 "push"W" $"#N" \n\t" \ 202 "jmp __handle_exception \n\t" \ 203 ".popsection") 204 205 #define EX_E(NAME, N) extern char NAME##_fault; \ 206 asm (".pushsection .text \n\t" \ 207 #NAME"_fault: \n\t" \ 208 "push"W" $"#N" \n\t" \ 209 "jmp __handle_exception \n\t" \ 210 ".popsection") 211 212 EX(de, 0); 213 EX(db, 1); 214 EX(nmi, 2); 215 EX(bp, 3); 216 EX(of, 4); 217 EX(br, 5); 218 EX(ud, 6); 219 EX(nm, 7); 220 EX_E(df, 8); 221 EX_E(ts, 10); 222 EX_E(np, 11); 223 EX_E(ss, 12); 224 EX_E(gp, 13); 225 EX_E(pf, 14); 226 EX(mf, 16); 227 EX_E(ac, 17); 228 EX(mc, 18); 229 EX(xm, 19); 230 EX_E(cp, 21); 231 232 asm (".pushsection .text \n\t" 233 "__handle_exception: \n\t" 234 #ifdef __x86_64__ 235 "push %r15; push %r14; push %r13; push %r12 \n\t" 236 "push %r11; push %r10; push %r9; push %r8 \n\t" 237 #endif 238 "push %"R "di; push %"R "si; push %"R "bp; sub $"S", %"R "sp \n\t" 239 "push %"R "bx; push %"R "dx; push %"R "cx; push %"R "ax \n\t" 240 #ifdef __x86_64__ 241 "mov %"R "sp, %"R "di \n\t" 242 #else 243 "mov %"R "sp, %"R "ax \n\t" 244 #endif 245 "call do_handle_exception \n\t" 246 "pop %"R "ax; pop %"R "cx; pop %"R "dx; pop %"R "bx \n\t" 247 "add $"S", %"R "sp; pop %"R "bp; pop %"R "si; pop %"R "di \n\t" 248 #ifdef __x86_64__ 249 "pop %r8; pop %r9; pop %r10; pop %r11 \n\t" 250 "pop %r12; pop %r13; pop %r14; pop %r15 \n\t" 251 #endif 252 "add $"S", %"R "sp \n\t" 253 "add $"S", %"R "sp \n\t" 254 "iret"W" \n\t" 255 ".popsection"); 256 257 static void *idt_handlers[32] = { 258 [0] = &de_fault, 259 [1] = &db_fault, 260 [2] = &nmi_fault, 261 [3] = &bp_fault, 262 [4] = &of_fault, 263 [5] = &br_fault, 264 [6] = &ud_fault, 265 [7] = &nm_fault, 266 [8] = &df_fault, 267 [10] = &ts_fault, 268 [11] = &np_fault, 269 [12] = &ss_fault, 270 [13] = &gp_fault, 271 [14] = &pf_fault, 272 [16] = &mf_fault, 273 [17] = &ac_fault, 274 [18] = &mc_fault, 275 [19] = &xm_fault, 276 [21] = &cp_fault, 277 }; 278 279 void setup_idt(void) 280 { 281 int i; 282 static bool idt_initialized = false; 283 284 if (idt_initialized) 285 return; 286 287 idt_initialized = true; 288 for (i = 0; i < 32; i++) { 289 if (idt_handlers[i]) 290 set_idt_entry(i, idt_handlers[i], 0); 291 } 292 handle_exception(0, check_exception_table); 293 handle_exception(6, check_exception_table); 294 handle_exception(13, check_exception_table); 295 } 296 297 void load_idt(void) 298 { 299 lidt(&idt_descr); 300 } 301 302 unsigned exception_vector(void) 303 { 304 return this_cpu_read_exception_vector(); 305 } 306 307 int write_cr4_checking(unsigned long val) 308 { 309 asm volatile(ASM_TRY("1f") 310 "mov %0,%%cr4\n\t" 311 "1:": : "r" (val)); 312 return exception_vector(); 313 } 314 315 unsigned exception_error_code(void) 316 { 317 return this_cpu_read_exception_error_code(); 318 } 319 320 bool exception_rflags_rf(void) 321 { 322 return this_cpu_read_exception_rflags_rf() & 1; 323 } 324 325 static char intr_alt_stack[4096]; 326 327 void set_gdt_entry(int sel, unsigned long base, u32 limit, u8 type, u8 flags) 328 { 329 gdt_entry_t *entry = &gdt[sel >> 3]; 330 331 /* Setup the descriptor base address */ 332 entry->base1 = (base & 0xFFFF); 333 entry->base2 = (base >> 16) & 0xFF; 334 entry->base3 = (base >> 24) & 0xFF; 335 336 /* Setup the descriptor limits, type and flags */ 337 entry->limit1 = (limit & 0xFFFF); 338 entry->type_limit_flags = ((limit & 0xF0000) >> 8) | ((flags & 0xF0) << 8) | type; 339 340 #ifdef __x86_64__ 341 if (!entry->s) { 342 struct system_desc64 *entry16 = (struct system_desc64 *)entry; 343 entry16->zero = 0; 344 entry16->base4 = base >> 32; 345 } 346 #endif 347 } 348 349 #ifndef __x86_64__ 350 void set_gdt_task_gate(u16 sel, u16 tss_sel) 351 { 352 set_gdt_entry(sel, tss_sel, 0, 0x85, 0); // task, present 353 } 354 355 void set_idt_task_gate(int vec, u16 sel) 356 { 357 idt_entry_t *e = &boot_idt[vec]; 358 359 memset(e, 0, sizeof *e); 360 361 e->selector = sel; 362 e->ist = 0; 363 e->type = 5; 364 e->dpl = 0; 365 e->p = 1; 366 } 367 368 /* 369 * 0 - main task 370 * 1 - interrupt task 371 */ 372 373 tss32_t tss_intr; 374 375 void setup_tss32(void) 376 { 377 u16 desc_size = sizeof(tss32_t); 378 379 tss[0].cr3 = read_cr3(); 380 tss_intr.cr3 = read_cr3(); 381 tss_intr.ss0 = tss_intr.ss1 = tss_intr.ss2 = 0x10; 382 tss_intr.esp = tss_intr.esp0 = tss_intr.esp1 = tss_intr.esp2 = 383 (u32)intr_alt_stack + 4096; 384 tss_intr.cs = 0x08; 385 tss_intr.ds = tss_intr.es = tss_intr.fs = tss_intr.ss = 0x10; 386 tss_intr.gs = read_gs(); 387 tss_intr.iomap_base = (u16)desc_size; 388 set_gdt_entry(TSS_INTR, (u32)&tss_intr, desc_size - 1, 0x89, 0); 389 } 390 391 void set_intr_task_gate(int e, void *fn) 392 { 393 tss_intr.eip = (u32)fn; 394 set_idt_task_gate(e, TSS_INTR); 395 } 396 397 void setup_alt_stack(void) 398 { 399 setup_tss32(); 400 } 401 402 void set_intr_alt_stack(int e, void *fn) 403 { 404 set_intr_task_gate(e, fn); 405 } 406 407 void print_current_tss_info(void) 408 { 409 u16 tr = str(); 410 411 if (tr != TSS_MAIN && tr != TSS_INTR) 412 printf("Unknown TSS %x\n", tr); 413 else 414 printf("TR=%x (%s) Main TSS back link %x. Intr TSS back link %x\n", 415 tr, tr ? "interrupt" : "main", tss[0].prev, tss_intr.prev); 416 } 417 #else 418 void set_intr_alt_stack(int e, void *addr) 419 { 420 set_idt_entry(e, addr, 0); 421 boot_idt[e].ist = 1; 422 } 423 424 void setup_alt_stack(void) 425 { 426 tss[0].ist1 = (u64)intr_alt_stack + 4096; 427 } 428 #endif 429 430 static bool exception; 431 static jmp_buf *exception_jmpbuf; 432 433 static void exception_handler_longjmp(void) 434 { 435 longjmp(*exception_jmpbuf, 1); 436 } 437 438 static void exception_handler(struct ex_regs *regs) 439 { 440 /* longjmp must happen after iret, so do not do it now. */ 441 exception = true; 442 regs->rip = (unsigned long)&exception_handler_longjmp; 443 regs->cs = read_cs(); 444 } 445 446 bool test_for_exception(unsigned int ex, void (*trigger_func)(void *data), 447 void *data) 448 { 449 handler old; 450 jmp_buf jmpbuf; 451 int ret; 452 453 old = handle_exception(ex, exception_handler); 454 ret = set_exception_jmpbuf(jmpbuf); 455 if (ret == 0) 456 trigger_func(data); 457 handle_exception(ex, old); 458 return ret; 459 } 460 461 void __set_exception_jmpbuf(jmp_buf *addr) 462 { 463 exception_jmpbuf = addr; 464 } 465 466 gdt_entry_t *get_tss_descr(void) 467 { 468 struct descriptor_table_ptr gdt_ptr; 469 gdt_entry_t *gdt; 470 471 sgdt(&gdt_ptr); 472 gdt = (gdt_entry_t *)gdt_ptr.base; 473 return &gdt[str() / 8]; 474 } 475 476 unsigned long get_gdt_entry_base(gdt_entry_t *entry) 477 { 478 unsigned long base; 479 base = entry->base1 | ((u32)entry->base2 << 16) | ((u32)entry->base3 << 24); 480 #ifdef __x86_64__ 481 if (!entry->s) { 482 base |= (u64)((struct system_desc64 *)entry)->base4 << 32; 483 } 484 #endif 485 return base; 486 } 487 488 unsigned long get_gdt_entry_limit(gdt_entry_t *entry) 489 { 490 unsigned long limit; 491 limit = entry->limit1 | ((u32)entry->limit2 << 16); 492 if (entry->g) { 493 limit = (limit << 12) | 0xFFF; 494 } 495 return limit; 496 } 497