1 // SPDX-License-Identifier: GPL-2.0-only
13  *	Mikael Pettersson	:	Power Management for UP-APIC.
43 #include <asm/pc-conf-reg.h>
64 #include <asm/intel-family.h>
124  * Handle interrupt mode configuration register (IMCR).
127  * local APIC. Before entering Symmetric I/O Mode, either
129  * PIC Mode by changing the IMCR.
147  * +1=force-enable
237  * so apic->write/read doesn't do anything
266  * get_physical_broadcast - Get number of physical broadcast IDs
275  * lapic_get_maxlvt - get the maximum number of local vector table entries
280 	 * - we always have APIC integrated on 64bit mode  in lapic_get_maxlvt()
281 	 * - 82489DXs do not report # of LVT entries  in lapic_get_maxlvt()
333 		 * See Intel SDM: TSC-Deadline Mode chapter. In xAPIC mode,  in __setup_APIC_LVTT()
425 		return -EINVAL;  in setup_APIC_eilvt()
433 		return -EBUSY;  in setup_APIC_eilvt()
470 	if (evt->features & CLOCK_EVT_FEAT_DUMMY)  in lapic_timer_shutdown()
484 	if (evt->features & CLOCK_EVT_FEAT_DUMMY)  in lapic_timer_set_periodic_oneshot()
528 	.irq				= -1,
577 	rev = (u32)m->driver_data;  in apic_validate_deadline_timer()
603 	levt->cpumask = cpumask_of(smp_processor_id());  in setup_APIC_timer()
606 		levt->name = "lapic-deadline";  in setup_APIC_timer()
607 		levt->features &= ~(CLOCK_EVT_FEAT_PERIODIC |  in setup_APIC_timer()
609 		levt->set_next_event = lapic_next_deadline;  in setup_APIC_timer()
634 	 * The clockevent device's ->mult and ->shift can both be  in lapic_update_tsc_freq()
664 static __initdata int lapic_cal_loops = -1;
710 	return -1;  in calibrate_by_pmtimer()
713 	apic_printk(APIC_VERBOSE, "... PM-Timer delta = %ld\n", deltapm);  in calibrate_by_pmtimer()
717 		return -1;  in calibrate_by_pmtimer()
721 	if (deltapm > (pm_100ms - pm_thresh) &&  in calibrate_by_pmtimer()
723 		apic_printk(APIC_VERBOSE, "... PM-Timer result ok\n");  in calibrate_by_pmtimer()
730 		"with PM-Timer: %ldms instead of 100ms\n", (long)res);  in calibrate_by_pmtimer()
735 	pr_info("APIC delta adjusted to PM-Timer: "  in calibrate_by_pmtimer()
744 					  "PM-Timer: %lu (%ld)\n",  in calibrate_by_pmtimer()
755 		return -1;  in lapic_init_clockevent()
784 	 * If interrupt delivery mode is legacy PIC or virtual wire without  in apic_needs_pit()
877 				if ((tsc_now - tsc_start) >= tsc_perj) {  in calibrate_APIC_clock()
900 	/* Build delta t1-t2 as apic timer counts down */  in calibrate_APIC_clock()
901 	delta = lapic_cal_t1 - lapic_cal_t2;  in calibrate_APIC_clock()
904 	deltatsc = (long)(lapic_cal_tsc2 - lapic_cal_tsc1);  in calibrate_APIC_clock()
907 	pm_referenced = !calibrate_by_pmtimer(lapic_cal_pm2 - lapic_cal_pm1,  in calibrate_APIC_clock()
936 		return -1;  in calibrate_APIC_clock()
939 	levt->features &= ~CLOCK_EVT_FEAT_DUMMY;  in calibrate_APIC_clock()
952 		levt->event_handler = lapic_cal_handler;  in calibrate_APIC_clock()
954 		lapic_cal_loops = -1;  in calibrate_APIC_clock()
967 		deltaj = lapic_cal_j2 - lapic_cal_j1;  in calibrate_APIC_clock()
971 		if (deltaj >= LAPIC_CAL_LOOPS-2 && deltaj <= LAPIC_CAL_LOOPS+2)  in calibrate_APIC_clock()
974 			levt->features |= CLOCK_EVT_FEAT_DUMMY;  in calibrate_APIC_clock()
978 	if (levt->features & CLOCK_EVT_FEAT_DUMMY) {  in calibrate_APIC_clock()
980 		return -1;  in calibrate_APIC_clock()
1048 	 * its possible that when we get here evt->event_handler is NULL.  in local_apic_timer_interrupt()
1052 	if (!evt->event_handler) {  in local_apic_timer_interrupt()
1061 	 * the NMI deadlock-detector uses this.  in local_apic_timer_interrupt()
1065 	evt->event_handler(evt);  in local_apic_timer_interrupt()
1073  * [ if a single-CPU system runs an SMP kernel then we call the local
1093  * clear_local_APIC - shutdown the local APIC
1113 		v = ERROR_APIC_VECTOR; /* any non-zero vector will do */  in clear_local_APIC()
1118 	 * any level-triggered sources.  in clear_local_APIC()
1167  * apic_soft_disable - Clears and software disables the local APIC on hotplug
1170  * MSR_IA32_APICBASE. Clearing that bit on systems based on the 3 wire APIC
1190  * disable_local_APIC - clear and disable the local APIC
1216  * re-entering the BIOS on shutdown.  Otherwise the BIOS may get confused and
1217  * not power-off.  Additionally clear all LVT entries before disable_local_APIC
1241  * sync_Arb_IDs - synchronize APIC bus arbitration IDs
1246 	 * Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 And not  in sync_Arb_IDs()
1274 	/* On 64-bit, the APIC must be integrated, Check local APIC only */  in __apic_intr_mode_select()
1281 	/* On 32-bit, the APIC may be integrated APIC or 82489DX */  in __apic_intr_mode_select()
1311 		pr_info("APIC: SMP mode deactivated\n");  in __apic_intr_mode_select()
1319 /* Select the interrupt delivery mode for the BSP */
1326  * An initial setup of the virtual wire mode.
1334 	 * through-I/O-APIC virtual wire mode might be active.  in init_bsp_APIC()
1363 	 * Set up the virtual wire mode.  in init_bsp_APIC()
1376 /* Init the interrupt delivery mode for the BSP */
1383 		pr_info("APIC: Keep in PIC mode(8259)\n");  in apic_intr_mode_init()
1386 		pr_info("APIC: Switch to virtual wire mode setup\n");  in apic_intr_mode_init()
1389 		pr_info("APIC: Switch to virtual wire mode setup with no configuration\n");  in apic_intr_mode_init()
1393 		pr_info("APIC: Switch to symmetric I/O mode setup\n");  in apic_intr_mode_init()
1396 		pr_info("APIC: Switch to symmetric I/O mode setup in no SMP routine\n");  in apic_intr_mode_init()
1419 	if (apic->disable_esr) {  in lapic_setup_esr()
1423 		 * ESR disabled - we can't do anything useful with the  in lapic_setup_esr()
1424 		 * errors anyway - mbligh  in lapic_setup_esr()
1466 		irr->regs[i] = apic_read(APIC_IRR + i * 0x10);  in apic_check_and_ack()
1470 		isr->regs[i] = apic_read(APIC_ISR + i * 0x10);  in apic_check_and_ack()
1477 	if (!bitmap_empty(isr->map, APIC_IR_BITS)) {  in apic_check_and_ack()
1483 		for_each_set_bit(bit, isr->map, APIC_IR_BITS)  in apic_check_and_ack()
1488 	return !bitmap_empty(irr->map, APIC_IR_BITS);  in apic_check_and_ack()
1520  * setup_local_APIC - setup the local APIC
1544 	/* Pound the ESR really hard over the head with a big hammer - mbligh */  in setup_local_APIC()
1545 	if (lapic_is_integrated() && apic->disable_esr) {  in setup_local_APIC()
1553 	BUG_ON(apic->apic_id_registered && !apic->apic_id_registered());  in setup_local_APIC()
1557 	 * an APIC.  See e.g. "AP-388 82489DX User's Manual" (Intel  in setup_local_APIC()
1560 	 * Except for APICs which operate in physical destination mode.  in setup_local_APIC()
1562 	if (apic->init_apic_ldr)  in setup_local_APIC()
1563 		apic->init_apic_ldr();  in setup_local_APIC()
1566 	 * Set Task Priority to 'accept all except vectors 0-31'.  An APIC  in setup_local_APIC()
1567 	 * vector in the 16-31 range could be delivered if TPR == 0, but we  in setup_local_APIC()
1597 	 * away, oh well :-(  in setup_local_APIC()
1599 	 * [ This bug can be reproduced easily with a level-triggered  in setup_local_APIC()
1606 	 * like LRU than MRU (the short-term load is more even across CPUs).  in setup_local_APIC()
1610 	 * - enable focus processor (bit==0)  in setup_local_APIC()
1611 	 * - 64bit mode always use processor focus  in setup_local_APIC()
1628 	 * set up through-local-APIC on the boot CPU's LINT0. This is not  in setup_local_APIC()
1629 	 * strictly necessary in pure symmetric-IO mode, but sometimes  in setup_local_APIC()
1633 	 * TODO: set up through-local-APIC from through-I/O-APIC? --macro  in setup_local_APIC()
1748 	/* Disable xapic and x2apic first and then reenable xapic mode */  in __x2apic_disable()
1776 			pr_warn("APIC locked in x2apic mode, can't disable\n");  in setup_nox2apic()
1859 			pr_info("x2apic: IRQ remapping doesn't support X2APIC mode\n");  in try_to_enable_x2apic()
1867 		 * used for non-remapped IRQ domains.  in try_to_enable_x2apic()
1876 		 * in physical mode, and CPUs with an APIC ID that cannot  in try_to_enable_x2apic()
1924 		pr_info("Not enabling interrupt remapping due to skipped IO-APIC setup\n");  in enable_IR_x2apic()
1934 		pr_info("Saving IO-APIC state failed: %d\n", ret);  in enable_IR_x2apic()
1939 	legacy_pic->mask_all();  in enable_IR_x2apic()
1945 	/* ir_stat contains the remap mode or an error code */  in enable_IR_x2apic()
1950 	legacy_pic->restore_mask();  in enable_IR_x2apic()
1956  * Detect and enable local APICs on non-SMP boards.
1958  * On AMD64 we trust the BIOS - if it says no APIC it is likely
2015 			pr_info("Local APIC disabled by BIOS -- reenabling.\n");  in apic_force_enable()
2053 		 * Over-ride BIOS and try to enable the local APIC only if  in detect_init_APIC()
2057 			pr_info("Local APIC disabled by BIOS -- "  in detect_init_APIC()
2079  * init_apic_mappings - initialize APIC mappings
2161  * spurious_interrupt - Catch all for interrupts raised on unused vectors
2198 	/* First tickle the hardware, only then report what went on. -- REW */  in DEFINE_IDTENTRY_SYSVEC()
2222  * connect_bsp_APIC - attach the APIC to the interrupt system
2233 		 * PIC mode, enable APIC mode in the IMCR, i.e.  connect BSP's  in connect_bsp_APIC()
2236 		apic_printk(APIC_VERBOSE, "leaving PIC mode, "  in connect_bsp_APIC()
2237 				"enabling APIC mode.\n");  in connect_bsp_APIC()
2244  * disconnect_bsp_APIC - detach the APIC from the interrupt system
2245  * @virt_wire_setup:	indicates, whether virtual wire mode is selected
2247  * Virtual wire mode is necessary to deliver legacy interrupts even when the
2257 		 * Put the board back into PIC mode (has an effect only on  in disconnect_bsp_APIC()
2262 		apic_printk(APIC_VERBOSE, "disabling APIC mode, "  in disconnect_bsp_APIC()
2263 				"entering PIC mode.\n");  in disconnect_bsp_APIC()
2269 	/* Go back to Virtual Wire compatibility mode */  in disconnect_bsp_APIC()
2321 u32 cpuid_to_apicid[] = { [0 ... NR_CPUS - 1] = BAD_APICID, };
2332 	u32 mask = (1U << (fls(smp_num_siblings) - 1)) - 1;  in cpu_mark_primary_thread()
2374 	 * cpuid <-> apicid mapping is persistent, so when a cpu is up,  in allocate_logical_cpuid()
2387 		return -EINVAL;  in allocate_logical_cpuid()
2433 		return -ENODEV;  in generic_processor_info()
2443 		return -EINVAL;  in generic_processor_info()
2449 		return -EINVAL;  in generic_processor_info()
2462 	msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW;  in __irq_msi_compose_msg()
2463 	msg->arch_addr_lo.dest_mode_logical = apic->dest_mode_logical;  in __irq_msi_compose_msg()
2464 	msg->arch_addr_lo.destid_0_7 = cfg->dest_apicid & 0xFF;  in __irq_msi_compose_msg()
2466 	msg->arch_data.delivery_mode = APIC_DELIVERY_MODE_FIXED;  in __irq_msi_compose_msg()
2467 	msg->arch_data.vector = cfg->vector;  in __irq_msi_compose_msg()
2469 	msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH;  in __irq_msi_compose_msg()
2474 	 * address APICs which can't be addressed in the normal 32-bit  in __irq_msi_compose_msg()
2477 	 * 5-11 to be used, giving support for 15 bits of APIC IDs in total.  in __irq_msi_compose_msg()
2480 		msg->arch_addr_hi.destid_8_31 = cfg->dest_apicid >> 8;  in __irq_msi_compose_msg()
2481 	else if (virt_ext_dest_id && cfg->dest_apicid < 0x8000)  in __irq_msi_compose_msg()
2482 		msg->arch_addr_lo.virt_destid_8_14 = cfg->dest_apicid >> 8;  in __irq_msi_compose_msg()
2484 		WARN_ON_ONCE(cfg->dest_apicid > 0xFF);  in __irq_msi_compose_msg()
2489 	u32 dest = msg->arch_addr_lo.destid_0_7;  in x86_msi_msg_get_destid()
2492 		dest |= msg->arch_addr_hi.destid_8_31 << 8;  in x86_msi_msg_get_destid()
2500 	apic_write(APIC_ID, apic->set_apic_id(boot_cpu_physical_apicid));  in apic_bsp_up_setup()
2506  * apic_bsp_setup - Setup function for local apic and io-apic
2507  * @upmode:		Force UP mode (for APIC_init_uniprocessor)
2623 	 * IO-APIC and PIC have their own resume routines.  in lapic_resume()
2626 	 * and interrupt-remapping.  in lapic_resume()
2629 	legacy_pic->mask_all();  in lapic_resume()
2682  * This device has no shutdown method - fully functioning local APICs
2723 	pr_info("APIC: %s detected, Multi Chassis\n", d->ident);  in set_multi()
2750  * apic_is_clustered_box() -- Check if we can expect good TSC
2754  * multi-chassis.
2807 			return -EINVAL;  in apic_set_verbosity()
2821 		return -EINVAL;  in apic_set_verbosity()
2832 		return -1;  in lapic_insert_resource()
2836 	lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1;  in lapic_insert_resource()
2851 		return -EINVAL;  in apic_set_disabled_cpu_apicid()
2860 		return -EINVAL;  in apic_set_extnmi()
2869 		pr_warn("Unknown external NMI delivery mode `%s' ignored\n", arg);  in apic_set_extnmi()
2870 		return -EINVAL;  in apic_set_extnmi()