Lines Matching +full:interrupt +full:- +full:affinity
1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
4 * Copyright (C) 2005-2006 Thomas Gleixner
15 #include <linux/interrupt.h>
52 while (irqd_irq_inprogress(&desc->irq_data)) in __synchronize_hardirq()
55 /* Ok, that indicated we're done: double-check carefully. */ in __synchronize_hardirq()
56 raw_spin_lock_irqsave(&desc->lock, flags); in __synchronize_hardirq()
57 inprogress = irqd_irq_inprogress(&desc->irq_data); in __synchronize_hardirq()
72 raw_spin_unlock_irqrestore(&desc->lock, flags); in __synchronize_hardirq()
79 * synchronize_hardirq - wait for pending hard IRQ handlers (on other CPUs)
80 * @irq: interrupt number to wait for
83 * interrupt to complete before returning. If you use this
93 * This function may be called - with care - from IRQ context.
95 * It does not check whether there is an interrupt in flight at the
97 * called with interrupts disabled and the target CPU of the interrupt
106 return !atomic_read(&desc->threads_active); in synchronize_hardirq()
120 wait_event(desc->wait_for_threads, !atomic_read(&desc->threads_active)); in __synchronize_irq()
124 * synchronize_irq - wait for pending IRQ handlers (on other CPUs)
125 * @irq: interrupt number to wait for
127 * This function waits for any pending IRQ handlers for this interrupt
132 * an interrupt thread is associated to @irq.
135 * that the interrupt is not pending in any CPU and waiting for
152 if (!desc || !irqd_can_balance(&desc->irq_data) || in __irq_can_set_affinity()
153 !desc->irq_data.chip || !desc->irq_data.chip->irq_set_affinity) in __irq_can_set_affinity()
159 * irq_can_set_affinity - Check if the affinity of a given irq can be set
160 * @irq: Interrupt to check
169 * irq_can_set_affinity_usr - Check if affinity of a irq can be set from user space
170 * @irq: Interrupt to check
180 !irqd_affinity_is_managed(&desc->irq_data); in irq_can_set_affinity_usr()
184 * irq_set_thread_affinity - Notify irq threads to adjust affinity
185 * @desc: irq descriptor which has affinity changed
187 * We just set IRQTF_AFFINITY and delegate the affinity setting
188 * to the interrupt thread itself. We can not call
189 * set_cpus_allowed_ptr() here as we hold desc->lock and this
190 * code can be called from hard interrupt context.
197 if (action->thread) { in irq_set_thread_affinity()
198 set_bit(IRQTF_AFFINITY, &action->thread_flags); in irq_set_thread_affinity()
199 wake_up_process(action->thread); in irq_set_thread_affinity()
201 if (action->secondary && action->secondary->thread) { in irq_set_thread_affinity()
202 set_bit(IRQTF_AFFINITY, &action->secondary->thread_flags); in irq_set_thread_affinity()
203 wake_up_process(action->secondary->thread); in irq_set_thread_affinity()
216 pr_warn_once("irq_chip %s did not update eff. affinity mask of irq %u\n", in irq_validate_effective_affinity()
217 chip->name, data->irq); in irq_validate_effective_affinity()
234 if (!chip || !chip->irq_set_affinity) in irq_do_set_affinity()
235 return -EINVAL; in irq_do_set_affinity()
238 * If this is a managed interrupt and housekeeping is enabled on in irq_do_set_affinity()
239 * it check whether the requested affinity mask intersects with in irq_do_set_affinity()
242 * the affinity setter from routing the interrupt to an isolated in irq_do_set_affinity()
251 * If all housekeeping CPUs in the affinity mask are offline, the in irq_do_set_affinity()
252 * interrupt will be migrated by the CPU hotplug code once a in irq_do_set_affinity()
253 * housekeeping CPU which belongs to the affinity mask comes in irq_do_set_affinity()
273 * unless we are being asked to force the affinity (in which in irq_do_set_affinity()
278 ret = chip->irq_set_affinity(data, tmp_mask, force); in irq_do_set_affinity()
280 ret = chip->irq_set_affinity(data, mask, force); in irq_do_set_affinity()
282 ret = -EINVAL; in irq_do_set_affinity()
287 cpumask_copy(desc->irq_common_data.affinity, mask); in irq_do_set_affinity()
312 return -EBUSY; in irq_set_affinity_pending()
326 if (ret == -EBUSY && !force) in irq_try_set_affinity()
337 * Handle irq chips which can handle affinity only in activated in irq_set_affinity_deactivated()
340 * If the interrupt is not yet activated, just store the affinity in irq_set_affinity_deactivated()
342 * driver has to make sure anyway that the interrupt is in a in irq_set_affinity_deactivated()
349 cpumask_copy(desc->irq_common_data.affinity, mask); in irq_set_affinity_deactivated()
362 if (!chip || !chip->irq_set_affinity) in irq_set_affinity_locked()
363 return -EINVAL; in irq_set_affinity_locked()
375 if (desc->affinity_notify) { in irq_set_affinity_locked()
376 kref_get(&desc->affinity_notify->kref); in irq_set_affinity_locked()
377 if (!schedule_work(&desc->affinity_notify->work)) { in irq_set_affinity_locked()
379 kref_put(&desc->affinity_notify->kref, in irq_set_affinity_locked()
380 desc->affinity_notify->release); in irq_set_affinity_locked()
389 * irq_update_affinity_desc - Update affinity management for an interrupt
390 * @irq: The interrupt number to update
391 * @affinity: Pointer to the affinity descriptor
393 * This interface can be used to configure the affinity management of
396 * There are certain limitations on when it may be used - attempts to use it
399 * managed/non-managed interrupt accounting. In addition, attempts to use it on
400 * an interrupt which is already started or which has already been configured
404 struct irq_affinity_desc *affinity) in irq_update_affinity_desc() argument
416 return -EOPNOTSUPP; in irq_update_affinity_desc()
420 return -EINVAL; in irq_update_affinity_desc()
422 /* Requires the interrupt to be shut down */ in irq_update_affinity_desc()
423 if (irqd_is_started(&desc->irq_data)) { in irq_update_affinity_desc()
424 ret = -EBUSY; in irq_update_affinity_desc()
429 if (irqd_affinity_is_managed(&desc->irq_data)) { in irq_update_affinity_desc()
430 ret = -EBUSY; in irq_update_affinity_desc()
435 * Deactivate the interrupt. That's required to undo in irq_update_affinity_desc()
438 activated = irqd_is_activated(&desc->irq_data); in irq_update_affinity_desc()
440 irq_domain_deactivate_irq(&desc->irq_data); in irq_update_affinity_desc()
442 if (affinity->is_managed) { in irq_update_affinity_desc()
443 irqd_set(&desc->irq_data, IRQD_AFFINITY_MANAGED); in irq_update_affinity_desc()
444 irqd_set(&desc->irq_data, IRQD_MANAGED_SHUTDOWN); in irq_update_affinity_desc()
447 cpumask_copy(desc->irq_common_data.affinity, &affinity->mask); in irq_update_affinity_desc()
451 irq_domain_activate_irq(&desc->irq_data, false); in irq_update_affinity_desc()
466 return -EINVAL; in __irq_set_affinity()
468 raw_spin_lock_irqsave(&desc->lock, flags); in __irq_set_affinity()
470 raw_spin_unlock_irqrestore(&desc->lock, flags); in __irq_set_affinity()
475 * irq_set_affinity - Set the irq affinity of a given irq
476 * @irq: Interrupt to set affinity
488 * irq_force_affinity - Force the irq affinity of a given irq
489 * @irq: Interrupt to set affinity
511 return -EINVAL; in __irq_apply_affinity_hint()
512 desc->affinity_hint = m; in __irq_apply_affinity_hint()
524 struct irq_desc *desc = irq_to_desc(notify->irq); in irq_affinity_notify()
531 raw_spin_lock_irqsave(&desc->lock, flags); in irq_affinity_notify()
532 if (irq_move_pending(&desc->irq_data)) in irq_affinity_notify()
535 cpumask_copy(cpumask, desc->irq_common_data.affinity); in irq_affinity_notify()
536 raw_spin_unlock_irqrestore(&desc->lock, flags); in irq_affinity_notify()
538 notify->notify(notify, cpumask); in irq_affinity_notify()
542 kref_put(¬ify->kref, notify->release); in irq_affinity_notify()
546 * irq_set_affinity_notifier - control notification of IRQ affinity changes
547 * @irq: Interrupt for which to enable/disable notification
567 return -EINVAL; in irq_set_affinity_notifier()
571 notify->irq = irq; in irq_set_affinity_notifier()
572 kref_init(¬ify->kref); in irq_set_affinity_notifier()
573 INIT_WORK(¬ify->work, irq_affinity_notify); in irq_set_affinity_notifier()
576 raw_spin_lock_irqsave(&desc->lock, flags); in irq_set_affinity_notifier()
577 old_notify = desc->affinity_notify; in irq_set_affinity_notifier()
578 desc->affinity_notify = notify; in irq_set_affinity_notifier()
579 raw_spin_unlock_irqrestore(&desc->lock, flags); in irq_set_affinity_notifier()
582 if (cancel_work_sync(&old_notify->work)) { in irq_set_affinity_notifier()
584 kref_put(&old_notify->kref, old_notify->release); in irq_set_affinity_notifier()
586 kref_put(&old_notify->kref, old_notify->release); in irq_set_affinity_notifier()
595 * Generic version of the affinity autoselector.
610 * Preserve the managed affinity setting and a userspace affinity in irq_setup_affinity()
613 if (irqd_affinity_is_managed(&desc->irq_data) || in irq_setup_affinity()
614 irqd_has_set(&desc->irq_data, IRQD_AFFINITY_SET)) { in irq_setup_affinity()
615 if (cpumask_intersects(desc->irq_common_data.affinity, in irq_setup_affinity()
617 set = desc->irq_common_data.affinity; in irq_setup_affinity()
619 irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET); in irq_setup_affinity()
633 ret = irq_do_set_affinity(&desc->irq_data, &mask, false); in irq_setup_affinity()
638 /* Wrapper for ALPHA specific affinity selector magic */
648 * irq_set_vcpu_affinity - Set vcpu affinity for the interrupt
649 * @irq: interrupt number to set affinity
654 * affinity for an irq. The vCPU specific data is passed from
656 * KVM -> IOMMU -> irq_set_vcpu_affinity().
664 int ret = -ENOSYS; in irq_set_vcpu_affinity()
667 return -EINVAL; in irq_set_vcpu_affinity()
672 if (chip && chip->irq_set_vcpu_affinity) in irq_set_vcpu_affinity()
675 data = data->parent_data; in irq_set_vcpu_affinity()
682 ret = chip->irq_set_vcpu_affinity(data, vcpu_info); in irq_set_vcpu_affinity()
691 if (!desc->depth++) in __disable_irq()
701 return -EINVAL; in __disable_irq_nosync()
708 * disable_irq_nosync - disable an irq without waiting
709 * @irq: Interrupt to disable
711 * Disable the selected interrupt line. Disables and Enables are
725 * disable_irq - disable an irq and wait for completion
726 * @irq: Interrupt to disable
728 * Disable the selected interrupt line. Enables and Disables are
730 * This function waits for any pending IRQ handlers for this interrupt
735 * an interrupt thread is associated to @irq.
747 * disable_hardirq - disables an irq and waits for hardirq completion
748 * @irq: Interrupt to disable
750 * Disable the selected interrupt line. Enables and Disables are
753 * interrupt to complete before returning. If you use this function while
756 * When used to optimistically disable an interrupt from atomic context
761 * This function may be called - with care - from IRQ context.
773 * disable_nmi_nosync - disable an nmi without waiting
774 * @irq: Interrupt to disable
776 * Disable the selected interrupt line. Disables and enables are
778 * The interrupt to disable must have been requested through request_nmi.
789 switch (desc->depth) { in __enable_irq()
796 if (desc->istate & IRQS_SUSPENDED) in __enable_irq()
802 * interrupt might be marked NOAUTOEN so irq_startup() in __enable_irq()
805 * a managed and shutdown interrupt from the S3 resume in __enable_irq()
815 desc->depth--; in __enable_irq()
820 * enable_irq - enable handling of an irq
821 * @irq: Interrupt to enable
825 * IRQ line is re-enabled.
828 * desc->irq_data.chip->bus_lock and desc->chip->bus_sync_unlock are NULL !
837 if (WARN(!desc->irq_data.chip, in enable_irq()
848 * enable_nmi - enable handling of an nmi
849 * @irq: Interrupt to enable
851 * The interrupt to enable must have been requested through request_nmi.
854 * IRQ line is re-enabled.
864 int ret = -ENXIO; in set_irq_wake_real()
866 if (irq_desc_get_chip(desc)->flags & IRQCHIP_SKIP_SET_WAKE) in set_irq_wake_real()
869 if (desc->irq_data.chip->irq_set_wake) in set_irq_wake_real()
870 ret = desc->irq_data.chip->irq_set_wake(&desc->irq_data, on); in set_irq_wake_real()
876 * irq_set_irq_wake - control irq power management wakeup
877 * @irq: interrupt to control
882 * just as they match for non-wakeup mode support.
901 return -EINVAL; in irq_set_irq_wake()
905 ret = -EINVAL; in irq_set_irq_wake()
909 /* wakeup-capable irqs can be shared between drivers that in irq_set_irq_wake()
913 if (desc->wake_depth++ == 0) { in irq_set_irq_wake()
916 desc->wake_depth = 0; in irq_set_irq_wake()
918 irqd_set(&desc->irq_data, IRQD_WAKEUP_STATE); in irq_set_irq_wake()
921 if (desc->wake_depth == 0) { in irq_set_irq_wake()
923 } else if (--desc->wake_depth == 0) { in irq_set_irq_wake()
926 desc->wake_depth = 1; in irq_set_irq_wake()
928 irqd_clear(&desc->irq_data, IRQD_WAKEUP_STATE); in irq_set_irq_wake()
953 if (!desc->action || in can_request_irq()
954 irqflags & desc->action->flags & IRQF_SHARED) in can_request_irq()
963 struct irq_chip *chip = desc->irq_data.chip; in __irq_set_trigger()
966 if (!chip || !chip->irq_set_type) { in __irq_set_trigger()
969 * flow-types? in __irq_set_trigger()
973 chip ? (chip->name ? : "unknown") : "unknown"); in __irq_set_trigger()
977 if (chip->flags & IRQCHIP_SET_TYPE_MASKED) { in __irq_set_trigger()
978 if (!irqd_irq_masked(&desc->irq_data)) in __irq_set_trigger()
980 if (!irqd_irq_disabled(&desc->irq_data)) in __irq_set_trigger()
986 ret = chip->irq_set_type(&desc->irq_data, flags); in __irq_set_trigger()
991 irqd_clear(&desc->irq_data, IRQD_TRIGGER_MASK); in __irq_set_trigger()
992 irqd_set(&desc->irq_data, flags); in __irq_set_trigger()
996 flags = irqd_get_trigger_type(&desc->irq_data); in __irq_set_trigger()
998 irqd_clear(&desc->irq_data, IRQD_LEVEL); in __irq_set_trigger()
1002 irqd_set(&desc->irq_data, IRQD_LEVEL); in __irq_set_trigger()
1009 flags, irq_desc_get_irq(desc), chip->irq_set_type); in __irq_set_trigger()
1023 return -EINVAL; in irq_set_parent()
1025 desc->parent_irq = parent_irq; in irq_set_parent()
1034 * Default primary interrupt handler for threaded interrupts. Is
1061 * Check whether we need to change the affinity of the interrupt thread.
1068 if (!test_and_clear_bit(IRQTF_AFFINITY, &action->thread_flags)) in irq_thread_check_affinity()
1078 set_bit(IRQTF_AFFINITY, &action->thread_flags); in irq_thread_check_affinity()
1082 raw_spin_lock_irq(&desc->lock); in irq_thread_check_affinity()
1084 * This code is triggered unconditionally. Check the affinity in irq_thread_check_affinity()
1087 if (cpumask_available(desc->irq_common_data.affinity)) { in irq_thread_check_affinity()
1090 m = irq_data_get_effective_affinity_mask(&desc->irq_data); in irq_thread_check_affinity()
1094 raw_spin_unlock_irq(&desc->lock); in irq_thread_check_affinity()
1114 &action->thread_flags)) { in irq_wait_for_interrupt()
1119 return -1; in irq_wait_for_interrupt()
1123 &action->thread_flags)) { in irq_wait_for_interrupt()
1133 * handler finished. unmask if the interrupt has not been disabled and
1139 if (!(desc->istate & IRQS_ONESHOT) || in irq_finalize_oneshot()
1140 action->handler == irq_forced_secondary_handler) in irq_finalize_oneshot()
1144 raw_spin_lock_irq(&desc->lock); in irq_finalize_oneshot()
1150 * The thread is faster done than the hard interrupt handler in irq_finalize_oneshot()
1152 * interrupt can come in again and masks the line, leaves due in irq_finalize_oneshot()
1156 * versus "desc->threads_oneshot |= action->thread_mask;" in in irq_finalize_oneshot()
1160 if (unlikely(irqd_irq_inprogress(&desc->irq_data))) { in irq_finalize_oneshot()
1161 raw_spin_unlock_irq(&desc->lock); in irq_finalize_oneshot()
1172 if (test_bit(IRQTF_RUNTHREAD, &action->thread_flags)) in irq_finalize_oneshot()
1175 desc->threads_oneshot &= ~action->thread_mask; in irq_finalize_oneshot()
1177 if (!desc->threads_oneshot && !irqd_irq_disabled(&desc->irq_data) && in irq_finalize_oneshot()
1178 irqd_irq_masked(&desc->irq_data)) in irq_finalize_oneshot()
1182 raw_spin_unlock_irq(&desc->lock); in irq_finalize_oneshot()
1188 * preemptible - many of them need to sleep and wait for slow busses to
1193 irqreturn_t ret = action->thread_fn(action->irq, action->dev_id); in irq_thread_fn()
1196 atomic_inc(&desc->threads_handled); in irq_thread_fn()
1224 if (atomic_dec_and_test(&desc->threads_active)) in wake_threads_waitq()
1225 wake_up(&desc->wait_for_threads); in wake_threads_waitq()
1234 if (WARN_ON_ONCE(!(current->flags & PF_EXITING))) in irq_thread_dtor()
1240 tsk->comm, tsk->pid, action->irq); in irq_thread_dtor()
1243 desc = irq_to_desc(action->irq); in irq_thread_dtor()
1246 * desc->threads_active and wake possible waiters. in irq_thread_dtor()
1248 if (test_and_clear_bit(IRQTF_RUNTHREAD, &action->thread_flags)) in irq_thread_dtor()
1251 /* Prevent a stale desc->threads_oneshot */ in irq_thread_dtor()
1257 struct irqaction *secondary = action->secondary; in irq_wake_secondary()
1262 raw_spin_lock_irq(&desc->lock); in irq_wake_secondary()
1264 raw_spin_unlock_irq(&desc->lock); in irq_wake_secondary()
1268 * Internal function to notify that a interrupt thread is ready.
1273 set_bit(IRQTF_READY, &action->thread_flags); in irq_thread_set_ready()
1274 wake_up(&desc->wait_for_threads); in irq_thread_set_ready()
1278 * Internal function to wake up a interrupt thread and wait until it is
1284 if (!action || !action->thread) in wake_up_and_wait_for_irq_thread_ready()
1287 wake_up_process(action->thread); in wake_up_and_wait_for_irq_thread_ready()
1288 wait_event(desc->wait_for_threads, in wake_up_and_wait_for_irq_thread_ready()
1289 test_bit(IRQTF_READY, &action->thread_flags)); in wake_up_and_wait_for_irq_thread_ready()
1293 * Interrupt handler thread
1299 struct irq_desc *desc = irq_to_desc(action->irq); in irq_thread()
1308 &action->thread_flags)) in irq_thread()
1337 * irq_wake_thread - wake the irq thread for the action identified by dev_id
1338 * @irq: Interrupt line
1351 raw_spin_lock_irqsave(&desc->lock, flags); in irq_wake_thread()
1353 if (action->dev_id == dev_id) { in irq_wake_thread()
1354 if (action->thread) in irq_wake_thread()
1359 raw_spin_unlock_irqrestore(&desc->lock, flags); in irq_wake_thread()
1367 if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT)) in irq_setup_forced_threading()
1374 if (new->handler == irq_default_primary_handler) in irq_setup_forced_threading()
1377 new->flags |= IRQF_ONESHOT; in irq_setup_forced_threading()
1384 if (new->handler && new->thread_fn) { in irq_setup_forced_threading()
1386 new->secondary = kzalloc(sizeof(struct irqaction), GFP_KERNEL); in irq_setup_forced_threading()
1387 if (!new->secondary) in irq_setup_forced_threading()
1388 return -ENOMEM; in irq_setup_forced_threading()
1389 new->secondary->handler = irq_forced_secondary_handler; in irq_setup_forced_threading()
1390 new->secondary->thread_fn = new->thread_fn; in irq_setup_forced_threading()
1391 new->secondary->dev_id = new->dev_id; in irq_setup_forced_threading()
1392 new->secondary->irq = new->irq; in irq_setup_forced_threading()
1393 new->secondary->name = new->name; in irq_setup_forced_threading()
1396 set_bit(IRQTF_FORCED_THREAD, &new->thread_flags); in irq_setup_forced_threading()
1397 new->thread_fn = new->handler; in irq_setup_forced_threading()
1398 new->handler = irq_default_primary_handler; in irq_setup_forced_threading()
1404 struct irq_data *d = &desc->irq_data; in irq_request_resources()
1405 struct irq_chip *c = d->chip; in irq_request_resources()
1407 return c->irq_request_resources ? c->irq_request_resources(d) : 0; in irq_request_resources()
1412 struct irq_data *d = &desc->irq_data; in irq_release_resources()
1413 struct irq_chip *c = d->chip; in irq_release_resources()
1415 if (c->irq_release_resources) in irq_release_resources()
1416 c->irq_release_resources(d); in irq_release_resources()
1425 if (d->parent_data) in irq_supports_nmi()
1429 if (d->chip->irq_bus_lock || d->chip->irq_bus_sync_unlock) in irq_supports_nmi()
1432 return d->chip->flags & IRQCHIP_SUPPORTS_NMI; in irq_supports_nmi()
1438 struct irq_chip *c = d->chip; in irq_nmi_setup()
1440 return c->irq_nmi_setup ? c->irq_nmi_setup(d) : -EINVAL; in irq_nmi_setup()
1446 struct irq_chip *c = d->chip; in irq_nmi_teardown()
1448 if (c->irq_nmi_teardown) in irq_nmi_teardown()
1449 c->irq_nmi_teardown(d); in irq_nmi_teardown()
1458 t = kthread_create(irq_thread, new, "irq/%d-%s", irq, in setup_irq_thread()
1459 new->name); in setup_irq_thread()
1461 t = kthread_create(irq_thread, new, "irq/%d-s-%s", irq, in setup_irq_thread()
1462 new->name); in setup_irq_thread()
1470 * the thread dies to avoid that the interrupt code in setup_irq_thread()
1473 new->thread = get_task_struct(t); in setup_irq_thread()
1475 * Tell the thread to set its affinity. This is in setup_irq_thread()
1476 * important for shared interrupt handlers as we do in setup_irq_thread()
1481 * on which the requesting code placed the interrupt. in setup_irq_thread()
1483 set_bit(IRQTF_AFFINITY, &new->thread_flags); in setup_irq_thread()
1488 * Internal function to register an irqaction - typically used to
1493 * desc->request_mutex Provides serialization against a concurrent free_irq()
1495 * desc->lock Provides serialization against hard interrupts
1497 * chip_bus_lock and desc->lock are sufficient for all other management and
1498 * interrupt related functions. desc->request_mutex solely serializes
1509 return -EINVAL; in __setup_irq()
1511 if (desc->irq_data.chip == &no_irq_chip) in __setup_irq()
1512 return -ENOSYS; in __setup_irq()
1513 if (!try_module_get(desc->owner)) in __setup_irq()
1514 return -ENODEV; in __setup_irq()
1516 new->irq = irq; in __setup_irq()
1520 * then use the default for this interrupt. in __setup_irq()
1522 if (!(new->flags & IRQF_TRIGGER_MASK)) in __setup_irq()
1523 new->flags |= irqd_get_trigger_type(&desc->irq_data); in __setup_irq()
1526 * Check whether the interrupt nests into another interrupt in __setup_irq()
1531 if (!new->thread_fn) { in __setup_irq()
1532 ret = -EINVAL; in __setup_irq()
1537 * the driver for non nested interrupt handling by the in __setup_irq()
1540 new->handler = irq_nested_primary_handler; in __setup_irq()
1551 * and the interrupt does not nest into another interrupt in __setup_irq()
1554 if (new->thread_fn && !nested) { in __setup_irq()
1558 if (new->secondary) { in __setup_irq()
1559 ret = setup_irq_thread(new->secondary, irq, true); in __setup_irq()
1574 if (desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE) in __setup_irq()
1575 new->flags &= ~IRQF_ONESHOT; in __setup_irq()
1580 * chip bus lock and desc->lock. Also protects against handing out in __setup_irq()
1584 mutex_lock(&desc->request_mutex); in __setup_irq()
1594 if (!desc->action) { in __setup_irq()
1598 new->name, irq, desc->irq_data.chip->name); in __setup_irq()
1605 * protected against a concurrent interrupt and any of the other in __setup_irq()
1607 * desc->request_mutex or the optional bus lock. in __setup_irq()
1609 raw_spin_lock_irqsave(&desc->lock, flags); in __setup_irq()
1610 old_ptr = &desc->action; in __setup_irq()
1619 * Interrupt lines used for NMIs cannot be shared. in __setup_irq()
1625 new->name, irq, desc->irq_data.chip->name); in __setup_irq()
1626 ret = -EINVAL; in __setup_irq()
1634 if (irqd_trigger_type_was_set(&desc->irq_data)) { in __setup_irq()
1635 oldtype = irqd_get_trigger_type(&desc->irq_data); in __setup_irq()
1637 oldtype = new->flags & IRQF_TRIGGER_MASK; in __setup_irq()
1638 irqd_set_trigger_type(&desc->irq_data, oldtype); in __setup_irq()
1641 if (!((old->flags & new->flags) & IRQF_SHARED) || in __setup_irq()
1642 (oldtype != (new->flags & IRQF_TRIGGER_MASK))) in __setup_irq()
1645 if ((old->flags & IRQF_ONESHOT) && in __setup_irq()
1646 (new->flags & IRQF_COND_ONESHOT)) in __setup_irq()
1647 new->flags |= IRQF_ONESHOT; in __setup_irq()
1648 else if ((old->flags ^ new->flags) & IRQF_ONESHOT) in __setup_irq()
1651 /* All handlers must agree on per-cpuness */ in __setup_irq()
1652 if ((old->flags & IRQF_PERCPU) != in __setup_irq()
1653 (new->flags & IRQF_PERCPU)) in __setup_irq()
1656 /* add new interrupt at end of irq queue */ in __setup_irq()
1659 * Or all existing action->thread_mask bits, in __setup_irq()
1663 thread_mask |= old->thread_mask; in __setup_irq()
1664 old_ptr = &old->next; in __setup_irq()
1675 if (new->flags & IRQF_ONESHOT) { in __setup_irq()
1681 ret = -EBUSY; in __setup_irq()
1686 * desc->thread_active to indicate that the in __setup_irq()
1689 * completes. When all threads of a shared interrupt in __setup_irq()
1690 * line have completed desc->threads_active becomes in __setup_irq()
1691 * zero and the interrupt line is unmasked. See in __setup_irq()
1695 * interrupt handlers, then desc->threads_active is in __setup_irq()
1702 * all existing action->thread_mask bits. in __setup_irq()
1704 new->thread_mask = 1UL << ffz(thread_mask); in __setup_irq()
1706 } else if (new->handler == irq_default_primary_handler && in __setup_irq()
1707 !(desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)) { in __setup_irq()
1709 * The interrupt was requested with handler = NULL, so in __setup_irq()
1719 * say for sure which type this interrupt really in __setup_irq()
1724 new->name, irq); in __setup_irq()
1725 ret = -EINVAL; in __setup_irq()
1731 if (new->flags & IRQF_TRIGGER_MASK) { in __setup_irq()
1733 new->flags & IRQF_TRIGGER_MASK); in __setup_irq()
1740 * Activate the interrupt. That activation must happen in __setup_irq()
1743 * that. enable_irq() of an interrupt requested with in __setup_irq()
1754 desc->istate &= ~(IRQS_AUTODETECT | IRQS_SPURIOUS_DISABLED | \ in __setup_irq()
1756 irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS); in __setup_irq()
1758 if (new->flags & IRQF_PERCPU) { in __setup_irq()
1759 irqd_set(&desc->irq_data, IRQD_PER_CPU); in __setup_irq()
1761 if (new->flags & IRQF_NO_DEBUG) in __setup_irq()
1768 if (new->flags & IRQF_ONESHOT) in __setup_irq()
1769 desc->istate |= IRQS_ONESHOT; in __setup_irq()
1772 if (new->flags & IRQF_NOBALANCING) { in __setup_irq()
1774 irqd_set(&desc->irq_data, IRQD_NO_BALANCING); in __setup_irq()
1777 if (!(new->flags & IRQF_NO_AUTOEN) && in __setup_irq()
1783 * auto enable. The sharing interrupt might request in __setup_irq()
1787 WARN_ON_ONCE(new->flags & IRQF_SHARED); in __setup_irq()
1789 desc->depth = 1; in __setup_irq()
1792 } else if (new->flags & IRQF_TRIGGER_MASK) { in __setup_irq()
1793 unsigned int nmsk = new->flags & IRQF_TRIGGER_MASK; in __setup_irq()
1794 unsigned int omsk = irqd_get_trigger_type(&desc->irq_data); in __setup_irq()
1807 desc->irq_count = 0; in __setup_irq()
1808 desc->irqs_unhandled = 0; in __setup_irq()
1814 if (shared && (desc->istate & IRQS_SPURIOUS_DISABLED)) { in __setup_irq()
1815 desc->istate &= ~IRQS_SPURIOUS_DISABLED; in __setup_irq()
1819 raw_spin_unlock_irqrestore(&desc->lock, flags); in __setup_irq()
1821 mutex_unlock(&desc->request_mutex); in __setup_irq()
1826 wake_up_and_wait_for_irq_thread_ready(desc, new->secondary); in __setup_irq()
1829 new->dir = NULL; in __setup_irq()
1834 if (!(new->flags & IRQF_PROBE_SHARED)) { in __setup_irq()
1836 irq, new->flags, new->name, old->flags, old->name); in __setup_irq()
1841 ret = -EBUSY; in __setup_irq()
1844 raw_spin_unlock_irqrestore(&desc->lock, flags); in __setup_irq()
1846 if (!desc->action) in __setup_irq()
1850 mutex_unlock(&desc->request_mutex); in __setup_irq()
1853 if (new->thread) { in __setup_irq()
1854 struct task_struct *t = new->thread; in __setup_irq()
1856 new->thread = NULL; in __setup_irq()
1859 if (new->secondary && new->secondary->thread) { in __setup_irq()
1860 struct task_struct *t = new->secondary->thread; in __setup_irq()
1862 new->secondary->thread = NULL; in __setup_irq()
1866 module_put(desc->owner); in __setup_irq()
1871 * Internal function to unregister an irqaction - used to free
1876 unsigned irq = desc->irq_data.irq; in __free_irq()
1882 mutex_lock(&desc->request_mutex); in __free_irq()
1884 raw_spin_lock_irqsave(&desc->lock, flags); in __free_irq()
1890 action_ptr = &desc->action; in __free_irq()
1895 WARN(1, "Trying to free already-free IRQ %d\n", irq); in __free_irq()
1896 raw_spin_unlock_irqrestore(&desc->lock, flags); in __free_irq()
1898 mutex_unlock(&desc->request_mutex); in __free_irq()
1902 if (action->dev_id == dev_id) in __free_irq()
1904 action_ptr = &action->next; in __free_irq()
1907 /* Found it - now remove it from the list of entries: */ in __free_irq()
1908 *action_ptr = action->next; in __free_irq()
1913 if (!desc->action) { in __free_irq()
1921 if (WARN_ON_ONCE(desc->affinity_hint)) in __free_irq()
1922 desc->affinity_hint = NULL; in __free_irq()
1925 raw_spin_unlock_irqrestore(&desc->lock, flags); in __free_irq()
1936 * The still held desc->request_mutex() protects against a in __free_irq()
1947 * interrupt in flight at the hardware level. in __free_irq()
1953 * It's a shared IRQ -- the driver ought to be prepared for an IRQ in __free_irq()
1960 if (action->flags & IRQF_SHARED) { in __free_irq()
1962 action->handler(irq, dev_id); in __free_irq()
1973 if (action->thread) { in __free_irq()
1974 kthread_stop_put(action->thread); in __free_irq()
1975 if (action->secondary && action->secondary->thread) in __free_irq()
1976 kthread_stop_put(action->secondary->thread); in __free_irq()
1980 if (!desc->action) { in __free_irq()
1987 * There is no interrupt on the fly anymore. Deactivate it in __free_irq()
1990 raw_spin_lock_irqsave(&desc->lock, flags); in __free_irq()
1991 irq_domain_deactivate_irq(&desc->irq_data); in __free_irq()
1992 raw_spin_unlock_irqrestore(&desc->lock, flags); in __free_irq()
1999 mutex_unlock(&desc->request_mutex); in __free_irq()
2001 irq_chip_pm_put(&desc->irq_data); in __free_irq()
2002 module_put(desc->owner); in __free_irq()
2003 kfree(action->secondary); in __free_irq()
2008 * free_irq - free an interrupt allocated with request_irq
2009 * @irq: Interrupt line to free
2012 * Remove an interrupt handler. The handler is removed and if the
2013 * interrupt line is no longer in use by any driver it is disabled.
2014 * On a shared IRQ the caller must ensure the interrupt is disabled
2019 * This function must not be called from interrupt context.
2033 if (WARN_ON(desc->affinity_notify)) in free_irq()
2034 desc->affinity_notify = NULL; in free_irq()
2042 devname = action->name; in free_irq()
2048 /* This function must be called with desc->lock held */
2053 desc->istate &= ~IRQS_NMI; in __cleanup_nmi()
2055 if (!WARN_ON(desc->action == NULL)) { in __cleanup_nmi()
2056 irq_pm_remove_action(desc, desc->action); in __cleanup_nmi()
2057 devname = desc->action->name; in __cleanup_nmi()
2058 unregister_handler_proc(irq, desc->action); in __cleanup_nmi()
2060 kfree(desc->action); in __cleanup_nmi()
2061 desc->action = NULL; in __cleanup_nmi()
2069 irq_chip_pm_put(&desc->irq_data); in __cleanup_nmi()
2070 module_put(desc->owner); in __cleanup_nmi()
2088 if (WARN_ON(desc->depth == 0)) in free_nmi()
2091 raw_spin_lock_irqsave(&desc->lock, flags); in free_nmi()
2096 raw_spin_unlock_irqrestore(&desc->lock, flags); in free_nmi()
2102 * request_threaded_irq - allocate an interrupt line
2103 * @irq: Interrupt line to allocate
2110 * @irqflags: Interrupt type flags
2114 * This call allocates interrupt resources and enables the
2115 * interrupt line and IRQ handling. From the point this
2117 * your handler function must clear any interrupt the board
2119 * and to set up the interrupt handler in the right order.
2123 * still called in hard interrupt context and has to check
2124 * whether the interrupt originates from the device. If yes it
2125 * needs to disable the interrupt on the device and return
2134 * If your interrupt is shared you must pass a non NULL dev_id
2135 * as this is required when freeing the interrupt.
2139 * IRQF_SHARED Interrupt is shared
2141 * IRQF_ONESHOT Run thread_fn with interrupt line masked
2152 return -ENOTCONN; in request_threaded_irq()
2155 * Sanity-check: shared interrupts must pass in a real dev-ID, in request_threaded_irq()
2157 * which interrupt is which (messes up the interrupt freeing in request_threaded_irq()
2161 * The sharing interrupt might request it while it's still disabled in request_threaded_irq()
2171 return -EINVAL; in request_threaded_irq()
2175 return -EINVAL; in request_threaded_irq()
2179 return -EINVAL; in request_threaded_irq()
2183 return -EINVAL; in request_threaded_irq()
2189 return -ENOMEM; in request_threaded_irq()
2191 action->handler = handler; in request_threaded_irq()
2192 action->thread_fn = thread_fn; in request_threaded_irq()
2193 action->flags = irqflags; in request_threaded_irq()
2194 action->name = devname; in request_threaded_irq()
2195 action->dev_id = dev_id; in request_threaded_irq()
2197 retval = irq_chip_pm_get(&desc->irq_data); in request_threaded_irq()
2206 irq_chip_pm_put(&desc->irq_data); in request_threaded_irq()
2207 kfree(action->secondary); in request_threaded_irq()
2214 * It's a shared IRQ -- the driver ought to be prepared for it in request_threaded_irq()
2235 * request_any_context_irq - allocate an interrupt line
2236 * @irq: Interrupt line to allocate
2239 * @flags: Interrupt type flags
2243 * This call allocates interrupt resources and enables the
2244 * interrupt line and IRQ handling. It selects either a
2258 return -ENOTCONN; in request_any_context_irq()
2262 return -EINVAL; in request_any_context_irq()
2276 * request_nmi - allocate an interrupt line for NMI delivery
2277 * @irq: Interrupt line to allocate
2280 * @irqflags: Interrupt type flags
2284 * This call allocates interrupt resources and enables the
2285 * interrupt line and IRQ handling. It sets up the IRQ line
2288 * An interrupt line delivering NMIs cannot be shared and IRQ handling
2291 * Interrupt lines requested for NMI delivering must produce per cpu
2298 * If the interrupt line cannot be used to deliver NMIs, function
2310 return -ENOTCONN; in request_nmi()
2314 return -EINVAL; in request_nmi()
2317 return -EINVAL; in request_nmi()
2320 return -EINVAL; in request_nmi()
2329 return -EINVAL; in request_nmi()
2333 return -ENOMEM; in request_nmi()
2335 action->handler = handler; in request_nmi()
2336 action->flags = irqflags | IRQF_NO_THREAD | IRQF_NOBALANCING; in request_nmi()
2337 action->name = name; in request_nmi()
2338 action->dev_id = dev_id; in request_nmi()
2340 retval = irq_chip_pm_get(&desc->irq_data); in request_nmi()
2348 raw_spin_lock_irqsave(&desc->lock, flags); in request_nmi()
2351 desc->istate |= IRQS_NMI; in request_nmi()
2355 raw_spin_unlock_irqrestore(&desc->lock, flags); in request_nmi()
2356 return -EINVAL; in request_nmi()
2359 raw_spin_unlock_irqrestore(&desc->lock, flags); in request_nmi()
2364 irq_chip_pm_put(&desc->irq_data); in request_nmi()
2382 * use the default for this interrupt. in enable_percpu_irq()
2386 type = irqd_get_trigger_type(&desc->irq_data); in enable_percpu_irq()
2411 * irq_percpu_is_enabled - Check whether the per cpu irq is enabled
2415 * state of a per cpu interrupt on the current cpu.
2428 is_enabled = cpumask_test_cpu(cpu, desc->percpu_enabled); in irq_percpu_is_enabled()
2468 raw_spin_lock_irqsave(&desc->lock, flags); in __free_percpu_irq()
2470 action = desc->action; in __free_percpu_irq()
2471 if (!action || action->percpu_dev_id != dev_id) { in __free_percpu_irq()
2472 WARN(1, "Trying to free already-free IRQ %d\n", irq); in __free_percpu_irq()
2476 if (!cpumask_empty(desc->percpu_enabled)) { in __free_percpu_irq()
2478 irq, cpumask_first(desc->percpu_enabled)); in __free_percpu_irq()
2482 /* Found it - now remove it from the list of entries: */ in __free_percpu_irq()
2483 desc->action = NULL; in __free_percpu_irq()
2485 desc->istate &= ~IRQS_NMI; in __free_percpu_irq()
2487 raw_spin_unlock_irqrestore(&desc->lock, flags); in __free_percpu_irq()
2491 irq_chip_pm_put(&desc->irq_data); in __free_percpu_irq()
2492 module_put(desc->owner); in __free_percpu_irq()
2496 raw_spin_unlock_irqrestore(&desc->lock, flags); in __free_percpu_irq()
2501 * remove_percpu_irq - free a per-cpu interrupt
2502 * @irq: Interrupt line to free
2503 * @act: irqaction for the interrupt
2512 __free_percpu_irq(irq, act->percpu_dev_id); in remove_percpu_irq()
2516 * free_percpu_irq - free an interrupt allocated with request_percpu_irq
2517 * @irq: Interrupt line to free
2520 * Remove a percpu interrupt handler. The handler is removed, but
2521 * the interrupt line is not disabled. This must be done on each
2525 * This function must not be called from interrupt context.
2554 * setup_percpu_irq - setup a per-cpu interrupt
2555 * @irq: Interrupt line to setup
2556 * @act: irqaction for the interrupt
2558 * Used to statically setup per-cpu interrupts in the early boot process.
2566 return -EINVAL; in setup_percpu_irq()
2568 retval = irq_chip_pm_get(&desc->irq_data); in setup_percpu_irq()
2575 irq_chip_pm_put(&desc->irq_data); in setup_percpu_irq()
2581 * __request_percpu_irq - allocate a percpu interrupt line
2582 * @irq: Interrupt line to allocate
2584 * @flags: Interrupt type flags (IRQF_TIMER only)
2588 * This call allocates interrupt resources and enables the
2589 * interrupt on the local CPU. If the interrupt is supposed to be
2593 * Dev_id must be globally unique. It is a per-cpu variable, and
2606 return -EINVAL; in __request_percpu_irq()
2611 return -EINVAL; in __request_percpu_irq()
2614 return -EINVAL; in __request_percpu_irq()
2618 return -ENOMEM; in __request_percpu_irq()
2620 action->handler = handler; in __request_percpu_irq()
2621 action->flags = flags | IRQF_PERCPU | IRQF_NO_SUSPEND; in __request_percpu_irq()
2622 action->name = devname; in __request_percpu_irq()
2623 action->percpu_dev_id = dev_id; in __request_percpu_irq()
2625 retval = irq_chip_pm_get(&desc->irq_data); in __request_percpu_irq()
2634 irq_chip_pm_put(&desc->irq_data); in __request_percpu_irq()
2643 * request_percpu_nmi - allocate a percpu interrupt line for NMI delivery
2644 * @irq: Interrupt line to allocate
2649 * This call allocates interrupt resources for a per CPU NMI. Per CPU NMIs
2653 * Dev_id must be globally unique. It is a per-cpu variable, and
2657 * Interrupt lines requested for NMI delivering should have auto enabling
2660 * If the interrupt line cannot be used to deliver NMIs, function
2672 return -EINVAL; in request_percpu_nmi()
2680 return -EINVAL; in request_percpu_nmi()
2684 return -EINVAL; in request_percpu_nmi()
2688 return -ENOMEM; in request_percpu_nmi()
2690 action->handler = handler; in request_percpu_nmi()
2691 action->flags = IRQF_PERCPU | IRQF_NO_SUSPEND | IRQF_NO_THREAD in request_percpu_nmi()
2693 action->name = name; in request_percpu_nmi()
2694 action->percpu_dev_id = dev_id; in request_percpu_nmi()
2696 retval = irq_chip_pm_get(&desc->irq_data); in request_percpu_nmi()
2704 raw_spin_lock_irqsave(&desc->lock, flags); in request_percpu_nmi()
2705 desc->istate |= IRQS_NMI; in request_percpu_nmi()
2706 raw_spin_unlock_irqrestore(&desc->lock, flags); in request_percpu_nmi()
2711 irq_chip_pm_put(&desc->irq_data); in request_percpu_nmi()
2719 * prepare_percpu_nmi - performs CPU local setup for NMI delivery
2720 * @irq: Interrupt line to prepare for NMI delivery
2722 * This call prepares an interrupt line to deliver NMI on the current CPU,
2723 * before that interrupt line gets enabled with enable_percpu_nmi().
2725 * As a CPU local operation, this should be called from non-preemptible
2728 * If the interrupt line cannot be used to deliver NMIs, function
2742 return -EINVAL; in prepare_percpu_nmi()
2745 KERN_ERR "prepare_percpu_nmi called for a non-NMI interrupt: irq %u\n", in prepare_percpu_nmi()
2747 ret = -EINVAL; in prepare_percpu_nmi()
2763 * teardown_percpu_nmi - undoes NMI setup of IRQ line
2764 * @irq: Interrupt line from which CPU local NMI configuration should be
2771 * As a CPU local operation, this should be called from non-preemptible
2797 int err = -EINVAL; in __irq_get_irqchip_state()
2802 return -ENODEV; in __irq_get_irqchip_state()
2803 if (chip->irq_get_irqchip_state) in __irq_get_irqchip_state()
2806 data = data->parent_data; in __irq_get_irqchip_state()
2813 err = chip->irq_get_irqchip_state(data, which, state); in __irq_get_irqchip_state()
2818 * irq_get_irqchip_state - returns the irqchip state of a interrupt.
2819 * @irq: Interrupt line that is forwarded to a VM
2824 * interrupt, returning into @state the bit corresponding to
2828 * interrupt controller has per-cpu registers.
2836 int err = -EINVAL; in irq_get_irqchip_state()
2852 * irq_set_irqchip_state - set the state of a forwarded interrupt.
2853 * @irq: Interrupt line that is forwarded to a VM
2857 * This call sets the internal irqchip state of an interrupt,
2861 * interrupt controller has per-cpu registers.
2870 int err = -EINVAL; in irq_set_irqchip_state()
2881 err = -ENODEV; in irq_set_irqchip_state()
2884 if (chip->irq_set_irqchip_state) in irq_set_irqchip_state()
2887 data = data->parent_data; in irq_set_irqchip_state()
2894 err = chip->irq_set_irqchip_state(data, which, val); in irq_set_irqchip_state()
2903 * irq_has_action - Check whether an interrupt is requested
2920 * irq_check_status_bit - Check whether bits in the irq descriptor status are set
2934 res = !!(desc->status_use_accessors & bitmask); in irq_check_status_bit()