Lines Matching +full:irq +full:- +full:device
1 // SPDX-License-Identifier: GPL-2.0-only
17 #include <linux/irqchip/arm-gic-v3.h>
24 #include "vgic-mmio.h"
31 static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
36 int __sz = vgic_its_get_abi(i)->t##_esz; \
37 struct kvm *__k = (i)->dev->kvm; \
44 __ret = -EINVAL; \
53 int __sz = vgic_its_get_abi(i)->t##_esz; \
54 struct kvm *__k = (i)->dev->kvm; \
62 __ret = -EINVAL; \
79 struct vgic_dist *dist = &kvm->arch.vgic; in vgic_add_lpi()
80 struct vgic_irq *irq = vgic_get_irq(kvm, intid), *oldirq; in vgic_add_lpi() local
85 if (irq) in vgic_add_lpi()
86 return irq; in vgic_add_lpi()
88 irq = kzalloc(sizeof(struct vgic_irq), GFP_KERNEL_ACCOUNT); in vgic_add_lpi()
89 if (!irq) in vgic_add_lpi()
90 return ERR_PTR(-ENOMEM); in vgic_add_lpi()
92 ret = xa_reserve_irq(&dist->lpi_xa, intid, GFP_KERNEL_ACCOUNT); in vgic_add_lpi()
94 kfree(irq); in vgic_add_lpi()
98 INIT_LIST_HEAD(&irq->ap_list); in vgic_add_lpi()
99 raw_spin_lock_init(&irq->irq_lock); in vgic_add_lpi()
101 irq->config = VGIC_CONFIG_EDGE; in vgic_add_lpi()
102 kref_init(&irq->refcount); in vgic_add_lpi()
103 irq->intid = intid; in vgic_add_lpi()
104 irq->target_vcpu = vcpu; in vgic_add_lpi()
105 irq->group = 1; in vgic_add_lpi()
107 xa_lock_irqsave(&dist->lpi_xa, flags); in vgic_add_lpi()
113 oldirq = xa_load(&dist->lpi_xa, intid); in vgic_add_lpi()
116 kfree(irq); in vgic_add_lpi()
117 irq = oldirq; in vgic_add_lpi()
122 ret = xa_err(__xa_store(&dist->lpi_xa, intid, irq, 0)); in vgic_add_lpi()
124 xa_release(&dist->lpi_xa, intid); in vgic_add_lpi()
125 kfree(irq); in vgic_add_lpi()
129 xa_unlock_irqrestore(&dist->lpi_xa, flags); in vgic_add_lpi()
142 ret = update_lpi_config(kvm, irq, NULL, false); in vgic_add_lpi()
144 vgic_put_irq(kvm, irq); in vgic_add_lpi()
148 ret = vgic_v3_lpi_sync_pending_status(kvm, irq); in vgic_add_lpi()
150 vgic_put_irq(kvm, irq); in vgic_add_lpi()
154 return irq; in vgic_add_lpi()
177 ((coll)->target_addr != COLLECTION_NOT_MAPPED))
182 struct vgic_irq *irq; member
188 * struct vgic_its_abi - ITS abi ops and settings
190 * @dte_esz: device table entry size
225 return &its_table_abi_versions[its->abi_rev]; in vgic_its_get_abi()
232 its->abi_rev = rev; in vgic_its_set_abi()
234 return abi->commit(its); in vgic_its_set_abi()
238 * Find and returns a device in the device table for an ITS.
243 struct its_device *device; in find_its_device() local
245 list_for_each_entry(device, &its->device_list, dev_list) in find_its_device()
246 if (device_id == device->device_id) in find_its_device()
247 return device; in find_its_device()
254 * Device ID/Event ID pair on an ITS.
260 struct its_device *device; in find_ite() local
263 device = find_its_device(its, device_id); in find_ite()
264 if (device == NULL) in find_ite()
267 list_for_each_entry(ite, &device->itt_head, ite_list) in find_ite()
268 if (ite->event_id == event_id) in find_ite()
276 list_for_each_entry(dev, &(its)->device_list, dev_list) \
277 list_for_each_entry(ite, &(dev)->itt_head, ite_list)
282 #define VITS_MAX_EVENTID (BIT(VITS_TYPER_IDBITS) - 1)
284 #define VITS_MAX_DEVID (BIT(VITS_TYPER_DEVBITS) - 1)
285 #define VITS_DTE_MAX_DEVID_OFFSET (BIT(14) - 1)
286 #define VITS_ITE_MAX_EVENTID_OFFSET (BIT(16) - 1)
296 list_for_each_entry(collection, &its->collection_list, coll_list) { in find_collection()
297 if (coll_id == collection->collection_id) in find_collection()
310 * If filter_vcpu is not NULL, applies only if the IRQ is targeting this
313 static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq, in update_lpi_config() argument
316 u64 propbase = GICR_PROPBASER_ADDRESS(kvm->arch.vgic.propbaser); in update_lpi_config()
321 ret = kvm_read_guest_lock(kvm, propbase + irq->intid - GIC_LPI_OFFSET, in update_lpi_config()
327 raw_spin_lock_irqsave(&irq->irq_lock, flags); in update_lpi_config()
329 if (!filter_vcpu || filter_vcpu == irq->target_vcpu) { in update_lpi_config()
330 irq->priority = LPI_PROP_PRIORITY(prop); in update_lpi_config()
331 irq->enabled = LPI_PROP_ENABLE_BIT(prop); in update_lpi_config()
333 if (!irq->hw) { in update_lpi_config()
334 vgic_queue_irq_unlock(kvm, irq, flags); in update_lpi_config()
339 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); in update_lpi_config()
341 if (irq->hw) in update_lpi_config()
342 return its_prop_update_vlpi(irq->host_irq, prop, needs_inv); in update_lpi_config()
347 static int update_affinity(struct vgic_irq *irq, struct kvm_vcpu *vcpu) in update_affinity() argument
352 raw_spin_lock_irqsave(&irq->irq_lock, flags); in update_affinity()
353 irq->target_vcpu = vcpu; in update_affinity()
354 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); in update_affinity()
356 if (irq->hw) { in update_affinity()
359 ret = its_get_vlpi(irq->host_irq, &map); in update_affinity()
364 atomic_dec(&map.vpe->vlpi_count); in update_affinity()
365 map.vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe; in update_affinity()
366 atomic_inc(&map.vpe->vlpi_count); in update_affinity()
368 ret = its_map_vlpi(irq->host_irq, &map); in update_affinity()
377 return kvm_get_vcpu_by_id(kvm, col->target_addr); in collection_to_vcpu()
390 if (!its_is_collection_mapped(ite->collection)) in update_affinity_ite()
393 vcpu = collection_to_vcpu(kvm, ite->collection); in update_affinity_ite()
394 update_affinity(ite->irq, vcpu); in update_affinity_ite()
404 struct its_device *device; in update_affinity_collection() local
407 for_each_lpi_its(device, ite, its) { in update_affinity_collection()
408 if (ite->collection != coll) in update_affinity_collection()
429 gpa_t pendbase = GICR_PENDBASER_ADDRESS(vcpu->arch.vgic_cpu.pendbaser); in its_sync_lpi_pending_table()
430 struct vgic_dist *dist = &vcpu->kvm->arch.vgic; in its_sync_lpi_pending_table()
432 struct vgic_irq *irq; in its_sync_lpi_pending_table() local
433 int last_byte_offset = -1; in its_sync_lpi_pending_table()
437 xa_for_each(&dist->lpi_xa, intid, irq) { in its_sync_lpi_pending_table()
448 ret = kvm_read_guest_lock(vcpu->kvm, in its_sync_lpi_pending_table()
457 irq = vgic_get_irq(vcpu->kvm, intid); in its_sync_lpi_pending_table()
458 if (!irq) in its_sync_lpi_pending_table()
461 raw_spin_lock_irqsave(&irq->irq_lock, flags); in its_sync_lpi_pending_table()
462 if (irq->target_vcpu == vcpu) in its_sync_lpi_pending_table()
463 irq->pending_latch = pendmask & (1U << bit_nr); in its_sync_lpi_pending_table()
464 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); in its_sync_lpi_pending_table()
465 vgic_put_irq(vcpu->kvm, irq); in its_sync_lpi_pending_table()
484 * DevBits low - as least for the time being. in vgic_mmio_read_its_typer()
488 reg |= GIC_ENCODE_SZ(abi->ite_esz, 4) << GITS_TYPER_ITT_ENTRY_SIZE_SHIFT; in vgic_mmio_read_its_typer()
499 val = (its->abi_rev << GITS_IIDR_REV_SHIFT) & GITS_IIDR_REV_MASK; in vgic_mmio_read_its_iidr()
512 return -EINVAL; in vgic_mmio_uaccess_write_its_iidr()
550 return ERR_PTR(-EINVAL); in __vgic_doorbell_to_its()
552 if (kvm_io_dev->ops != &kvm_io_gic_ops) in __vgic_doorbell_to_its()
553 return ERR_PTR(-EINVAL); in __vgic_doorbell_to_its()
556 if (iodev->iodev_type != IODEV_ITS) in __vgic_doorbell_to_its()
557 return ERR_PTR(-EINVAL); in __vgic_doorbell_to_its()
559 return iodev->its; in __vgic_doorbell_to_its()
573 struct vgic_irq *irq; in vgic_its_check_cache() local
584 irq = xa_load(&its->translation_cache, cache_key); in vgic_its_check_cache()
585 if (!vgic_try_get_irq_kref(irq)) in vgic_its_check_cache()
586 irq = NULL; in vgic_its_check_cache()
590 return irq; in vgic_its_check_cache()
595 struct vgic_irq *irq) in vgic_its_cache_translation() argument
601 if (irq->hw) in vgic_its_cache_translation()
605 * The irq refcount is guaranteed to be nonzero while holding the in vgic_its_cache_translation()
608 lockdep_assert_held(&its->its_lock); in vgic_its_cache_translation()
609 vgic_get_irq_kref(irq); in vgic_its_cache_translation()
611 old = xa_store(&its->translation_cache, cache_key, irq, GFP_KERNEL_ACCOUNT); in vgic_its_cache_translation()
614 * Put the reference taken on @irq if the store fails. Intentionally do in vgic_its_cache_translation()
618 vgic_put_irq(kvm, irq); in vgic_its_cache_translation()
633 struct kvm *kvm = its->dev->kvm; in vgic_its_invalidate_cache()
634 struct vgic_irq *irq; in vgic_its_invalidate_cache() local
637 xa_for_each(&its->translation_cache, idx, irq) { in vgic_its_invalidate_cache()
638 xa_erase(&its->translation_cache, idx); in vgic_its_invalidate_cache()
639 vgic_put_irq(kvm, irq); in vgic_its_invalidate_cache()
650 list_for_each_entry_rcu(dev, &kvm->devices, vm_node) { in vgic_its_invalidate_all_caches()
651 if (dev->ops != &kvm_arm_vgic_its_ops) in vgic_its_invalidate_all_caches()
654 its = dev->private; in vgic_its_invalidate_all_caches()
662 u32 devid, u32 eventid, struct vgic_irq **irq) in vgic_its_resolve_lpi() argument
667 if (!its->enabled) in vgic_its_resolve_lpi()
668 return -EBUSY; in vgic_its_resolve_lpi()
671 if (!ite || !its_is_collection_mapped(ite->collection)) in vgic_its_resolve_lpi()
674 vcpu = collection_to_vcpu(kvm, ite->collection); in vgic_its_resolve_lpi()
679 return -EBUSY; in vgic_its_resolve_lpi()
681 vgic_its_cache_translation(kvm, its, devid, eventid, ite->irq); in vgic_its_resolve_lpi()
683 *irq = ite->irq; in vgic_its_resolve_lpi()
692 return ERR_PTR(-ENODEV); in vgic_msi_to_its()
694 if (!(msi->flags & KVM_MSI_VALID_DEVID)) in vgic_msi_to_its()
695 return ERR_PTR(-EINVAL); in vgic_msi_to_its()
697 address = (u64)msi->address_hi << 32 | msi->address_lo; in vgic_msi_to_its()
704 * and make this IRQ pending, possibly injecting it.
712 struct vgic_irq *irq = NULL; in vgic_its_trigger_msi() local
716 err = vgic_its_resolve_lpi(kvm, its, devid, eventid, &irq); in vgic_its_trigger_msi()
720 if (irq->hw) in vgic_its_trigger_msi()
721 return irq_set_irqchip_state(irq->host_irq, in vgic_its_trigger_msi()
724 raw_spin_lock_irqsave(&irq->irq_lock, flags); in vgic_its_trigger_msi()
725 irq->pending_latch = true; in vgic_its_trigger_msi()
726 vgic_queue_irq_unlock(kvm, irq, flags); in vgic_its_trigger_msi()
733 struct vgic_irq *irq; in vgic_its_inject_cached_translation() local
737 db = (u64)msi->address_hi << 32 | msi->address_lo; in vgic_its_inject_cached_translation()
738 irq = vgic_its_check_cache(kvm, db, msi->devid, msi->data); in vgic_its_inject_cached_translation()
739 if (!irq) in vgic_its_inject_cached_translation()
740 return -EWOULDBLOCK; in vgic_its_inject_cached_translation()
742 raw_spin_lock_irqsave(&irq->irq_lock, flags); in vgic_its_inject_cached_translation()
743 irq->pending_latch = true; in vgic_its_inject_cached_translation()
744 vgic_queue_irq_unlock(kvm, irq, flags); in vgic_its_inject_cached_translation()
745 vgic_put_irq(kvm, irq); in vgic_its_inject_cached_translation()
768 mutex_lock(&its->its_lock); in vgic_its_inject_msi()
769 ret = vgic_its_trigger_msi(kvm, its, msi->devid, msi->data); in vgic_its_inject_msi()
770 mutex_unlock(&its->its_lock); in vgic_its_inject_msi()
789 list_del(&ite->ite_list); in its_free_ite()
792 if (ite->irq) { in its_free_ite()
793 if (ite->irq->hw) in its_free_ite()
794 WARN_ON(its_unmap_vlpi(ite->irq->host_irq)); in its_free_ite()
796 vgic_put_irq(kvm, ite->irq); in its_free_ite()
804 return (le64_to_cpu(its_cmd[word]) >> shift) & (BIT_ULL(size) - 1); in its_cmd_mask_field()
829 if (ite && its_is_collection_mapped(ite->collection)) { in vgic_its_cmd_handle_discard()
830 struct its_device *device = find_its_device(its, device_id); in vgic_its_cmd_handle_discard() local
831 int ite_esz = vgic_its_get_abi(its)->ite_esz; in vgic_its_cmd_handle_discard()
832 gpa_t gpa = device->itt_addr + ite->event_id * ite_esz; in vgic_its_cmd_handle_discard()
866 if (!its_is_collection_mapped(ite->collection)) in vgic_its_cmd_handle_movi()
873 ite->collection = collection; in vgic_its_cmd_handle_movi()
878 return update_affinity(ite->irq, vcpu); in vgic_its_cmd_handle_movi()
887 idx = srcu_read_lock(&its->dev->kvm->srcu); in __is_visible_gfn_locked()
888 ret = kvm_is_visible_gfn(its->dev->kvm, gfn); in __is_visible_gfn_locked()
889 srcu_read_unlock(&its->dev->kvm->srcu, idx); in __is_visible_gfn_locked()
915 /* as GITS_TYPER.CIL == 0, ITS supports 16-bit collection ID */ in vgic_its_check_id()
942 /* Each 1st level entry is represented by a 64-bit value. */ in vgic_its_check_id()
943 if (kvm_read_guest_lock(its->dev->kvm, in vgic_its_check_id()
969 * Translation Table, which starts at device->itt_addr.
971 static bool vgic_its_check_event_id(struct vgic_its *its, struct its_device *device, in vgic_its_check_event_id() argument
975 int ite_esz = abi->ite_esz; in vgic_its_check_event_id()
978 /* max table size is: BIT_ULL(device->num_eventid_bits) * ite_esz */ in vgic_its_check_event_id()
979 if (event_id >= BIT_ULL(device->num_eventid_bits)) in vgic_its_check_event_id()
982 gpa = device->itt_addr + event_id * ite_esz; in vgic_its_check_event_id()
998 return -ENOMEM; in vgic_its_alloc_collection()
1000 collection->collection_id = coll_id; in vgic_its_alloc_collection()
1001 collection->target_addr = COLLECTION_NOT_MAPPED; in vgic_its_alloc_collection()
1003 list_add_tail(&collection->coll_list, &its->collection_list); in vgic_its_alloc_collection()
1012 struct its_device *device; in vgic_its_free_collection() local
1024 for_each_lpi_its(device, ite, its) in vgic_its_free_collection()
1025 if (ite->collection && in vgic_its_free_collection()
1026 ite->collection->collection_id == coll_id) in vgic_its_free_collection()
1027 ite->collection = NULL; in vgic_its_free_collection()
1029 list_del(&collection->coll_list); in vgic_its_free_collection()
1034 static struct its_ite *vgic_its_alloc_ite(struct its_device *device, in vgic_its_alloc_ite() argument
1042 return ERR_PTR(-ENOMEM); in vgic_its_alloc_ite()
1044 ite->event_id = event_id; in vgic_its_alloc_ite()
1045 ite->collection = collection; in vgic_its_alloc_ite()
1047 list_add_tail(&ite->ite_list, &device->itt_head); in vgic_its_alloc_ite()
1063 struct its_device *device; in vgic_its_cmd_handle_mapi() local
1065 struct vgic_irq *irq; in vgic_its_cmd_handle_mapi() local
1068 device = find_its_device(its, device_id); in vgic_its_cmd_handle_mapi()
1069 if (!device) in vgic_its_cmd_handle_mapi()
1072 if (!vgic_its_check_event_id(its, device, event_id)) in vgic_its_cmd_handle_mapi()
1080 lpi_nr >= max_lpis_propbaser(kvm->arch.vgic.propbaser)) in vgic_its_cmd_handle_mapi()
1091 if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL)) in vgic_its_cmd_handle_mapi()
1100 ite = vgic_its_alloc_ite(device, collection, event_id); in vgic_its_cmd_handle_mapi()
1110 irq = vgic_add_lpi(kvm, lpi_nr, vcpu); in vgic_its_cmd_handle_mapi()
1111 if (IS_ERR(irq)) { in vgic_its_cmd_handle_mapi()
1115 return PTR_ERR(irq); in vgic_its_cmd_handle_mapi()
1117 ite->irq = irq; in vgic_its_cmd_handle_mapi()
1124 struct its_device *device) in vgic_its_free_device() argument
1129 * The spec says that unmapping a device with still valid in vgic_its_free_device()
1133 list_for_each_entry_safe(ite, temp, &device->itt_head, ite_list) in vgic_its_free_device()
1138 list_del(&device->dev_list); in vgic_its_free_device()
1139 kfree(device); in vgic_its_free_device()
1147 list_for_each_entry_safe(cur, temp, &its->device_list, dev_list) in vgic_its_free_device_list()
1156 list_for_each_entry_safe(cur, temp, &its->collection_list, coll_list) in vgic_its_free_collection_list()
1157 vgic_its_free_collection(its, cur->collection_id); in vgic_its_free_collection_list()
1165 struct its_device *device; in vgic_its_alloc_device() local
1167 device = kzalloc(sizeof(*device), GFP_KERNEL_ACCOUNT); in vgic_its_alloc_device()
1168 if (!device) in vgic_its_alloc_device()
1169 return ERR_PTR(-ENOMEM); in vgic_its_alloc_device()
1171 device->device_id = device_id; in vgic_its_alloc_device()
1172 device->itt_addr = itt_addr; in vgic_its_alloc_device()
1173 device->num_eventid_bits = num_eventid_bits; in vgic_its_alloc_device()
1174 INIT_LIST_HEAD(&device->itt_head); in vgic_its_alloc_device()
1176 list_add_tail(&device->dev_list, &its->device_list); in vgic_its_alloc_device()
1177 return device; in vgic_its_alloc_device()
1181 * MAPD maps or unmaps a device ID to Interrupt Translation Tables (ITTs).
1191 struct its_device *device; in vgic_its_cmd_handle_mapd() local
1194 if (!vgic_its_check_id(its, its->baser_device_table, device_id, &gpa)) in vgic_its_cmd_handle_mapd()
1200 device = find_its_device(its, device_id); in vgic_its_cmd_handle_mapd()
1203 * The spec says that calling MAPD on an already mapped device in vgic_its_cmd_handle_mapd()
1204 * invalidates all cached data for this device. We implement this in vgic_its_cmd_handle_mapd()
1205 * by removing the mapping and re-establishing it. in vgic_its_cmd_handle_mapd()
1207 if (device) in vgic_its_cmd_handle_mapd()
1208 vgic_its_free_device(kvm, its, device); in vgic_its_cmd_handle_mapd()
1211 * The spec does not say whether unmapping a not-mapped device in vgic_its_cmd_handle_mapd()
1217 device = vgic_its_alloc_device(its, device_id, itt_addr, in vgic_its_cmd_handle_mapd()
1220 return PTR_ERR_OR_ZERO(device); in vgic_its_cmd_handle_mapd()
1252 if (!vgic_its_check_id(its, its->baser_coll_table, in vgic_its_cmd_handle_mapc()
1260 collection->target_addr = vcpu->vcpu_id; in vgic_its_cmd_handle_mapc()
1262 collection->target_addr = vcpu->vcpu_id; in vgic_its_cmd_handle_mapc()
1286 ite->irq->pending_latch = false; in vgic_its_cmd_handle_clear()
1288 if (ite->irq->hw) in vgic_its_cmd_handle_clear()
1289 return irq_set_irqchip_state(ite->irq->host_irq, in vgic_its_cmd_handle_clear()
1295 int vgic_its_inv_lpi(struct kvm *kvm, struct vgic_irq *irq) in vgic_its_inv_lpi() argument
1297 return update_lpi_config(kvm, irq, NULL, true); in vgic_its_inv_lpi()
1316 return vgic_its_inv_lpi(kvm, ite->irq); in vgic_its_cmd_handle_inv()
1320 * vgic_its_invall - invalidate all LPIs targeting a given vcpu
1329 struct kvm *kvm = vcpu->kvm; in vgic_its_invall()
1330 struct vgic_dist *dist = &kvm->arch.vgic; in vgic_its_invall()
1331 struct vgic_irq *irq; in vgic_its_invall() local
1334 xa_for_each(&dist->lpi_xa, intid, irq) { in vgic_its_invall()
1335 irq = vgic_get_irq(kvm, intid); in vgic_its_invall()
1336 if (!irq) in vgic_its_invall()
1339 update_lpi_config(kvm, irq, vcpu, false); in vgic_its_invall()
1340 vgic_put_irq(kvm, irq); in vgic_its_invall()
1343 if (vcpu->arch.vgic_cpu.vgic_v3.its_vpe.its_vm) in vgic_its_invall()
1344 its_invall_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe); in vgic_its_invall()
1350 * The INVALL command requests flushing of all IRQ data in this collection.
1352 * of mapped LPIs and update the configuration for each IRQ which targets
1353 * the specified vcpu. The configuration will be read from the in-memory
1378 * However the spec says that no IRQ must target the old redistributor
1385 struct vgic_dist *dist = &kvm->arch.vgic; in vgic_its_cmd_handle_movall()
1387 struct vgic_irq *irq; in vgic_its_cmd_handle_movall() local
1400 xa_for_each(&dist->lpi_xa, intid, irq) { in vgic_its_cmd_handle_movall()
1401 irq = vgic_get_irq(kvm, intid); in vgic_its_cmd_handle_movall()
1402 if (!irq) in vgic_its_cmd_handle_movall()
1405 update_affinity(irq, vcpu2); in vgic_its_cmd_handle_movall()
1407 vgic_put_irq(kvm, irq); in vgic_its_cmd_handle_movall()
1435 int ret = -ENODEV; in vgic_its_handle_command()
1437 mutex_lock(&its->its_lock); in vgic_its_handle_command()
1477 mutex_unlock(&its->its_lock); in vgic_its_handle_command()
1522 return extract_bytes(its->cbaser, addr & 7, len); in vgic_mmio_read_its_cbaser()
1530 if (its->enabled) in vgic_mmio_write_its_cbaser()
1533 mutex_lock(&its->cmd_lock); in vgic_mmio_write_its_cbaser()
1534 its->cbaser = update_64bit_reg(its->cbaser, addr & 7, len, val); in vgic_mmio_write_its_cbaser()
1535 its->cbaser = vgic_sanitise_its_cbaser(its->cbaser); in vgic_mmio_write_its_cbaser()
1536 its->creadr = 0; in vgic_mmio_write_its_cbaser()
1541 its->cwriter = its->creadr; in vgic_mmio_write_its_cbaser()
1542 mutex_unlock(&its->cmd_lock); in vgic_mmio_write_its_cbaser()
1556 if (!its->enabled) in vgic_its_process_commands()
1559 cbaser = GITS_CBASER_ADDRESS(its->cbaser); in vgic_its_process_commands()
1561 while (its->cwriter != its->creadr) { in vgic_its_process_commands()
1562 int ret = kvm_read_guest_lock(kvm, cbaser + its->creadr, in vgic_its_process_commands()
1574 its->creadr += ITS_CMD_SIZE; in vgic_its_process_commands()
1575 if (its->creadr == ITS_CMD_BUFFER_SIZE(its->cbaser)) in vgic_its_process_commands()
1576 its->creadr = 0; in vgic_its_process_commands()
1595 mutex_lock(&its->cmd_lock); in vgic_mmio_write_its_cwriter()
1597 reg = update_64bit_reg(its->cwriter, addr & 7, len, val); in vgic_mmio_write_its_cwriter()
1599 if (reg >= ITS_CMD_BUFFER_SIZE(its->cbaser)) { in vgic_mmio_write_its_cwriter()
1600 mutex_unlock(&its->cmd_lock); in vgic_mmio_write_its_cwriter()
1603 its->cwriter = reg; in vgic_mmio_write_its_cwriter()
1607 mutex_unlock(&its->cmd_lock); in vgic_mmio_write_its_cwriter()
1614 return extract_bytes(its->cwriter, addr & 0x7, len); in vgic_mmio_read_its_cwriter()
1621 return extract_bytes(its->creadr, addr & 0x7, len); in vgic_mmio_read_its_creadr()
1632 mutex_lock(&its->cmd_lock); in vgic_mmio_uaccess_write_its_creadr()
1634 if (its->enabled) { in vgic_mmio_uaccess_write_its_creadr()
1635 ret = -EBUSY; in vgic_mmio_uaccess_write_its_creadr()
1640 if (cmd_offset >= ITS_CMD_BUFFER_SIZE(its->cbaser)) { in vgic_mmio_uaccess_write_its_creadr()
1641 ret = -EINVAL; in vgic_mmio_uaccess_write_its_creadr()
1645 its->creadr = cmd_offset; in vgic_mmio_uaccess_write_its_creadr()
1647 mutex_unlock(&its->cmd_lock); in vgic_mmio_uaccess_write_its_creadr()
1660 reg = its->baser_device_table; in vgic_mmio_read_its_baser()
1663 reg = its->baser_coll_table; in vgic_mmio_read_its_baser()
1684 if (its->enabled) in vgic_mmio_write_its_baser()
1689 regptr = &its->baser_device_table; in vgic_mmio_write_its_baser()
1690 entry_size = abi->dte_esz; in vgic_mmio_write_its_baser()
1694 regptr = &its->baser_coll_table; in vgic_mmio_write_its_baser()
1695 entry_size = abi->cte_esz; in vgic_mmio_write_its_baser()
1707 reg |= (entry_size - 1) << GITS_BASER_ENTRY_SIZE_SHIFT; in vgic_mmio_write_its_baser()
1715 mutex_lock(&its->its_lock); in vgic_mmio_write_its_baser()
1724 mutex_unlock(&its->its_lock); in vgic_mmio_write_its_baser()
1734 mutex_lock(&its->cmd_lock); in vgic_mmio_read_its_ctlr()
1735 if (its->creadr == its->cwriter) in vgic_mmio_read_its_ctlr()
1737 if (its->enabled) in vgic_mmio_read_its_ctlr()
1739 mutex_unlock(&its->cmd_lock); in vgic_mmio_read_its_ctlr()
1748 mutex_lock(&its->cmd_lock); in vgic_mmio_write_its_ctlr()
1752 * device/collection BASER are invalid in vgic_mmio_write_its_ctlr()
1754 if (!its->enabled && (val & GITS_CTLR_ENABLE) && in vgic_mmio_write_its_ctlr()
1755 (!(its->baser_device_table & GITS_BASER_VALID) || in vgic_mmio_write_its_ctlr()
1756 !(its->baser_coll_table & GITS_BASER_VALID) || in vgic_mmio_write_its_ctlr()
1757 !(its->cbaser & GITS_CBASER_VALID))) in vgic_mmio_write_its_ctlr()
1760 its->enabled = !!(val & GITS_CTLR_ENABLE); in vgic_mmio_write_its_ctlr()
1761 if (!its->enabled) in vgic_mmio_write_its_ctlr()
1771 mutex_unlock(&its->cmd_lock); in vgic_mmio_write_its_ctlr()
1831 if (!(vcpu->arch.vgic_cpu.pendbaser & GICR_PENDBASER_PTZ)) in vgic_enable_lpis()
1838 struct vgic_io_device *iodev = &its->iodev; in vgic_register_its_iodev()
1841 mutex_lock(&kvm->slots_lock); in vgic_register_its_iodev()
1842 if (!IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) { in vgic_register_its_iodev()
1843 ret = -EBUSY; in vgic_register_its_iodev()
1847 its->vgic_its_base = addr; in vgic_register_its_iodev()
1848 iodev->regions = its_registers; in vgic_register_its_iodev()
1849 iodev->nr_regions = ARRAY_SIZE(its_registers); in vgic_register_its_iodev()
1850 kvm_iodevice_init(&iodev->dev, &kvm_io_gic_ops); in vgic_register_its_iodev()
1852 iodev->base_addr = its->vgic_its_base; in vgic_register_its_iodev()
1853 iodev->iodev_type = IODEV_ITS; in vgic_register_its_iodev()
1854 iodev->its = its; in vgic_register_its_iodev()
1855 ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, iodev->base_addr, in vgic_register_its_iodev()
1856 KVM_VGIC_V3_ITS_SIZE, &iodev->dev); in vgic_register_its_iodev()
1858 mutex_unlock(&kvm->slots_lock); in vgic_register_its_iodev()
1880 return -ENODEV; in vgic_its_create()
1884 return -ENOMEM; in vgic_its_create()
1886 mutex_lock(&dev->kvm->arch.config_lock); in vgic_its_create()
1888 if (vgic_initialized(dev->kvm)) { in vgic_its_create()
1889 ret = vgic_v4_init(dev->kvm); in vgic_its_create()
1891 mutex_unlock(&dev->kvm->arch.config_lock); in vgic_its_create()
1897 mutex_init(&its->its_lock); in vgic_its_create()
1898 mutex_init(&its->cmd_lock); in vgic_its_create()
1902 mutex_lock(&its->cmd_lock); in vgic_its_create()
1903 mutex_lock(&its->its_lock); in vgic_its_create()
1904 mutex_unlock(&its->its_lock); in vgic_its_create()
1905 mutex_unlock(&its->cmd_lock); in vgic_its_create()
1908 its->vgic_its_base = VGIC_ADDR_UNDEF; in vgic_its_create()
1910 INIT_LIST_HEAD(&its->device_list); in vgic_its_create()
1911 INIT_LIST_HEAD(&its->collection_list); in vgic_its_create()
1912 xa_init(&its->translation_cache); in vgic_its_create()
1914 dev->kvm->arch.vgic.msis_require_devid = true; in vgic_its_create()
1915 dev->kvm->arch.vgic.has_its = true; in vgic_its_create()
1916 its->enabled = false; in vgic_its_create()
1917 its->dev = dev; in vgic_its_create()
1919 its->baser_device_table = INITIAL_BASER_VALUE | in vgic_its_create()
1921 its->baser_coll_table = INITIAL_BASER_VALUE | in vgic_its_create()
1923 dev->kvm->arch.vgic.propbaser = INITIAL_PROPBASER_VALUE; in vgic_its_create()
1925 dev->private = its; in vgic_its_create()
1927 ret = vgic_its_set_abi(its, NR_ITS_ABIS - 1); in vgic_its_create()
1929 mutex_unlock(&dev->kvm->arch.config_lock); in vgic_its_create()
1936 struct kvm *kvm = kvm_dev->kvm; in vgic_its_destroy()
1937 struct vgic_its *its = kvm_dev->private; in vgic_its_destroy()
1939 mutex_lock(&its->its_lock); in vgic_its_destroy()
1944 xa_destroy(&its->translation_cache); in vgic_its_destroy()
1946 mutex_unlock(&its->its_lock); in vgic_its_destroy()
1955 gpa_t offset = attr->attr; in vgic_its_has_attr_regs()
1961 return -EINVAL; in vgic_its_has_attr_regs()
1967 return -ENXIO; in vgic_its_has_attr_regs()
1982 its = dev->private; in vgic_its_attr_regs_access()
1983 offset = attr->attr; in vgic_its_attr_regs_access()
1986 * Although the spec supports upper/lower 32-bit accesses to in vgic_its_attr_regs_access()
1987 * 64-bit ITS registers, the userspace ABI requires 64-bit in vgic_its_attr_regs_access()
1988 * accesses to all 64-bit wide registers. We therefore only in vgic_its_attr_regs_access()
1989 * support 32-bit accesses to GITS_CTLR, GITS_IIDR and GITS ID in vgic_its_attr_regs_access()
1998 return -EINVAL; in vgic_its_attr_regs_access()
2000 mutex_lock(&dev->kvm->lock); in vgic_its_attr_regs_access()
2002 if (!lock_all_vcpus(dev->kvm)) { in vgic_its_attr_regs_access()
2003 mutex_unlock(&dev->kvm->lock); in vgic_its_attr_regs_access()
2004 return -EBUSY; in vgic_its_attr_regs_access()
2007 mutex_lock(&dev->kvm->arch.config_lock); in vgic_its_attr_regs_access()
2009 if (IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) { in vgic_its_attr_regs_access()
2010 ret = -ENXIO; in vgic_its_attr_regs_access()
2018 ret = -ENXIO; in vgic_its_attr_regs_access()
2022 addr = its->vgic_its_base + offset; in vgic_its_attr_regs_access()
2024 len = region->access_flags & VGIC_ACCESS_64bit ? 8 : 4; in vgic_its_attr_regs_access()
2027 if (region->uaccess_its_write) in vgic_its_attr_regs_access()
2028 ret = region->uaccess_its_write(dev->kvm, its, addr, in vgic_its_attr_regs_access()
2031 region->its_write(dev->kvm, its, addr, len, *reg); in vgic_its_attr_regs_access()
2033 *reg = region->its_read(dev->kvm, its, addr, len); in vgic_its_attr_regs_access()
2036 mutex_unlock(&dev->kvm->arch.config_lock); in vgic_its_attr_regs_access()
2037 unlock_all_vcpus(dev->kvm); in vgic_its_attr_regs_access()
2038 mutex_unlock(&dev->kvm->lock); in vgic_its_attr_regs_access()
2048 if (list_is_last(&dev->dev_list, h)) in compute_next_devid_offset()
2051 next_offset = next->device_id - dev->device_id; in compute_next_devid_offset()
2061 if (list_is_last(&ite->ite_list, h)) in compute_next_eventid_offset()
2064 next_offset = next->event_id - ite->event_id; in compute_next_eventid_offset()
2070 * typedef entry_fn_t - Callback called on a table entry restore path
2083 * scan_its_table - Scan a contiguous table in guest RAM and applies a function
2101 struct kvm *kvm = its->dev->kvm; in scan_its_table()
2128 len -= byte_offset; in scan_its_table()
2134 * vgic_its_save_ite - Save an interrupt translation entry at @gpa
2142 next_offset = compute_next_eventid_offset(&dev->itt_head, ite); in vgic_its_save_ite()
2144 ((u64)ite->irq->intid << KVM_ITS_ITE_PINTID_SHIFT) | in vgic_its_save_ite()
2145 ite->collection->collection_id; in vgic_its_save_ite()
2152 * vgic_its_restore_ite - restore an interrupt translation entry
2164 struct kvm *kvm = its->dev->kvm; in vgic_its_restore_ite()
2168 struct vgic_irq *irq; in vgic_its_restore_ite() local
2184 return -EINVAL; in vgic_its_restore_ite()
2187 if (event_id + offset >= BIT_ULL(dev->num_eventid_bits)) in vgic_its_restore_ite()
2188 return -EINVAL; in vgic_its_restore_ite()
2192 return -EINVAL; in vgic_its_restore_ite()
2195 return -EINVAL; in vgic_its_restore_ite()
2202 vcpu = kvm_get_vcpu_by_id(kvm, collection->target_addr); in vgic_its_restore_ite()
2204 irq = vgic_add_lpi(kvm, lpi_id, vcpu); in vgic_its_restore_ite()
2205 if (IS_ERR(irq)) { in vgic_its_restore_ite()
2207 return PTR_ERR(irq); in vgic_its_restore_ite()
2209 ite->irq = irq; in vgic_its_restore_ite()
2220 if (itea->event_id < iteb->event_id) in vgic_its_ite_cmp()
2221 return -1; in vgic_its_ite_cmp()
2226 static int vgic_its_save_itt(struct vgic_its *its, struct its_device *device) in vgic_its_save_itt() argument
2229 gpa_t base = device->itt_addr; in vgic_its_save_itt()
2232 int ite_esz = abi->ite_esz; in vgic_its_save_itt()
2234 list_sort(NULL, &device->itt_head, vgic_its_ite_cmp); in vgic_its_save_itt()
2236 list_for_each_entry(ite, &device->itt_head, ite_list) { in vgic_its_save_itt()
2237 gpa_t gpa = base + ite->event_id * ite_esz; in vgic_its_save_itt()
2245 if (ite->irq->hw && !kvm_vgic_global_state.has_gicv4_1) in vgic_its_save_itt()
2246 return -EACCES; in vgic_its_save_itt()
2248 ret = vgic_its_save_ite(its, device, ite, gpa); in vgic_its_save_itt()
2256 * vgic_its_restore_itt - restore the ITT of a device
2259 * @dev: device handle
2266 gpa_t base = dev->itt_addr; in vgic_its_restore_itt()
2268 int ite_esz = abi->ite_esz; in vgic_its_restore_itt()
2269 size_t max_size = BIT_ULL(dev->num_eventid_bits) * ite_esz; in vgic_its_restore_itt()
2282 * vgic_its_save_dte - Save a device table entry at a given GPA
2285 * @dev: ITS device
2294 itt_addr_field = dev->itt_addr >> 8; in vgic_its_save_dte()
2295 next_offset = compute_next_devid_offset(&its->device_list, dev); in vgic_its_save_dte()
2299 (dev->num_eventid_bits - 1)); in vgic_its_save_dte()
2306 * vgic_its_restore_dte - restore a device table entry
2309 * @id: device id the DTE corresponds to
2320 u64 baser = its->baser_device_table; in vgic_its_restore_dte()
2342 return -EINVAL; in vgic_its_restore_dte()
2350 vgic_its_free_device(its->dev->kvm, its, dev); in vgic_its_restore_dte()
2363 if (deva->device_id < devb->device_id) in vgic_its_device_cmp()
2364 return -1; in vgic_its_device_cmp()
2370 * vgic_its_save_device_tables - Save the device table and all ITT
2374 * returns the GPA of the device entry
2378 u64 baser = its->baser_device_table; in vgic_its_save_device_tables()
2384 list_sort(NULL, &its->device_list, vgic_its_device_cmp); in vgic_its_save_device_tables()
2386 list_for_each_entry(dev, &its->device_list, dev_list) { in vgic_its_save_device_tables()
2391 dev->device_id, &eaddr)) in vgic_its_save_device_tables()
2392 return -EINVAL; in vgic_its_save_device_tables()
2406 * handle_l1_dte - callback used for L1 device table entries (2 stage case)
2421 int l2_start_id = id * (SZ_64K / abi->dte_esz); in handle_l1_dte()
2423 int dte_esz = abi->dte_esz; in handle_l1_dte()
2441 * vgic_its_restore_device_tables - Restore the device table and all ITT
2447 u64 baser = its->baser_device_table; in vgic_its_restore_device_tables()
2462 l1_esz = abi->dte_esz; in vgic_its_restore_device_tables()
2472 vgic_its_free_device_list(its->dev->kvm, its); in vgic_its_restore_device_tables()
2484 ((u64)collection->target_addr << KVM_ITS_CTE_RDBASE_SHIFT) | in vgic_its_save_cte()
2485 collection->collection_id); in vgic_its_save_cte()
2494 * interpreted as end-of-table), and a negative error value for generic errors.
2499 struct kvm *kvm = its->dev->kvm; in vgic_its_restore_cte()
2516 return -EINVAL; in vgic_its_restore_cte()
2520 return -EEXIST; in vgic_its_restore_cte()
2522 if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL)) in vgic_its_restore_cte()
2523 return -EINVAL; in vgic_its_restore_cte()
2528 collection->target_addr = target_addr; in vgic_its_restore_cte()
2533 * vgic_its_save_collection_table - Save the collection table into
2539 u64 baser = its->baser_coll_table; in vgic_its_save_collection_table()
2543 int ret, cte_esz = abi->cte_esz; in vgic_its_save_collection_table()
2550 list_for_each_entry(collection, &its->collection_list, coll_list) { in vgic_its_save_collection_table()
2569 * vgic_its_restore_collection_table - reads the collection table
2576 u64 baser = its->baser_coll_table; in vgic_its_restore_collection_table()
2577 int cte_esz = abi->cte_esz; in vgic_its_restore_collection_table()
2601 vgic_its_free_collection_list(its->dev->kvm, its); in vgic_its_restore_collection_table()
2607 * vgic_its_save_tables_v0 - Save the ITS tables into guest ARM
2622 * vgic_its_restore_tables_v0 - Restore the ITS tables from guest RAM
2636 vgic_its_free_collection_list(its->dev->kvm, its); in vgic_its_restore_tables_v0()
2645 its->baser_coll_table &= ~GITS_BASER_ENTRY_SIZE_MASK; in vgic_its_commit_v0()
2646 its->baser_device_table &= ~GITS_BASER_ENTRY_SIZE_MASK; in vgic_its_commit_v0()
2648 its->baser_coll_table |= (GIC_ENCODE_SZ(abi->cte_esz, 5) in vgic_its_commit_v0()
2651 its->baser_device_table |= (GIC_ENCODE_SZ(abi->dte_esz, 5) in vgic_its_commit_v0()
2659 its->baser_coll_table &= ~GITS_BASER_VALID; in vgic_its_reset()
2660 its->baser_device_table &= ~GITS_BASER_VALID; in vgic_its_reset()
2661 its->cbaser = 0; in vgic_its_reset()
2662 its->creadr = 0; in vgic_its_reset()
2663 its->cwriter = 0; in vgic_its_reset()
2664 its->enabled = 0; in vgic_its_reset()
2672 switch (attr->group) { in vgic_its_has_attr()
2674 switch (attr->attr) { in vgic_its_has_attr()
2680 switch (attr->attr) { in vgic_its_has_attr()
2694 return -ENXIO; in vgic_its_has_attr()
2705 mutex_lock(&kvm->lock); in vgic_its_ctrl()
2708 mutex_unlock(&kvm->lock); in vgic_its_ctrl()
2709 return -EBUSY; in vgic_its_ctrl()
2712 mutex_lock(&kvm->arch.config_lock); in vgic_its_ctrl()
2713 mutex_lock(&its->its_lock); in vgic_its_ctrl()
2720 ret = abi->save_tables(its); in vgic_its_ctrl()
2723 ret = abi->restore_tables(its); in vgic_its_ctrl()
2727 mutex_unlock(&its->its_lock); in vgic_its_ctrl()
2728 mutex_unlock(&kvm->arch.config_lock); in vgic_its_ctrl()
2730 mutex_unlock(&kvm->lock); in vgic_its_ctrl()
2735 * kvm_arch_allow_write_without_running_vcpu - allow writing guest memory
2746 struct vgic_dist *dist = &kvm->arch.vgic; in kvm_arch_allow_write_without_running_vcpu()
2748 return dist->table_write_in_progress; in kvm_arch_allow_write_without_running_vcpu()
2754 struct vgic_its *its = dev->private; in vgic_its_set_attr()
2757 switch (attr->group) { in vgic_its_set_attr()
2759 u64 __user *uaddr = (u64 __user *)(long)attr->addr; in vgic_its_set_attr()
2760 unsigned long type = (unsigned long)attr->attr; in vgic_its_set_attr()
2764 return -ENODEV; in vgic_its_set_attr()
2767 return -EFAULT; in vgic_its_set_attr()
2769 ret = vgic_check_iorange(dev->kvm, its->vgic_its_base, in vgic_its_set_attr()
2774 return vgic_register_its_iodev(dev->kvm, its, addr); in vgic_its_set_attr()
2777 return vgic_its_ctrl(dev->kvm, its, attr->attr); in vgic_its_set_attr()
2779 u64 __user *uaddr = (u64 __user *)(long)attr->addr; in vgic_its_set_attr()
2783 return -EFAULT; in vgic_its_set_attr()
2788 return -ENXIO; in vgic_its_set_attr()
2794 switch (attr->group) { in vgic_its_get_attr()
2796 struct vgic_its *its = dev->private; in vgic_its_get_attr()
2797 u64 addr = its->vgic_its_base; in vgic_its_get_attr()
2798 u64 __user *uaddr = (u64 __user *)(long)attr->addr; in vgic_its_get_attr()
2799 unsigned long type = (unsigned long)attr->attr; in vgic_its_get_attr()
2802 return -ENODEV; in vgic_its_get_attr()
2805 return -EFAULT; in vgic_its_get_attr()
2809 u64 __user *uaddr = (u64 __user *)(long)attr->addr; in vgic_its_get_attr()
2819 return -ENXIO; in vgic_its_get_attr()
2826 .name = "kvm-arm-vgic-its",