Lines Matching full:iommu
32 #include <linux/intel-iommu.h>
234 drhd->iommu->node = node; in dmar_parse_one_rhsa()
525 printk("IOMMU: can't validate: %llx\n", drhd->address); in check_zero_address()
571 x86_init.iommu.iommu_init = intel_iommu_init; in detect_intel_iommu()
583 struct intel_iommu *iommu; in alloc_iommu() local
595 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL); in alloc_iommu()
596 if (!iommu) in alloc_iommu()
599 iommu->seq_id = iommu_allocated++; in alloc_iommu()
600 sprintf (iommu->name, "dmar%d", iommu->seq_id); in alloc_iommu()
602 iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE); in alloc_iommu()
603 if (!iommu->reg) { in alloc_iommu()
604 printk(KERN_ERR "IOMMU: can't map the region\n"); in alloc_iommu()
607 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG); in alloc_iommu()
608 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG); in alloc_iommu()
610 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) { in alloc_iommu()
615 agaw = iommu_calculate_agaw(iommu); in alloc_iommu()
618 "Cannot get a valid agaw for iommu (seq_id = %d)\n", in alloc_iommu()
619 iommu->seq_id); in alloc_iommu()
622 msagaw = iommu_calculate_max_sagaw(iommu); in alloc_iommu()
625 "Cannot get a valid max agaw for iommu (seq_id = %d)\n", in alloc_iommu()
626 iommu->seq_id); in alloc_iommu()
629 iommu->agaw = agaw; in alloc_iommu()
630 iommu->msagaw = msagaw; in alloc_iommu()
632 iommu->node = -1; in alloc_iommu()
635 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap), in alloc_iommu()
636 cap_max_fault_reg_offset(iommu->cap)); in alloc_iommu()
639 iounmap(iommu->reg); in alloc_iommu()
640 iommu->reg = ioremap(drhd->reg_base_addr, map_size); in alloc_iommu()
641 if (!iommu->reg) { in alloc_iommu()
642 printk(KERN_ERR "IOMMU: can't map the region\n"); in alloc_iommu()
647 ver = readl(iommu->reg + DMAR_VER_REG); in alloc_iommu()
648 pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n", in alloc_iommu()
649 iommu->seq_id, in alloc_iommu()
652 (unsigned long long)iommu->cap, in alloc_iommu()
653 (unsigned long long)iommu->ecap); in alloc_iommu()
655 raw_spin_lock_init(&iommu->register_lock); in alloc_iommu()
657 drhd->iommu = iommu; in alloc_iommu()
661 iounmap(iommu->reg); in alloc_iommu()
663 kfree(iommu); in alloc_iommu()
667 void free_iommu(struct intel_iommu *iommu) in free_iommu() argument
669 if (!iommu) in free_iommu()
672 free_dmar_iommu(iommu); in free_iommu()
674 if (iommu->reg) in free_iommu()
675 iounmap(iommu->reg); in free_iommu()
676 kfree(iommu); in free_iommu()
692 static int qi_check_fault(struct intel_iommu *iommu, int index) in qi_check_fault() argument
696 struct q_inval *qi = iommu->qi; in qi_check_fault()
702 fault = readl(iommu->reg + DMAR_FSTS_REG); in qi_check_fault()
710 head = readl(iommu->reg + DMAR_IQH_REG); in qi_check_fault()
718 __iommu_flush_cache(iommu, &qi->desc[index], in qi_check_fault()
720 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG); in qi_check_fault()
730 head = readl(iommu->reg + DMAR_IQH_REG); in qi_check_fault()
733 tail = readl(iommu->reg + DMAR_IQT_REG); in qi_check_fault()
736 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG); in qi_check_fault()
749 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG); in qi_check_fault()
758 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu) in qi_submit_sync() argument
761 struct q_inval *qi = iommu->qi; in qi_submit_sync()
794 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc)); in qi_submit_sync()
795 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc)); in qi_submit_sync()
804 writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG); in qi_submit_sync()
814 rc = qi_check_fault(iommu, index); in qi_submit_sync()
837 void qi_global_iec(struct intel_iommu *iommu) in qi_global_iec() argument
845 qi_submit_sync(&desc, iommu); in qi_global_iec()
848 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm, in qi_flush_context() argument
857 qi_submit_sync(&desc, iommu); in qi_flush_context()
860 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr, in qi_flush_iotlb() argument
868 if (cap_write_drain(iommu->cap)) in qi_flush_iotlb()
871 if (cap_read_drain(iommu->cap)) in qi_flush_iotlb()
879 qi_submit_sync(&desc, iommu); in qi_flush_iotlb()
882 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep, in qi_flush_dev_iotlb() argument
900 qi_submit_sync(&desc, iommu); in qi_flush_dev_iotlb()
906 void dmar_disable_qi(struct intel_iommu *iommu) in dmar_disable_qi() argument
912 if (!ecap_qis(iommu->ecap)) in dmar_disable_qi()
915 raw_spin_lock_irqsave(&iommu->register_lock, flags); in dmar_disable_qi()
917 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG); in dmar_disable_qi()
924 while ((readl(iommu->reg + DMAR_IQT_REG) != in dmar_disable_qi()
925 readl(iommu->reg + DMAR_IQH_REG)) && in dmar_disable_qi()
929 iommu->gcmd &= ~DMA_GCMD_QIE; in dmar_disable_qi()
930 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); in dmar_disable_qi()
932 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, in dmar_disable_qi()
935 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); in dmar_disable_qi()
941 static void __dmar_enable_qi(struct intel_iommu *iommu) in __dmar_enable_qi() argument
945 struct q_inval *qi = iommu->qi; in __dmar_enable_qi()
950 raw_spin_lock_irqsave(&iommu->register_lock, flags); in __dmar_enable_qi()
953 writel(0, iommu->reg + DMAR_IQT_REG); in __dmar_enable_qi()
955 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc)); in __dmar_enable_qi()
957 iommu->gcmd |= DMA_GCMD_QIE; in __dmar_enable_qi()
958 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); in __dmar_enable_qi()
961 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts); in __dmar_enable_qi()
963 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); in __dmar_enable_qi()
971 int dmar_enable_qi(struct intel_iommu *iommu) in dmar_enable_qi() argument
976 if (!ecap_qis(iommu->ecap)) in dmar_enable_qi()
982 if (iommu->qi) in dmar_enable_qi()
985 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC); in dmar_enable_qi()
986 if (!iommu->qi) in dmar_enable_qi()
989 qi = iommu->qi; in dmar_enable_qi()
992 desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0); in dmar_enable_qi()
995 iommu->qi = 0; in dmar_enable_qi()
1005 iommu->qi = 0; in dmar_enable_qi()
1014 __dmar_enable_qi(iommu); in dmar_enable_qi()
1019 /* iommu interrupt handling. Most stuff are MSI-like. */
1074 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data); in dmar_msi_unmask() local
1078 raw_spin_lock_irqsave(&iommu->register_lock, flag); in dmar_msi_unmask()
1079 writel(0, iommu->reg + DMAR_FECTL_REG); in dmar_msi_unmask()
1081 readl(iommu->reg + DMAR_FECTL_REG); in dmar_msi_unmask()
1082 raw_spin_unlock_irqrestore(&iommu->register_lock, flag); in dmar_msi_unmask()
1088 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data); in dmar_msi_mask() local
1091 raw_spin_lock_irqsave(&iommu->register_lock, flag); in dmar_msi_mask()
1092 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG); in dmar_msi_mask()
1094 readl(iommu->reg + DMAR_FECTL_REG); in dmar_msi_mask()
1095 raw_spin_unlock_irqrestore(&iommu->register_lock, flag); in dmar_msi_mask()
1100 struct intel_iommu *iommu = irq_get_handler_data(irq); in dmar_msi_write() local
1103 raw_spin_lock_irqsave(&iommu->register_lock, flag); in dmar_msi_write()
1104 writel(msg->data, iommu->reg + DMAR_FEDATA_REG); in dmar_msi_write()
1105 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG); in dmar_msi_write()
1106 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG); in dmar_msi_write()
1107 raw_spin_unlock_irqrestore(&iommu->register_lock, flag); in dmar_msi_write()
1112 struct intel_iommu *iommu = irq_get_handler_data(irq); in dmar_msi_read() local
1115 raw_spin_lock_irqsave(&iommu->register_lock, flag); in dmar_msi_read()
1116 msg->data = readl(iommu->reg + DMAR_FEDATA_REG); in dmar_msi_read()
1117 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG); in dmar_msi_read()
1118 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG); in dmar_msi_read()
1119 raw_spin_unlock_irqrestore(&iommu->register_lock, flag); in dmar_msi_read()
1122 static int dmar_fault_do_one(struct intel_iommu *iommu, int type, in dmar_fault_do_one() argument
1151 struct intel_iommu *iommu = dev_id; in dmar_fault() local
1156 raw_spin_lock_irqsave(&iommu->register_lock, flag); in dmar_fault()
1157 fault_status = readl(iommu->reg + DMAR_FSTS_REG); in dmar_fault()
1167 reg = cap_fault_reg_offset(iommu->cap); in dmar_fault()
1176 data = readl(iommu->reg + reg + in dmar_fault()
1184 data = readl(iommu->reg + reg + in dmar_fault()
1188 guest_addr = dmar_readq(iommu->reg + reg + in dmar_fault()
1192 writel(DMA_FRCD_F, iommu->reg + reg + in dmar_fault()
1195 raw_spin_unlock_irqrestore(&iommu->register_lock, flag); in dmar_fault()
1197 dmar_fault_do_one(iommu, type, fault_reason, in dmar_fault()
1201 if (fault_index >= cap_num_fault_regs(iommu->cap)) in dmar_fault()
1203 raw_spin_lock_irqsave(&iommu->register_lock, flag); in dmar_fault()
1207 fault_status = readl(iommu->reg + DMAR_FSTS_REG); in dmar_fault()
1208 writel(fault_status, iommu->reg + DMAR_FSTS_REG); in dmar_fault()
1210 raw_spin_unlock_irqrestore(&iommu->register_lock, flag); in dmar_fault()
1214 int dmar_set_interrupt(struct intel_iommu *iommu) in dmar_set_interrupt() argument
1221 if (iommu->irq) in dmar_set_interrupt()
1226 printk(KERN_ERR "IOMMU: no free vectors\n"); in dmar_set_interrupt()
1230 irq_set_handler_data(irq, iommu); in dmar_set_interrupt()
1231 iommu->irq = irq; in dmar_set_interrupt()
1236 iommu->irq = 0; in dmar_set_interrupt()
1241 ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu); in dmar_set_interrupt()
1243 printk(KERN_ERR "IOMMU: can't request irq\n"); in dmar_set_interrupt()
1256 struct intel_iommu *iommu = drhd->iommu; in enable_drhd_fault_handling() local
1257 ret = dmar_set_interrupt(iommu); in enable_drhd_fault_handling()
1269 dmar_fault(iommu->irq, iommu); in enable_drhd_fault_handling()
1278 int dmar_reenable_qi(struct intel_iommu *iommu) in dmar_reenable_qi() argument
1280 if (!ecap_qis(iommu->ecap)) in dmar_reenable_qi()
1283 if (!iommu->qi) in dmar_reenable_qi()
1289 dmar_disable_qi(iommu); in dmar_reenable_qi()
1295 __dmar_enable_qi(iommu); in dmar_reenable_qi()