15 #include <linux/iommu.h>
21 #include "iommu.h"
25  * Intel IOMMU system wide PASID name space:
62 	pages = alloc_pages_node(info->iommu->node,  in intel_pasid_alloc_table()
74 	if (!ecap_coherent(info->iommu->ecap))  in intel_pasid_alloc_table()
149 		entries = alloc_pgtable_page(info->iommu->node, GFP_ATOMIC);  in intel_pasid_get_entry()
164 		if (!ecap_coherent(info->iommu->ecap)) {  in intel_pasid_get_entry()
192 pasid_cache_invalidation_with_pasid(struct intel_iommu *iommu,  in pasid_cache_invalidation_with_pasid()  argument
203 	qi_submit_sync(iommu, &desc, 1, 0);  in pasid_cache_invalidation_with_pasid()
207 devtlb_invalidation_with_pasid(struct intel_iommu *iommu,  in devtlb_invalidation_with_pasid()  argument
228 		qi_flush_dev_iotlb(iommu, sid, pfsid, qdep, 0, 64 - VTD_PAGE_SHIFT);  in devtlb_invalidation_with_pasid()
230 		qi_flush_dev_iotlb_pasid(iommu, sid, pfsid, pasid, qdep, 0, 64 - VTD_PAGE_SHIFT);  in devtlb_invalidation_with_pasid()
233 void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev,  in intel_pasid_tear_down_entry()  argument
239 	spin_lock(&iommu->lock);  in intel_pasid_tear_down_entry()
242 		spin_unlock(&iommu->lock);  in intel_pasid_tear_down_entry()
249 	spin_unlock(&iommu->lock);  in intel_pasid_tear_down_entry()
251 	if (!ecap_coherent(iommu->ecap))  in intel_pasid_tear_down_entry()
254 	pasid_cache_invalidation_with_pasid(iommu, did, pasid);  in intel_pasid_tear_down_entry()
257 		qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);  in intel_pasid_tear_down_entry()
259 		iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);  in intel_pasid_tear_down_entry()
262 	if (!cap_caching_mode(iommu->cap))  in intel_pasid_tear_down_entry()
263 		devtlb_invalidation_with_pasid(iommu, dev, pasid);  in intel_pasid_tear_down_entry()
270 static void pasid_flush_caches(struct intel_iommu *iommu,  in pasid_flush_caches()  argument
274 	if (!ecap_coherent(iommu->ecap))  in pasid_flush_caches()
277 	if (cap_caching_mode(iommu->cap)) {  in pasid_flush_caches()
278 		pasid_cache_invalidation_with_pasid(iommu, did, pasid);  in pasid_flush_caches()
279 		qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);  in pasid_flush_caches()
281 		iommu_flush_write_buffer(iommu);  in pasid_flush_caches()
289 int intel_pasid_setup_first_level(struct intel_iommu *iommu,  in intel_pasid_setup_first_level()  argument
295 	if (!ecap_flts(iommu->ecap)) {  in intel_pasid_setup_first_level()
297 		       iommu->name);  in intel_pasid_setup_first_level()
301 	if ((flags & PASID_FLAG_FL5LP) && !cap_fl5lp_support(iommu->cap)) {  in intel_pasid_setup_first_level()
303 		       iommu->name);  in intel_pasid_setup_first_level()
307 	spin_lock(&iommu->lock);  in intel_pasid_setup_first_level()
310 		spin_unlock(&iommu->lock);  in intel_pasid_setup_first_level()
315 		spin_unlock(&iommu->lock);  in intel_pasid_setup_first_level()
331 	pasid_set_address_width(pte, iommu->agaw);  in intel_pasid_setup_first_level()
332 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));  in intel_pasid_setup_first_level()
338 	spin_unlock(&iommu->lock);  in intel_pasid_setup_first_level()
340 	pasid_flush_caches(iommu, pte, pasid, did);  in intel_pasid_setup_first_level()
346  * Skip top levels of page tables for iommu which has less agaw
350 			   struct intel_iommu *iommu,  in iommu_skip_agaw()  argument
355 	for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {  in iommu_skip_agaw()
367 int intel_pasid_setup_second_level(struct intel_iommu *iommu,  in intel_pasid_setup_second_level()  argument
381 	if (!ecap_slts(iommu->ecap)) {  in intel_pasid_setup_second_level()
383 		       iommu->name);  in intel_pasid_setup_second_level()
388 	agaw = iommu_skip_agaw(domain, iommu, &pgd);  in intel_pasid_setup_second_level()
395 	did = domain_id_iommu(domain, iommu);  in intel_pasid_setup_second_level()
397 	spin_lock(&iommu->lock);  in intel_pasid_setup_second_level()
400 		spin_unlock(&iommu->lock);  in intel_pasid_setup_second_level()
405 		spin_unlock(&iommu->lock);  in intel_pasid_setup_second_level()
415 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));  in intel_pasid_setup_second_level()
420 	spin_unlock(&iommu->lock);  in intel_pasid_setup_second_level()
422 	pasid_flush_caches(iommu, pte, pasid, did);  in intel_pasid_setup_second_level()
430 int intel_pasid_setup_dirty_tracking(struct intel_iommu *iommu,  in intel_pasid_setup_dirty_tracking()  argument
437 	spin_lock(&iommu->lock);  in intel_pasid_setup_dirty_tracking()
441 		spin_unlock(&iommu->lock);  in intel_pasid_setup_dirty_tracking()
451 		spin_unlock(&iommu->lock);  in intel_pasid_setup_dirty_tracking()
460 		spin_unlock(&iommu->lock);  in intel_pasid_setup_dirty_tracking()
468 	spin_unlock(&iommu->lock);  in intel_pasid_setup_dirty_tracking()
470 	if (!ecap_coherent(iommu->ecap))  in intel_pasid_setup_dirty_tracking()
487 	pasid_cache_invalidation_with_pasid(iommu, did, pasid);  in intel_pasid_setup_dirty_tracking()
489 	iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);  in intel_pasid_setup_dirty_tracking()
492 	if (!cap_caching_mode(iommu->cap))  in intel_pasid_setup_dirty_tracking()
493 		devtlb_invalidation_with_pasid(iommu, dev, pasid);  in intel_pasid_setup_dirty_tracking()
501 int intel_pasid_setup_pass_through(struct intel_iommu *iommu,  in intel_pasid_setup_pass_through()  argument
507 	spin_lock(&iommu->lock);  in intel_pasid_setup_pass_through()
510 		spin_unlock(&iommu->lock);  in intel_pasid_setup_pass_through()
515 		spin_unlock(&iommu->lock);  in intel_pasid_setup_pass_through()
521 	pasid_set_address_width(pte, iommu->agaw);  in intel_pasid_setup_pass_through()
524 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));  in intel_pasid_setup_pass_through()
526 	spin_unlock(&iommu->lock);  in intel_pasid_setup_pass_through()
528 	pasid_flush_caches(iommu, pte, pasid, did);  in intel_pasid_setup_pass_through()
536 void intel_pasid_setup_page_snoop_control(struct intel_iommu *iommu,  in intel_pasid_setup_page_snoop_control()  argument
542 	spin_lock(&iommu->lock);  in intel_pasid_setup_page_snoop_control()
545 		spin_unlock(&iommu->lock);  in intel_pasid_setup_page_snoop_control()
551 	spin_unlock(&iommu->lock);  in intel_pasid_setup_page_snoop_control()
553 	if (!ecap_coherent(iommu->ecap))  in intel_pasid_setup_page_snoop_control()
567 	pasid_cache_invalidation_with_pasid(iommu, did, pasid);  in intel_pasid_setup_page_snoop_control()
568 	qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);  in intel_pasid_setup_page_snoop_control()
571 	if (!cap_caching_mode(iommu->cap))  in intel_pasid_setup_page_snoop_control()
572 		devtlb_invalidation_with_pasid(iommu, dev, pasid);  in intel_pasid_setup_page_snoop_control()
577  * @iommu:      IOMMU which the device belong to
586 int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev,  in intel_pasid_setup_nested()  argument
592 	u16 did = domain_id_iommu(domain, iommu);  in intel_pasid_setup_nested()
601 		if (!cap_fl5lp_support(iommu->cap)) {  in intel_pasid_setup_nested()
613 	if ((s1_cfg->flags & IOMMU_VTD_S1_SRE) && !ecap_srs(iommu->ecap)) {  in intel_pasid_setup_nested()
615 				   iommu->name);  in intel_pasid_setup_nested()
619 	if ((s1_cfg->flags & IOMMU_VTD_S1_EAFE) && !ecap_eafs(iommu->ecap)) {  in intel_pasid_setup_nested()
621 				   iommu->name);  in intel_pasid_setup_nested()
625 	spin_lock(&iommu->lock);  in intel_pasid_setup_nested()
628 		spin_unlock(&iommu->lock);  in intel_pasid_setup_nested()
632 		spin_unlock(&iommu->lock);  in intel_pasid_setup_nested()
659 	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));  in intel_pasid_setup_nested()
664 	spin_unlock(&iommu->lock);  in intel_pasid_setup_nested()
666 	pasid_flush_caches(iommu, pte, pasid, did);  in intel_pasid_setup_nested()