1 /* 2 * QEMU emulation of an Intel IOMMU (VT-d) 3 * (DMA Remapping device) 4 * 5 * Copyright (C) 2013 Knut Omang, Oracle <knut.omang@oracle.com> 6 * Copyright (C) 2014 Le Tan, <tamlokveer@gmail.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 18 * You should have received a copy of the GNU General Public License along 19 * with this program; if not, see <http://www.gnu.org/licenses/>. 20 */ 21 22 #include "qemu/osdep.h" 23 #include "qemu/error-report.h" 24 #include "qemu/main-loop.h" 25 #include "qapi/error.h" 26 #include "hw/sysbus.h" 27 #include "intel_iommu_internal.h" 28 #include "hw/pci/pci.h" 29 #include "hw/pci/pci_bus.h" 30 #include "hw/qdev-properties.h" 31 #include "hw/i386/pc.h" 32 #include "hw/i386/apic-msidef.h" 33 #include "hw/i386/x86-iommu.h" 34 #include "hw/pci-host/q35.h" 35 #include "system/kvm.h" 36 #include "system/dma.h" 37 #include "system/system.h" 38 #include "hw/i386/apic_internal.h" 39 #include "kvm/kvm_i386.h" 40 #include "migration/vmstate.h" 41 #include "trace.h" 42 43 /* context entry operations */ 44 #define VTD_CE_GET_RID2PASID(ce) \ 45 ((ce)->val[1] & VTD_SM_CONTEXT_ENTRY_RID2PASID_MASK) 46 #define VTD_CE_GET_PASID_DIR_TABLE(ce) \ 47 ((ce)->val[0] & VTD_PASID_DIR_BASE_ADDR_MASK) 48 49 /* pe operations */ 50 #define VTD_PE_GET_TYPE(pe) ((pe)->val[0] & VTD_SM_PASID_ENTRY_PGTT) 51 #define VTD_PE_GET_FL_LEVEL(pe) \ 52 (4 + (((pe)->val[2] >> 2) & VTD_SM_PASID_ENTRY_FLPM)) 53 #define VTD_PE_GET_SL_LEVEL(pe) \ 54 (2 + (((pe)->val[0] >> 2) & VTD_SM_PASID_ENTRY_AW)) 55 56 /* 57 * PCI bus number (or SID) is not reliable since the device is usaully 58 * initialized before guest can configure the PCI bridge 59 * (SECONDARY_BUS_NUMBER). 60 */ 61 struct vtd_as_key { 62 PCIBus *bus; 63 uint8_t devfn; 64 uint32_t pasid; 65 }; 66 67 /* bus/devfn is PCI device's real BDF not the aliased one */ 68 struct vtd_hiod_key { 69 PCIBus *bus; 70 uint8_t devfn; 71 }; 72 73 struct vtd_as_raw_key { 74 uint16_t sid; 75 uint32_t pasid; 76 }; 77 78 struct vtd_iotlb_key { 79 uint64_t gfn; 80 uint32_t pasid; 81 uint16_t sid; 82 uint8_t level; 83 }; 84 85 static void vtd_address_space_refresh_all(IntelIOMMUState *s); 86 static void vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier *n); 87 88 static void vtd_panic_require_caching_mode(void) 89 { 90 error_report("We need to set caching-mode=on for intel-iommu to enable " 91 "device assignment with IOMMU protection."); 92 exit(1); 93 } 94 95 static void vtd_define_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val, 96 uint64_t wmask, uint64_t w1cmask) 97 { 98 stq_le_p(&s->csr[addr], val); 99 stq_le_p(&s->wmask[addr], wmask); 100 stq_le_p(&s->w1cmask[addr], w1cmask); 101 } 102 103 static void vtd_define_quad_wo(IntelIOMMUState *s, hwaddr addr, uint64_t mask) 104 { 105 stq_le_p(&s->womask[addr], mask); 106 } 107 108 static void vtd_define_long(IntelIOMMUState *s, hwaddr addr, uint32_t val, 109 uint32_t wmask, uint32_t w1cmask) 110 { 111 stl_le_p(&s->csr[addr], val); 112 stl_le_p(&s->wmask[addr], wmask); 113 stl_le_p(&s->w1cmask[addr], w1cmask); 114 } 115 116 static void vtd_define_long_wo(IntelIOMMUState *s, hwaddr addr, uint32_t mask) 117 { 118 stl_le_p(&s->womask[addr], mask); 119 } 120 121 /* "External" get/set operations */ 122 static void vtd_set_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val) 123 { 124 uint64_t oldval = ldq_le_p(&s->csr[addr]); 125 uint64_t wmask = ldq_le_p(&s->wmask[addr]); 126 uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]); 127 stq_le_p(&s->csr[addr], 128 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val)); 129 } 130 131 static void vtd_set_long(IntelIOMMUState *s, hwaddr addr, uint32_t val) 132 { 133 uint32_t oldval = ldl_le_p(&s->csr[addr]); 134 uint32_t wmask = ldl_le_p(&s->wmask[addr]); 135 uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]); 136 stl_le_p(&s->csr[addr], 137 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val)); 138 } 139 140 static uint64_t vtd_get_quad(IntelIOMMUState *s, hwaddr addr) 141 { 142 uint64_t val = ldq_le_p(&s->csr[addr]); 143 uint64_t womask = ldq_le_p(&s->womask[addr]); 144 return val & ~womask; 145 } 146 147 static uint32_t vtd_get_long(IntelIOMMUState *s, hwaddr addr) 148 { 149 uint32_t val = ldl_le_p(&s->csr[addr]); 150 uint32_t womask = ldl_le_p(&s->womask[addr]); 151 return val & ~womask; 152 } 153 154 /* "Internal" get/set operations */ 155 static uint64_t vtd_get_quad_raw(IntelIOMMUState *s, hwaddr addr) 156 { 157 return ldq_le_p(&s->csr[addr]); 158 } 159 160 static uint32_t vtd_get_long_raw(IntelIOMMUState *s, hwaddr addr) 161 { 162 return ldl_le_p(&s->csr[addr]); 163 } 164 165 static void vtd_set_quad_raw(IntelIOMMUState *s, hwaddr addr, uint64_t val) 166 { 167 stq_le_p(&s->csr[addr], val); 168 } 169 170 static uint32_t vtd_set_clear_mask_long(IntelIOMMUState *s, hwaddr addr, 171 uint32_t clear, uint32_t mask) 172 { 173 uint32_t new_val = (ldl_le_p(&s->csr[addr]) & ~clear) | mask; 174 stl_le_p(&s->csr[addr], new_val); 175 return new_val; 176 } 177 178 static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState *s, hwaddr addr, 179 uint64_t clear, uint64_t mask) 180 { 181 uint64_t new_val = (ldq_le_p(&s->csr[addr]) & ~clear) | mask; 182 stq_le_p(&s->csr[addr], new_val); 183 return new_val; 184 } 185 186 static inline void vtd_iommu_lock(IntelIOMMUState *s) 187 { 188 qemu_mutex_lock(&s->iommu_lock); 189 } 190 191 static inline void vtd_iommu_unlock(IntelIOMMUState *s) 192 { 193 qemu_mutex_unlock(&s->iommu_lock); 194 } 195 196 static void vtd_update_scalable_state(IntelIOMMUState *s) 197 { 198 uint64_t val = vtd_get_quad_raw(s, DMAR_RTADDR_REG); 199 200 if (s->scalable_mode) { 201 s->root_scalable = val & VTD_RTADDR_SMT; 202 } 203 } 204 205 static void vtd_update_iq_dw(IntelIOMMUState *s) 206 { 207 uint64_t val = vtd_get_quad_raw(s, DMAR_IQA_REG); 208 209 if (s->ecap & VTD_ECAP_SMTS && 210 val & VTD_IQA_DW_MASK) { 211 s->iq_dw = true; 212 } else { 213 s->iq_dw = false; 214 } 215 } 216 217 /* Whether the address space needs to notify new mappings */ 218 static inline gboolean vtd_as_has_map_notifier(VTDAddressSpace *as) 219 { 220 return as->notifier_flags & IOMMU_NOTIFIER_MAP; 221 } 222 223 /* GHashTable functions */ 224 static gboolean vtd_iotlb_equal(gconstpointer v1, gconstpointer v2) 225 { 226 const struct vtd_iotlb_key *key1 = v1; 227 const struct vtd_iotlb_key *key2 = v2; 228 229 return key1->sid == key2->sid && 230 key1->pasid == key2->pasid && 231 key1->level == key2->level && 232 key1->gfn == key2->gfn; 233 } 234 235 static guint vtd_iotlb_hash(gconstpointer v) 236 { 237 const struct vtd_iotlb_key *key = v; 238 uint64_t hash64 = key->gfn | ((uint64_t)(key->sid) << VTD_IOTLB_SID_SHIFT) | 239 (uint64_t)(key->level - 1) << VTD_IOTLB_LVL_SHIFT | 240 (uint64_t)(key->pasid) << VTD_IOTLB_PASID_SHIFT; 241 242 return (guint)((hash64 >> 32) ^ (hash64 & 0xffffffffU)); 243 } 244 245 static gboolean vtd_as_equal(gconstpointer v1, gconstpointer v2) 246 { 247 const struct vtd_as_key *key1 = v1; 248 const struct vtd_as_key *key2 = v2; 249 250 return (key1->bus == key2->bus) && (key1->devfn == key2->devfn) && 251 (key1->pasid == key2->pasid); 252 } 253 254 /* 255 * Note that we use pointer to PCIBus as the key, so hashing/shifting 256 * based on the pointer value is intended. Note that we deal with 257 * collisions through vtd_as_equal(). 258 */ 259 static guint vtd_as_hash(gconstpointer v) 260 { 261 const struct vtd_as_key *key = v; 262 guint value = (guint)(uintptr_t)key->bus; 263 264 return (guint)(value << 8 | key->devfn); 265 } 266 267 /* Same implementation as vtd_as_hash() */ 268 static guint vtd_hiod_hash(gconstpointer v) 269 { 270 return vtd_as_hash(v); 271 } 272 273 static gboolean vtd_hiod_equal(gconstpointer v1, gconstpointer v2) 274 { 275 const struct vtd_hiod_key *key1 = v1; 276 const struct vtd_hiod_key *key2 = v2; 277 278 return (key1->bus == key2->bus) && (key1->devfn == key2->devfn); 279 } 280 281 static void vtd_hiod_destroy(gpointer v) 282 { 283 object_unref(v); 284 } 285 286 static gboolean vtd_hash_remove_by_domain(gpointer key, gpointer value, 287 gpointer user_data) 288 { 289 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 290 uint16_t domain_id = *(uint16_t *)user_data; 291 return entry->domain_id == domain_id; 292 } 293 294 /* The shift of an addr for a certain level of paging structure */ 295 static inline uint32_t vtd_pt_level_shift(uint32_t level) 296 { 297 assert(level != 0); 298 return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_LEVEL_BITS; 299 } 300 301 static inline uint64_t vtd_pt_level_page_mask(uint32_t level) 302 { 303 return ~((1ULL << vtd_pt_level_shift(level)) - 1); 304 } 305 306 static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value, 307 gpointer user_data) 308 { 309 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 310 VTDIOTLBPageInvInfo *info = (VTDIOTLBPageInvInfo *)user_data; 311 uint64_t gfn = (info->addr >> VTD_PAGE_SHIFT_4K) & info->mask; 312 uint64_t gfn_tlb = (info->addr & entry->mask) >> VTD_PAGE_SHIFT_4K; 313 314 if (entry->domain_id != info->domain_id) { 315 return false; 316 } 317 318 /* 319 * According to spec, IOTLB entries caching first-stage (PGTT=001b) or 320 * nested (PGTT=011b) mapping associated with specified domain-id are 321 * invalidated. Nested isn't supported yet, so only need to check 001b. 322 */ 323 if (entry->pgtt == VTD_SM_PASID_ENTRY_FLT) { 324 return true; 325 } 326 327 return (entry->gfn & info->mask) == gfn || entry->gfn == gfn_tlb; 328 } 329 330 static gboolean vtd_hash_remove_by_page_piotlb(gpointer key, gpointer value, 331 gpointer user_data) 332 { 333 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 334 VTDIOTLBPageInvInfo *info = (VTDIOTLBPageInvInfo *)user_data; 335 uint64_t gfn = (info->addr >> VTD_PAGE_SHIFT_4K) & info->mask; 336 uint64_t gfn_tlb = (info->addr & entry->mask) >> VTD_PAGE_SHIFT_4K; 337 338 /* 339 * According to spec, PASID-based-IOTLB Invalidation in page granularity 340 * doesn't invalidate IOTLB entries caching second-stage (PGTT=010b) 341 * or pass-through (PGTT=100b) mappings. Nested isn't supported yet, 342 * so only need to check first-stage (PGTT=001b) mappings. 343 */ 344 if (entry->pgtt != VTD_SM_PASID_ENTRY_FLT) { 345 return false; 346 } 347 348 return entry->domain_id == info->domain_id && entry->pasid == info->pasid && 349 ((entry->gfn & info->mask) == gfn || entry->gfn == gfn_tlb); 350 } 351 352 /* Reset all the gen of VTDAddressSpace to zero and set the gen of 353 * IntelIOMMUState to 1. Must be called with IOMMU lock held. 354 */ 355 static void vtd_reset_context_cache_locked(IntelIOMMUState *s) 356 { 357 VTDAddressSpace *vtd_as; 358 GHashTableIter as_it; 359 360 trace_vtd_context_cache_reset(); 361 362 g_hash_table_iter_init(&as_it, s->vtd_address_spaces); 363 364 while (g_hash_table_iter_next(&as_it, NULL, (void **)&vtd_as)) { 365 vtd_as->context_cache_entry.context_cache_gen = 0; 366 } 367 s->context_cache_gen = 1; 368 } 369 370 /* Must be called with IOMMU lock held. */ 371 static void vtd_reset_iotlb_locked(IntelIOMMUState *s) 372 { 373 assert(s->iotlb); 374 g_hash_table_remove_all(s->iotlb); 375 } 376 377 static void vtd_reset_iotlb(IntelIOMMUState *s) 378 { 379 vtd_iommu_lock(s); 380 vtd_reset_iotlb_locked(s); 381 vtd_iommu_unlock(s); 382 } 383 384 static void vtd_reset_caches(IntelIOMMUState *s) 385 { 386 vtd_iommu_lock(s); 387 vtd_reset_iotlb_locked(s); 388 vtd_reset_context_cache_locked(s); 389 vtd_iommu_unlock(s); 390 } 391 392 static uint64_t vtd_get_iotlb_gfn(hwaddr addr, uint32_t level) 393 { 394 return (addr & vtd_pt_level_page_mask(level)) >> VTD_PAGE_SHIFT_4K; 395 } 396 397 /* Must be called with IOMMU lock held */ 398 static VTDIOTLBEntry *vtd_lookup_iotlb(IntelIOMMUState *s, uint16_t source_id, 399 uint32_t pasid, hwaddr addr) 400 { 401 struct vtd_iotlb_key key; 402 VTDIOTLBEntry *entry; 403 unsigned level; 404 405 for (level = VTD_PT_LEVEL; level < VTD_PML4_LEVEL; level++) { 406 key.gfn = vtd_get_iotlb_gfn(addr, level); 407 key.level = level; 408 key.sid = source_id; 409 key.pasid = pasid; 410 entry = g_hash_table_lookup(s->iotlb, &key); 411 if (entry) { 412 goto out; 413 } 414 } 415 416 out: 417 return entry; 418 } 419 420 /* Must be with IOMMU lock held */ 421 static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id, 422 uint16_t domain_id, hwaddr addr, uint64_t pte, 423 uint8_t access_flags, uint32_t level, 424 uint32_t pasid, uint8_t pgtt) 425 { 426 VTDIOTLBEntry *entry = g_malloc(sizeof(*entry)); 427 struct vtd_iotlb_key *key = g_malloc(sizeof(*key)); 428 uint64_t gfn = vtd_get_iotlb_gfn(addr, level); 429 430 trace_vtd_iotlb_page_update(source_id, addr, pte, domain_id); 431 if (g_hash_table_size(s->iotlb) >= VTD_IOTLB_MAX_SIZE) { 432 trace_vtd_iotlb_reset("iotlb exceeds size limit"); 433 vtd_reset_iotlb_locked(s); 434 } 435 436 entry->gfn = gfn; 437 entry->domain_id = domain_id; 438 entry->pte = pte; 439 entry->access_flags = access_flags; 440 entry->mask = vtd_pt_level_page_mask(level); 441 entry->pasid = pasid; 442 entry->pgtt = pgtt; 443 444 key->gfn = gfn; 445 key->sid = source_id; 446 key->level = level; 447 key->pasid = pasid; 448 449 g_hash_table_replace(s->iotlb, key, entry); 450 } 451 452 /* Given the reg addr of both the message data and address, generate an 453 * interrupt via MSI. 454 */ 455 static void vtd_generate_interrupt(IntelIOMMUState *s, hwaddr mesg_addr_reg, 456 hwaddr mesg_data_reg) 457 { 458 MSIMessage msi; 459 460 assert(mesg_data_reg < DMAR_REG_SIZE); 461 assert(mesg_addr_reg < DMAR_REG_SIZE); 462 463 msi.address = vtd_get_long_raw(s, mesg_addr_reg); 464 msi.data = vtd_get_long_raw(s, mesg_data_reg); 465 466 trace_vtd_irq_generate(msi.address, msi.data); 467 468 apic_get_class(NULL)->send_msi(&msi); 469 } 470 471 /* Generate a fault event to software via MSI if conditions are met. 472 * Notice that the value of FSTS_REG being passed to it should be the one 473 * before any update. 474 */ 475 static void vtd_generate_fault_event(IntelIOMMUState *s, uint32_t pre_fsts) 476 { 477 if (pre_fsts & VTD_FSTS_PPF || pre_fsts & VTD_FSTS_PFO || 478 pre_fsts & VTD_FSTS_IQE) { 479 error_report_once("There are previous interrupt conditions " 480 "to be serviced by software, fault event " 481 "is not generated"); 482 return; 483 } 484 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, 0, VTD_FECTL_IP); 485 if (vtd_get_long_raw(s, DMAR_FECTL_REG) & VTD_FECTL_IM) { 486 error_report_once("Interrupt Mask set, irq is not generated"); 487 } else { 488 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG); 489 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 490 } 491 } 492 493 /* Check if the Fault (F) field of the Fault Recording Register referenced by 494 * @index is Set. 495 */ 496 static bool vtd_is_frcd_set(IntelIOMMUState *s, uint16_t index) 497 { 498 /* Each reg is 128-bit */ 499 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 500 addr += 8; /* Access the high 64-bit half */ 501 502 assert(index < DMAR_FRCD_REG_NR); 503 504 return vtd_get_quad_raw(s, addr) & VTD_FRCD_F; 505 } 506 507 /* Update the PPF field of Fault Status Register. 508 * Should be called whenever change the F field of any fault recording 509 * registers. 510 */ 511 static void vtd_update_fsts_ppf(IntelIOMMUState *s) 512 { 513 uint32_t i; 514 uint32_t ppf_mask = 0; 515 516 for (i = 0; i < DMAR_FRCD_REG_NR; i++) { 517 if (vtd_is_frcd_set(s, i)) { 518 ppf_mask = VTD_FSTS_PPF; 519 break; 520 } 521 } 522 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_PPF, ppf_mask); 523 trace_vtd_fsts_ppf(!!ppf_mask); 524 } 525 526 static void vtd_set_frcd_and_update_ppf(IntelIOMMUState *s, uint16_t index) 527 { 528 /* Each reg is 128-bit */ 529 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 530 addr += 8; /* Access the high 64-bit half */ 531 532 assert(index < DMAR_FRCD_REG_NR); 533 534 vtd_set_clear_mask_quad(s, addr, 0, VTD_FRCD_F); 535 vtd_update_fsts_ppf(s); 536 } 537 538 /* Must not update F field now, should be done later */ 539 static void vtd_record_frcd(IntelIOMMUState *s, uint16_t index, 540 uint64_t hi, uint64_t lo) 541 { 542 hwaddr frcd_reg_addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 543 544 assert(index < DMAR_FRCD_REG_NR); 545 546 vtd_set_quad_raw(s, frcd_reg_addr, lo); 547 vtd_set_quad_raw(s, frcd_reg_addr + 8, hi); 548 549 trace_vtd_frr_new(index, hi, lo); 550 } 551 552 /* Try to collapse multiple pending faults from the same requester */ 553 static bool vtd_try_collapse_fault(IntelIOMMUState *s, uint16_t source_id) 554 { 555 uint32_t i; 556 uint64_t frcd_reg; 557 hwaddr addr = DMAR_FRCD_REG_OFFSET + 8; /* The high 64-bit half */ 558 559 for (i = 0; i < DMAR_FRCD_REG_NR; i++) { 560 frcd_reg = vtd_get_quad_raw(s, addr); 561 if ((frcd_reg & VTD_FRCD_F) && 562 ((frcd_reg & VTD_FRCD_SID_MASK) == source_id)) { 563 return true; 564 } 565 addr += 16; /* 128-bit for each */ 566 } 567 return false; 568 } 569 570 /* Log and report an DMAR (address translation) fault to software */ 571 static void vtd_report_frcd_fault(IntelIOMMUState *s, uint64_t source_id, 572 uint64_t hi, uint64_t lo) 573 { 574 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 575 576 if (fsts_reg & VTD_FSTS_PFO) { 577 error_report_once("New fault is not recorded due to " 578 "Primary Fault Overflow"); 579 return; 580 } 581 582 if (vtd_try_collapse_fault(s, source_id)) { 583 error_report_once("New fault is not recorded due to " 584 "compression of faults"); 585 return; 586 } 587 588 if (vtd_is_frcd_set(s, s->next_frcd_reg)) { 589 error_report_once("Next Fault Recording Reg is used, " 590 "new fault is not recorded, set PFO field"); 591 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_PFO); 592 return; 593 } 594 595 vtd_record_frcd(s, s->next_frcd_reg, hi, lo); 596 597 if (fsts_reg & VTD_FSTS_PPF) { 598 error_report_once("There are pending faults already, " 599 "fault event is not generated"); 600 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); 601 s->next_frcd_reg++; 602 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) { 603 s->next_frcd_reg = 0; 604 } 605 } else { 606 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_FRI_MASK, 607 VTD_FSTS_FRI(s->next_frcd_reg)); 608 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); /* Will set PPF */ 609 s->next_frcd_reg++; 610 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) { 611 s->next_frcd_reg = 0; 612 } 613 /* This case actually cause the PPF to be Set. 614 * So generate fault event (interrupt). 615 */ 616 vtd_generate_fault_event(s, fsts_reg); 617 } 618 } 619 620 /* Log and report an DMAR (address translation) fault to software */ 621 static void vtd_report_dmar_fault(IntelIOMMUState *s, uint16_t source_id, 622 hwaddr addr, VTDFaultReason fault, 623 bool is_write, bool is_pasid, 624 uint32_t pasid) 625 { 626 uint64_t hi, lo; 627 628 assert(fault < VTD_FR_MAX); 629 630 trace_vtd_dmar_fault(source_id, fault, addr, is_write); 631 632 lo = VTD_FRCD_FI(addr); 633 hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault) | 634 VTD_FRCD_PV(pasid) | VTD_FRCD_PP(is_pasid); 635 if (!is_write) { 636 hi |= VTD_FRCD_T; 637 } 638 639 vtd_report_frcd_fault(s, source_id, hi, lo); 640 } 641 642 643 static void vtd_report_ir_fault(IntelIOMMUState *s, uint64_t source_id, 644 VTDFaultReason fault, uint16_t index) 645 { 646 uint64_t hi, lo; 647 648 lo = VTD_FRCD_IR_IDX(index); 649 hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault); 650 651 vtd_report_frcd_fault(s, source_id, hi, lo); 652 } 653 654 /* Handle Invalidation Queue Errors of queued invalidation interface error 655 * conditions. 656 */ 657 static void vtd_handle_inv_queue_error(IntelIOMMUState *s) 658 { 659 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 660 661 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_IQE); 662 vtd_generate_fault_event(s, fsts_reg); 663 } 664 665 /* Set the IWC field and try to generate an invalidation completion interrupt */ 666 static void vtd_generate_completion_event(IntelIOMMUState *s) 667 { 668 if (vtd_get_long_raw(s, DMAR_ICS_REG) & VTD_ICS_IWC) { 669 trace_vtd_inv_desc_wait_irq("One pending, skip current"); 670 return; 671 } 672 vtd_set_clear_mask_long(s, DMAR_ICS_REG, 0, VTD_ICS_IWC); 673 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, 0, VTD_IECTL_IP); 674 if (vtd_get_long_raw(s, DMAR_IECTL_REG) & VTD_IECTL_IM) { 675 trace_vtd_inv_desc_wait_irq("IM in IECTL_REG is set, " 676 "new event not generated"); 677 return; 678 } else { 679 /* Generate the interrupt event */ 680 trace_vtd_inv_desc_wait_irq("Generating complete event"); 681 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG); 682 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 683 } 684 } 685 686 static inline bool vtd_root_entry_present(IntelIOMMUState *s, 687 VTDRootEntry *re, 688 uint8_t devfn) 689 { 690 if (s->root_scalable && devfn > UINT8_MAX / 2) { 691 return re->hi & VTD_ROOT_ENTRY_P; 692 } 693 694 return re->lo & VTD_ROOT_ENTRY_P; 695 } 696 697 static int vtd_get_root_entry(IntelIOMMUState *s, uint8_t index, 698 VTDRootEntry *re) 699 { 700 dma_addr_t addr; 701 702 addr = s->root + index * sizeof(*re); 703 if (dma_memory_read(&address_space_memory, addr, 704 re, sizeof(*re), MEMTXATTRS_UNSPECIFIED)) { 705 re->lo = 0; 706 return -VTD_FR_ROOT_TABLE_INV; 707 } 708 re->lo = le64_to_cpu(re->lo); 709 re->hi = le64_to_cpu(re->hi); 710 return 0; 711 } 712 713 static inline bool vtd_ce_present(VTDContextEntry *context) 714 { 715 return context->lo & VTD_CONTEXT_ENTRY_P; 716 } 717 718 static int vtd_get_context_entry_from_root(IntelIOMMUState *s, 719 VTDRootEntry *re, 720 uint8_t index, 721 VTDContextEntry *ce) 722 { 723 dma_addr_t addr, ce_size; 724 725 /* we have checked that root entry is present */ 726 ce_size = s->root_scalable ? VTD_CTX_ENTRY_SCALABLE_SIZE : 727 VTD_CTX_ENTRY_LEGACY_SIZE; 728 729 if (s->root_scalable && index > UINT8_MAX / 2) { 730 index = index & (~VTD_DEVFN_CHECK_MASK); 731 addr = re->hi & VTD_ROOT_ENTRY_CTP; 732 } else { 733 addr = re->lo & VTD_ROOT_ENTRY_CTP; 734 } 735 736 addr = addr + index * ce_size; 737 if (dma_memory_read(&address_space_memory, addr, 738 ce, ce_size, MEMTXATTRS_UNSPECIFIED)) { 739 return -VTD_FR_CONTEXT_TABLE_INV; 740 } 741 742 ce->lo = le64_to_cpu(ce->lo); 743 ce->hi = le64_to_cpu(ce->hi); 744 if (ce_size == VTD_CTX_ENTRY_SCALABLE_SIZE) { 745 ce->val[2] = le64_to_cpu(ce->val[2]); 746 ce->val[3] = le64_to_cpu(ce->val[3]); 747 } 748 return 0; 749 } 750 751 static inline dma_addr_t vtd_ce_get_slpt_base(VTDContextEntry *ce) 752 { 753 return ce->lo & VTD_CONTEXT_ENTRY_SLPTPTR; 754 } 755 756 static inline uint64_t vtd_get_pte_addr(uint64_t pte, uint8_t aw) 757 { 758 return pte & VTD_PT_BASE_ADDR_MASK(aw); 759 } 760 761 /* Whether the pte indicates the address of the page frame */ 762 static inline bool vtd_is_last_pte(uint64_t pte, uint32_t level) 763 { 764 return level == VTD_PT_LEVEL || (pte & VTD_PT_PAGE_SIZE_MASK); 765 } 766 767 /* Get the content of a pte located in @base_addr[@index] */ 768 static uint64_t vtd_get_pte(dma_addr_t base_addr, uint32_t index) 769 { 770 uint64_t pte; 771 772 assert(index < VTD_PT_ENTRY_NR); 773 774 if (dma_memory_read(&address_space_memory, 775 base_addr + index * sizeof(pte), 776 &pte, sizeof(pte), MEMTXATTRS_UNSPECIFIED)) { 777 pte = (uint64_t)-1; 778 return pte; 779 } 780 pte = le64_to_cpu(pte); 781 return pte; 782 } 783 784 /* Given an iova and the level of paging structure, return the offset 785 * of current level. 786 */ 787 static inline uint32_t vtd_iova_level_offset(uint64_t iova, uint32_t level) 788 { 789 return (iova >> vtd_pt_level_shift(level)) & 790 ((1ULL << VTD_LEVEL_BITS) - 1); 791 } 792 793 /* Check Capability Register to see if the @level of page-table is supported */ 794 static inline bool vtd_is_sl_level_supported(IntelIOMMUState *s, uint32_t level) 795 { 796 return VTD_CAP_SAGAW_MASK & s->cap & 797 (1ULL << (level - 2 + VTD_CAP_SAGAW_SHIFT)); 798 } 799 800 static inline bool vtd_is_fl_level_supported(IntelIOMMUState *s, uint32_t level) 801 { 802 return level == VTD_PML4_LEVEL; 803 } 804 805 /* Return true if check passed, otherwise false */ 806 static inline bool vtd_pe_type_check(IntelIOMMUState *s, VTDPASIDEntry *pe) 807 { 808 switch (VTD_PE_GET_TYPE(pe)) { 809 case VTD_SM_PASID_ENTRY_FLT: 810 return !!(s->ecap & VTD_ECAP_FLTS); 811 case VTD_SM_PASID_ENTRY_SLT: 812 return !!(s->ecap & VTD_ECAP_SLTS); 813 case VTD_SM_PASID_ENTRY_NESTED: 814 /* Not support NESTED page table type yet */ 815 return false; 816 case VTD_SM_PASID_ENTRY_PT: 817 return !!(s->ecap & VTD_ECAP_PT); 818 default: 819 /* Unknown type */ 820 return false; 821 } 822 } 823 824 static inline bool vtd_pdire_present(VTDPASIDDirEntry *pdire) 825 { 826 return pdire->val & 1; 827 } 828 829 /** 830 * Caller of this function should check present bit if wants 831 * to use pdir entry for further usage except for fpd bit check. 832 */ 833 static int vtd_get_pdire_from_pdir_table(dma_addr_t pasid_dir_base, 834 uint32_t pasid, 835 VTDPASIDDirEntry *pdire) 836 { 837 uint32_t index; 838 dma_addr_t addr, entry_size; 839 840 index = VTD_PASID_DIR_INDEX(pasid); 841 entry_size = VTD_PASID_DIR_ENTRY_SIZE; 842 addr = pasid_dir_base + index * entry_size; 843 if (dma_memory_read(&address_space_memory, addr, 844 pdire, entry_size, MEMTXATTRS_UNSPECIFIED)) { 845 return -VTD_FR_PASID_DIR_ACCESS_ERR; 846 } 847 848 pdire->val = le64_to_cpu(pdire->val); 849 850 return 0; 851 } 852 853 static inline bool vtd_pe_present(VTDPASIDEntry *pe) 854 { 855 return pe->val[0] & VTD_PASID_ENTRY_P; 856 } 857 858 static int vtd_get_pe_in_pasid_leaf_table(IntelIOMMUState *s, 859 uint32_t pasid, 860 dma_addr_t addr, 861 VTDPASIDEntry *pe) 862 { 863 uint8_t pgtt; 864 uint32_t index; 865 dma_addr_t entry_size; 866 867 index = VTD_PASID_TABLE_INDEX(pasid); 868 entry_size = VTD_PASID_ENTRY_SIZE; 869 addr = addr + index * entry_size; 870 if (dma_memory_read(&address_space_memory, addr, 871 pe, entry_size, MEMTXATTRS_UNSPECIFIED)) { 872 return -VTD_FR_PASID_TABLE_ACCESS_ERR; 873 } 874 for (size_t i = 0; i < ARRAY_SIZE(pe->val); i++) { 875 pe->val[i] = le64_to_cpu(pe->val[i]); 876 } 877 878 /* Do translation type check */ 879 if (!vtd_pe_type_check(s, pe)) { 880 return -VTD_FR_PASID_TABLE_ENTRY_INV; 881 } 882 883 pgtt = VTD_PE_GET_TYPE(pe); 884 if (pgtt == VTD_SM_PASID_ENTRY_SLT && 885 !vtd_is_sl_level_supported(s, VTD_PE_GET_SL_LEVEL(pe))) { 886 return -VTD_FR_PASID_TABLE_ENTRY_INV; 887 } 888 889 if (pgtt == VTD_SM_PASID_ENTRY_FLT && 890 !vtd_is_fl_level_supported(s, VTD_PE_GET_FL_LEVEL(pe))) { 891 return -VTD_FR_PASID_TABLE_ENTRY_INV; 892 } 893 894 return 0; 895 } 896 897 /** 898 * Caller of this function should check present bit if wants 899 * to use pasid entry for further usage except for fpd bit check. 900 */ 901 static int vtd_get_pe_from_pdire(IntelIOMMUState *s, 902 uint32_t pasid, 903 VTDPASIDDirEntry *pdire, 904 VTDPASIDEntry *pe) 905 { 906 dma_addr_t addr = pdire->val & VTD_PASID_TABLE_BASE_ADDR_MASK; 907 908 return vtd_get_pe_in_pasid_leaf_table(s, pasid, addr, pe); 909 } 910 911 /** 912 * This function gets a pasid entry from a specified pasid 913 * table (includes dir and leaf table) with a specified pasid. 914 * Sanity check should be done to ensure return a present 915 * pasid entry to caller. 916 */ 917 static int vtd_get_pe_from_pasid_table(IntelIOMMUState *s, 918 dma_addr_t pasid_dir_base, 919 uint32_t pasid, 920 VTDPASIDEntry *pe) 921 { 922 int ret; 923 VTDPASIDDirEntry pdire; 924 925 ret = vtd_get_pdire_from_pdir_table(pasid_dir_base, 926 pasid, &pdire); 927 if (ret) { 928 return ret; 929 } 930 931 if (!vtd_pdire_present(&pdire)) { 932 return -VTD_FR_PASID_DIR_ENTRY_P; 933 } 934 935 ret = vtd_get_pe_from_pdire(s, pasid, &pdire, pe); 936 if (ret) { 937 return ret; 938 } 939 940 if (!vtd_pe_present(pe)) { 941 return -VTD_FR_PASID_ENTRY_P; 942 } 943 944 return 0; 945 } 946 947 static int vtd_ce_get_rid2pasid_entry(IntelIOMMUState *s, 948 VTDContextEntry *ce, 949 VTDPASIDEntry *pe, 950 uint32_t pasid) 951 { 952 dma_addr_t pasid_dir_base; 953 int ret = 0; 954 955 if (pasid == PCI_NO_PASID) { 956 pasid = VTD_CE_GET_RID2PASID(ce); 957 } 958 pasid_dir_base = VTD_CE_GET_PASID_DIR_TABLE(ce); 959 ret = vtd_get_pe_from_pasid_table(s, pasid_dir_base, pasid, pe); 960 961 return ret; 962 } 963 964 static int vtd_ce_get_pasid_fpd(IntelIOMMUState *s, 965 VTDContextEntry *ce, 966 bool *pe_fpd_set, 967 uint32_t pasid) 968 { 969 int ret; 970 dma_addr_t pasid_dir_base; 971 VTDPASIDDirEntry pdire; 972 VTDPASIDEntry pe; 973 974 if (pasid == PCI_NO_PASID) { 975 pasid = VTD_CE_GET_RID2PASID(ce); 976 } 977 pasid_dir_base = VTD_CE_GET_PASID_DIR_TABLE(ce); 978 979 /* 980 * No present bit check since fpd is meaningful even 981 * if the present bit is clear. 982 */ 983 ret = vtd_get_pdire_from_pdir_table(pasid_dir_base, pasid, &pdire); 984 if (ret) { 985 return ret; 986 } 987 988 if (pdire.val & VTD_PASID_DIR_FPD) { 989 *pe_fpd_set = true; 990 return 0; 991 } 992 993 if (!vtd_pdire_present(&pdire)) { 994 return -VTD_FR_PASID_DIR_ENTRY_P; 995 } 996 997 /* 998 * No present bit check since fpd is meaningful even 999 * if the present bit is clear. 1000 */ 1001 ret = vtd_get_pe_from_pdire(s, pasid, &pdire, &pe); 1002 if (ret) { 1003 return ret; 1004 } 1005 1006 if (pe.val[0] & VTD_PASID_ENTRY_FPD) { 1007 *pe_fpd_set = true; 1008 } 1009 1010 return 0; 1011 } 1012 1013 /* Get the page-table level that hardware should use for the second-level 1014 * page-table walk from the Address Width field of context-entry. 1015 */ 1016 static inline uint32_t vtd_ce_get_level(VTDContextEntry *ce) 1017 { 1018 return 2 + (ce->hi & VTD_CONTEXT_ENTRY_AW); 1019 } 1020 1021 static uint32_t vtd_get_iova_level(IntelIOMMUState *s, 1022 VTDContextEntry *ce, 1023 uint32_t pasid) 1024 { 1025 VTDPASIDEntry pe; 1026 1027 if (s->root_scalable) { 1028 vtd_ce_get_rid2pasid_entry(s, ce, &pe, pasid); 1029 if (s->flts) { 1030 return VTD_PE_GET_FL_LEVEL(&pe); 1031 } else { 1032 return VTD_PE_GET_SL_LEVEL(&pe); 1033 } 1034 } 1035 1036 return vtd_ce_get_level(ce); 1037 } 1038 1039 static inline uint32_t vtd_ce_get_agaw(VTDContextEntry *ce) 1040 { 1041 return 30 + (ce->hi & VTD_CONTEXT_ENTRY_AW) * 9; 1042 } 1043 1044 static uint32_t vtd_get_iova_agaw(IntelIOMMUState *s, 1045 VTDContextEntry *ce, 1046 uint32_t pasid) 1047 { 1048 VTDPASIDEntry pe; 1049 1050 if (s->root_scalable) { 1051 vtd_ce_get_rid2pasid_entry(s, ce, &pe, pasid); 1052 return 30 + ((pe.val[0] >> 2) & VTD_SM_PASID_ENTRY_AW) * 9; 1053 } 1054 1055 return vtd_ce_get_agaw(ce); 1056 } 1057 1058 static inline uint32_t vtd_ce_get_type(VTDContextEntry *ce) 1059 { 1060 return ce->lo & VTD_CONTEXT_ENTRY_TT; 1061 } 1062 1063 /* Only for Legacy Mode. Return true if check passed, otherwise false */ 1064 static inline bool vtd_ce_type_check(X86IOMMUState *x86_iommu, 1065 VTDContextEntry *ce) 1066 { 1067 switch (vtd_ce_get_type(ce)) { 1068 case VTD_CONTEXT_TT_MULTI_LEVEL: 1069 /* Always supported */ 1070 break; 1071 case VTD_CONTEXT_TT_DEV_IOTLB: 1072 if (!x86_iommu->dt_supported) { 1073 error_report_once("%s: DT specified but not supported", __func__); 1074 return false; 1075 } 1076 break; 1077 case VTD_CONTEXT_TT_PASS_THROUGH: 1078 if (!x86_iommu->pt_supported) { 1079 error_report_once("%s: PT specified but not supported", __func__); 1080 return false; 1081 } 1082 break; 1083 default: 1084 /* Unknown type */ 1085 error_report_once("%s: unknown ce type: %"PRIu32, __func__, 1086 vtd_ce_get_type(ce)); 1087 return false; 1088 } 1089 return true; 1090 } 1091 1092 static inline uint64_t vtd_iova_limit(IntelIOMMUState *s, 1093 VTDContextEntry *ce, uint8_t aw, 1094 uint32_t pasid) 1095 { 1096 uint32_t ce_agaw = vtd_get_iova_agaw(s, ce, pasid); 1097 return 1ULL << MIN(ce_agaw, aw); 1098 } 1099 1100 /* Return true if IOVA passes range check, otherwise false. */ 1101 static inline bool vtd_iova_sl_range_check(IntelIOMMUState *s, 1102 uint64_t iova, VTDContextEntry *ce, 1103 uint8_t aw, uint32_t pasid) 1104 { 1105 /* 1106 * Check if @iova is above 2^X-1, where X is the minimum of MGAW 1107 * in CAP_REG and AW in context-entry. 1108 */ 1109 return !(iova & ~(vtd_iova_limit(s, ce, aw, pasid) - 1)); 1110 } 1111 1112 static dma_addr_t vtd_get_iova_pgtbl_base(IntelIOMMUState *s, 1113 VTDContextEntry *ce, 1114 uint32_t pasid) 1115 { 1116 VTDPASIDEntry pe; 1117 1118 if (s->root_scalable) { 1119 vtd_ce_get_rid2pasid_entry(s, ce, &pe, pasid); 1120 if (s->flts) { 1121 return pe.val[2] & VTD_SM_PASID_ENTRY_FLPTPTR; 1122 } else { 1123 return pe.val[0] & VTD_SM_PASID_ENTRY_SLPTPTR; 1124 } 1125 } 1126 1127 return vtd_ce_get_slpt_base(ce); 1128 } 1129 1130 /* 1131 * Rsvd field masks for spte: 1132 * vtd_spte_rsvd 4k pages 1133 * vtd_spte_rsvd_large large pages 1134 * 1135 * We support only 3-level and 4-level page tables (see vtd_init() which 1136 * sets only VTD_CAP_SAGAW_39bit and maybe VTD_CAP_SAGAW_48bit bits in s->cap). 1137 */ 1138 #define VTD_SPTE_RSVD_LEN 5 1139 static uint64_t vtd_spte_rsvd[VTD_SPTE_RSVD_LEN]; 1140 static uint64_t vtd_spte_rsvd_large[VTD_SPTE_RSVD_LEN]; 1141 1142 static bool vtd_slpte_nonzero_rsvd(uint64_t slpte, uint32_t level) 1143 { 1144 uint64_t rsvd_mask; 1145 1146 /* 1147 * We should have caught a guest-mis-programmed level earlier, 1148 * via vtd_is_sl_level_supported. 1149 */ 1150 assert(level < VTD_SPTE_RSVD_LEN); 1151 /* 1152 * Zero level doesn't exist. The smallest level is VTD_PT_LEVEL=1 and 1153 * checked by vtd_is_last_pte(). 1154 */ 1155 assert(level); 1156 1157 if ((level == VTD_PD_LEVEL || level == VTD_PDP_LEVEL) && 1158 (slpte & VTD_PT_PAGE_SIZE_MASK)) { 1159 /* large page */ 1160 rsvd_mask = vtd_spte_rsvd_large[level]; 1161 } else { 1162 rsvd_mask = vtd_spte_rsvd[level]; 1163 } 1164 1165 return slpte & rsvd_mask; 1166 } 1167 1168 /* Given the @iova, get relevant @slptep. @slpte_level will be the last level 1169 * of the translation, can be used for deciding the size of large page. 1170 */ 1171 static int vtd_iova_to_slpte(IntelIOMMUState *s, VTDContextEntry *ce, 1172 uint64_t iova, bool is_write, 1173 uint64_t *slptep, uint32_t *slpte_level, 1174 bool *reads, bool *writes, uint8_t aw_bits, 1175 uint32_t pasid) 1176 { 1177 dma_addr_t addr = vtd_get_iova_pgtbl_base(s, ce, pasid); 1178 uint32_t level = vtd_get_iova_level(s, ce, pasid); 1179 uint32_t offset; 1180 uint64_t slpte; 1181 uint64_t access_right_check; 1182 1183 if (!vtd_iova_sl_range_check(s, iova, ce, aw_bits, pasid)) { 1184 error_report_once("%s: detected IOVA overflow (iova=0x%" PRIx64 "," 1185 "pasid=0x%" PRIx32 ")", __func__, iova, pasid); 1186 return -VTD_FR_ADDR_BEYOND_MGAW; 1187 } 1188 1189 /* FIXME: what is the Atomics request here? */ 1190 access_right_check = is_write ? VTD_SL_W : VTD_SL_R; 1191 1192 while (true) { 1193 offset = vtd_iova_level_offset(iova, level); 1194 slpte = vtd_get_pte(addr, offset); 1195 1196 if (slpte == (uint64_t)-1) { 1197 error_report_once("%s: detected read error on DMAR slpte " 1198 "(iova=0x%" PRIx64 ", pasid=0x%" PRIx32 ")", 1199 __func__, iova, pasid); 1200 if (level == vtd_get_iova_level(s, ce, pasid)) { 1201 /* Invalid programming of context-entry */ 1202 return -VTD_FR_CONTEXT_ENTRY_INV; 1203 } else { 1204 return -VTD_FR_PAGING_ENTRY_INV; 1205 } 1206 } 1207 *reads = (*reads) && (slpte & VTD_SL_R); 1208 *writes = (*writes) && (slpte & VTD_SL_W); 1209 if (!(slpte & access_right_check)) { 1210 error_report_once("%s: detected slpte permission error " 1211 "(iova=0x%" PRIx64 ", level=0x%" PRIx32 ", " 1212 "slpte=0x%" PRIx64 ", write=%d, pasid=0x%" 1213 PRIx32 ")", __func__, iova, level, 1214 slpte, is_write, pasid); 1215 return is_write ? -VTD_FR_WRITE : -VTD_FR_READ; 1216 } 1217 if (vtd_slpte_nonzero_rsvd(slpte, level)) { 1218 error_report_once("%s: detected splte reserve non-zero " 1219 "iova=0x%" PRIx64 ", level=0x%" PRIx32 1220 "slpte=0x%" PRIx64 ", pasid=0x%" PRIX32 ")", 1221 __func__, iova, level, slpte, pasid); 1222 return -VTD_FR_PAGING_ENTRY_RSVD; 1223 } 1224 1225 if (vtd_is_last_pte(slpte, level)) { 1226 *slptep = slpte; 1227 *slpte_level = level; 1228 break; 1229 } 1230 addr = vtd_get_pte_addr(slpte, aw_bits); 1231 level--; 1232 } 1233 1234 return 0; 1235 } 1236 1237 typedef int (*vtd_page_walk_hook)(const IOMMUTLBEvent *event, void *private); 1238 1239 /** 1240 * Constant information used during page walking 1241 * 1242 * @hook_fn: hook func to be called when detected page 1243 * @private: private data to be passed into hook func 1244 * @notify_unmap: whether we should notify invalid entries 1245 * @as: VT-d address space of the device 1246 * @aw: maximum address width 1247 * @domain: domain ID of the page walk 1248 */ 1249 typedef struct { 1250 VTDAddressSpace *as; 1251 vtd_page_walk_hook hook_fn; 1252 void *private; 1253 bool notify_unmap; 1254 uint8_t aw; 1255 uint16_t domain_id; 1256 } vtd_page_walk_info; 1257 1258 static int vtd_page_walk_one(IOMMUTLBEvent *event, vtd_page_walk_info *info) 1259 { 1260 VTDAddressSpace *as = info->as; 1261 vtd_page_walk_hook hook_fn = info->hook_fn; 1262 void *private = info->private; 1263 IOMMUTLBEntry *entry = &event->entry; 1264 DMAMap target = { 1265 .iova = entry->iova, 1266 .size = entry->addr_mask, 1267 .translated_addr = entry->translated_addr, 1268 .perm = entry->perm, 1269 }; 1270 const DMAMap *mapped = iova_tree_find(as->iova_tree, &target); 1271 1272 if (event->type == IOMMU_NOTIFIER_UNMAP && !info->notify_unmap) { 1273 trace_vtd_page_walk_one_skip_unmap(entry->iova, entry->addr_mask); 1274 return 0; 1275 } 1276 1277 assert(hook_fn); 1278 1279 /* Update local IOVA mapped ranges */ 1280 if (event->type == IOMMU_NOTIFIER_MAP) { 1281 if (mapped) { 1282 /* If it's exactly the same translation, skip */ 1283 if (!memcmp(mapped, &target, sizeof(target))) { 1284 trace_vtd_page_walk_one_skip_map(entry->iova, entry->addr_mask, 1285 entry->translated_addr); 1286 return 0; 1287 } else { 1288 /* 1289 * Translation changed. Normally this should not 1290 * happen, but it can happen when with buggy guest 1291 * OSes. Note that there will be a small window that 1292 * we don't have map at all. But that's the best 1293 * effort we can do. The ideal way to emulate this is 1294 * atomically modify the PTE to follow what has 1295 * changed, but we can't. One example is that vfio 1296 * driver only has VFIO_IOMMU_[UN]MAP_DMA but no 1297 * interface to modify a mapping (meanwhile it seems 1298 * meaningless to even provide one). Anyway, let's 1299 * mark this as a TODO in case one day we'll have 1300 * a better solution. 1301 */ 1302 IOMMUAccessFlags cache_perm = entry->perm; 1303 int ret; 1304 1305 /* Emulate an UNMAP */ 1306 event->type = IOMMU_NOTIFIER_UNMAP; 1307 entry->perm = IOMMU_NONE; 1308 trace_vtd_page_walk_one(info->domain_id, 1309 entry->iova, 1310 entry->translated_addr, 1311 entry->addr_mask, 1312 entry->perm); 1313 ret = hook_fn(event, private); 1314 if (ret) { 1315 return ret; 1316 } 1317 /* Drop any existing mapping */ 1318 iova_tree_remove(as->iova_tree, target); 1319 /* Recover the correct type */ 1320 event->type = IOMMU_NOTIFIER_MAP; 1321 entry->perm = cache_perm; 1322 } 1323 } 1324 iova_tree_insert(as->iova_tree, &target); 1325 } else { 1326 if (!mapped) { 1327 /* Skip since we didn't map this range at all */ 1328 trace_vtd_page_walk_one_skip_unmap(entry->iova, entry->addr_mask); 1329 return 0; 1330 } 1331 iova_tree_remove(as->iova_tree, target); 1332 } 1333 1334 trace_vtd_page_walk_one(info->domain_id, entry->iova, 1335 entry->translated_addr, entry->addr_mask, 1336 entry->perm); 1337 return hook_fn(event, private); 1338 } 1339 1340 /** 1341 * vtd_page_walk_level - walk over specific level for IOVA range 1342 * 1343 * @addr: base GPA addr to start the walk 1344 * @start: IOVA range start address 1345 * @end: IOVA range end address (start <= addr < end) 1346 * @read: whether parent level has read permission 1347 * @write: whether parent level has write permission 1348 * @info: constant information for the page walk 1349 */ 1350 static int vtd_page_walk_level(dma_addr_t addr, uint64_t start, 1351 uint64_t end, uint32_t level, bool read, 1352 bool write, vtd_page_walk_info *info) 1353 { 1354 bool read_cur, write_cur, entry_valid; 1355 uint32_t offset; 1356 uint64_t slpte; 1357 uint64_t subpage_size, subpage_mask; 1358 IOMMUTLBEvent event; 1359 uint64_t iova = start; 1360 uint64_t iova_next; 1361 int ret = 0; 1362 1363 trace_vtd_page_walk_level(addr, level, start, end); 1364 1365 subpage_size = 1ULL << vtd_pt_level_shift(level); 1366 subpage_mask = vtd_pt_level_page_mask(level); 1367 1368 while (iova < end) { 1369 iova_next = (iova & subpage_mask) + subpage_size; 1370 1371 offset = vtd_iova_level_offset(iova, level); 1372 slpte = vtd_get_pte(addr, offset); 1373 1374 if (slpte == (uint64_t)-1) { 1375 trace_vtd_page_walk_skip_read(iova, iova_next); 1376 goto next; 1377 } 1378 1379 if (vtd_slpte_nonzero_rsvd(slpte, level)) { 1380 trace_vtd_page_walk_skip_reserve(iova, iova_next); 1381 goto next; 1382 } 1383 1384 /* Permissions are stacked with parents' */ 1385 read_cur = read && (slpte & VTD_SL_R); 1386 write_cur = write && (slpte & VTD_SL_W); 1387 1388 /* 1389 * As long as we have either read/write permission, this is a 1390 * valid entry. The rule works for both page entries and page 1391 * table entries. 1392 */ 1393 entry_valid = read_cur | write_cur; 1394 1395 if (!vtd_is_last_pte(slpte, level) && entry_valid) { 1396 /* 1397 * This is a valid PDE (or even bigger than PDE). We need 1398 * to walk one further level. 1399 */ 1400 ret = vtd_page_walk_level(vtd_get_pte_addr(slpte, info->aw), 1401 iova, MIN(iova_next, end), level - 1, 1402 read_cur, write_cur, info); 1403 } else { 1404 /* 1405 * This means we are either: 1406 * 1407 * (1) the real page entry (either 4K page, or huge page) 1408 * (2) the whole range is invalid 1409 * 1410 * In either case, we send an IOTLB notification down. 1411 */ 1412 event.entry.target_as = &address_space_memory; 1413 event.entry.iova = iova & subpage_mask; 1414 event.entry.perm = IOMMU_ACCESS_FLAG(read_cur, write_cur); 1415 event.entry.addr_mask = ~subpage_mask; 1416 /* NOTE: this is only meaningful if entry_valid == true */ 1417 event.entry.translated_addr = vtd_get_pte_addr(slpte, info->aw); 1418 event.type = event.entry.perm ? IOMMU_NOTIFIER_MAP : 1419 IOMMU_NOTIFIER_UNMAP; 1420 ret = vtd_page_walk_one(&event, info); 1421 } 1422 1423 if (ret < 0) { 1424 return ret; 1425 } 1426 1427 next: 1428 iova = iova_next; 1429 } 1430 1431 return 0; 1432 } 1433 1434 /** 1435 * vtd_page_walk - walk specific IOVA range, and call the hook 1436 * 1437 * @s: intel iommu state 1438 * @ce: context entry to walk upon 1439 * @start: IOVA address to start the walk 1440 * @end: IOVA range end address (start <= addr < end) 1441 * @info: page walking information struct 1442 */ 1443 static int vtd_page_walk(IntelIOMMUState *s, VTDContextEntry *ce, 1444 uint64_t start, uint64_t end, 1445 vtd_page_walk_info *info, 1446 uint32_t pasid) 1447 { 1448 dma_addr_t addr = vtd_get_iova_pgtbl_base(s, ce, pasid); 1449 uint32_t level = vtd_get_iova_level(s, ce, pasid); 1450 1451 if (!vtd_iova_sl_range_check(s, start, ce, info->aw, pasid)) { 1452 return -VTD_FR_ADDR_BEYOND_MGAW; 1453 } 1454 1455 if (!vtd_iova_sl_range_check(s, end, ce, info->aw, pasid)) { 1456 /* Fix end so that it reaches the maximum */ 1457 end = vtd_iova_limit(s, ce, info->aw, pasid); 1458 } 1459 1460 return vtd_page_walk_level(addr, start, end, level, true, true, info); 1461 } 1462 1463 static int vtd_root_entry_rsvd_bits_check(IntelIOMMUState *s, 1464 VTDRootEntry *re) 1465 { 1466 /* Legacy Mode reserved bits check */ 1467 if (!s->root_scalable && 1468 (re->hi || (re->lo & VTD_ROOT_ENTRY_RSVD(s->aw_bits)))) 1469 goto rsvd_err; 1470 1471 /* Scalable Mode reserved bits check */ 1472 if (s->root_scalable && 1473 ((re->lo & VTD_ROOT_ENTRY_RSVD(s->aw_bits)) || 1474 (re->hi & VTD_ROOT_ENTRY_RSVD(s->aw_bits)))) 1475 goto rsvd_err; 1476 1477 return 0; 1478 1479 rsvd_err: 1480 error_report_once("%s: invalid root entry: hi=0x%"PRIx64 1481 ", lo=0x%"PRIx64, 1482 __func__, re->hi, re->lo); 1483 return -VTD_FR_ROOT_ENTRY_RSVD; 1484 } 1485 1486 static inline int vtd_context_entry_rsvd_bits_check(IntelIOMMUState *s, 1487 VTDContextEntry *ce) 1488 { 1489 if (!s->root_scalable && 1490 (ce->hi & VTD_CONTEXT_ENTRY_RSVD_HI || 1491 ce->lo & VTD_CONTEXT_ENTRY_RSVD_LO(s->aw_bits))) { 1492 error_report_once("%s: invalid context entry: hi=%"PRIx64 1493 ", lo=%"PRIx64" (reserved nonzero)", 1494 __func__, ce->hi, ce->lo); 1495 return -VTD_FR_CONTEXT_ENTRY_RSVD; 1496 } 1497 1498 if (s->root_scalable && 1499 (ce->val[0] & VTD_SM_CONTEXT_ENTRY_RSVD_VAL0(s->aw_bits) || 1500 ce->val[1] & VTD_SM_CONTEXT_ENTRY_RSVD_VAL1 || 1501 ce->val[2] || 1502 ce->val[3])) { 1503 error_report_once("%s: invalid context entry: val[3]=%"PRIx64 1504 ", val[2]=%"PRIx64 1505 ", val[1]=%"PRIx64 1506 ", val[0]=%"PRIx64" (reserved nonzero)", 1507 __func__, ce->val[3], ce->val[2], 1508 ce->val[1], ce->val[0]); 1509 return -VTD_FR_CONTEXT_ENTRY_RSVD; 1510 } 1511 1512 return 0; 1513 } 1514 1515 static int vtd_ce_rid2pasid_check(IntelIOMMUState *s, 1516 VTDContextEntry *ce) 1517 { 1518 VTDPASIDEntry pe; 1519 1520 /* 1521 * Make sure in Scalable Mode, a present context entry 1522 * has valid rid2pasid setting, which includes valid 1523 * rid2pasid field and corresponding pasid entry setting 1524 */ 1525 return vtd_ce_get_rid2pasid_entry(s, ce, &pe, PCI_NO_PASID); 1526 } 1527 1528 /* Map a device to its corresponding domain (context-entry) */ 1529 static int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num, 1530 uint8_t devfn, VTDContextEntry *ce) 1531 { 1532 VTDRootEntry re; 1533 int ret_fr; 1534 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 1535 1536 ret_fr = vtd_get_root_entry(s, bus_num, &re); 1537 if (ret_fr) { 1538 return ret_fr; 1539 } 1540 1541 if (!vtd_root_entry_present(s, &re, devfn)) { 1542 /* Not error - it's okay we don't have root entry. */ 1543 trace_vtd_re_not_present(bus_num); 1544 return -VTD_FR_ROOT_ENTRY_P; 1545 } 1546 1547 ret_fr = vtd_root_entry_rsvd_bits_check(s, &re); 1548 if (ret_fr) { 1549 return ret_fr; 1550 } 1551 1552 ret_fr = vtd_get_context_entry_from_root(s, &re, devfn, ce); 1553 if (ret_fr) { 1554 return ret_fr; 1555 } 1556 1557 if (!vtd_ce_present(ce)) { 1558 /* Not error - it's okay we don't have context entry. */ 1559 trace_vtd_ce_not_present(bus_num, devfn); 1560 return -VTD_FR_CONTEXT_ENTRY_P; 1561 } 1562 1563 ret_fr = vtd_context_entry_rsvd_bits_check(s, ce); 1564 if (ret_fr) { 1565 return ret_fr; 1566 } 1567 1568 /* Check if the programming of context-entry is valid */ 1569 if (!s->root_scalable && 1570 !vtd_is_sl_level_supported(s, vtd_ce_get_level(ce))) { 1571 error_report_once("%s: invalid context entry: hi=%"PRIx64 1572 ", lo=%"PRIx64" (level %d not supported)", 1573 __func__, ce->hi, ce->lo, 1574 vtd_ce_get_level(ce)); 1575 return -VTD_FR_CONTEXT_ENTRY_INV; 1576 } 1577 1578 if (!s->root_scalable) { 1579 /* Do translation type check */ 1580 if (!vtd_ce_type_check(x86_iommu, ce)) { 1581 /* Errors dumped in vtd_ce_type_check() */ 1582 return -VTD_FR_CONTEXT_ENTRY_INV; 1583 } 1584 } else { 1585 /* 1586 * Check if the programming of context-entry.rid2pasid 1587 * and corresponding pasid setting is valid, and thus 1588 * avoids to check pasid entry fetching result in future 1589 * helper function calling. 1590 */ 1591 ret_fr = vtd_ce_rid2pasid_check(s, ce); 1592 if (ret_fr) { 1593 return ret_fr; 1594 } 1595 } 1596 1597 return 0; 1598 } 1599 1600 static int vtd_sync_shadow_page_hook(const IOMMUTLBEvent *event, 1601 void *private) 1602 { 1603 memory_region_notify_iommu(private, 0, *event); 1604 return 0; 1605 } 1606 1607 static uint16_t vtd_get_domain_id(IntelIOMMUState *s, 1608 VTDContextEntry *ce, 1609 uint32_t pasid) 1610 { 1611 VTDPASIDEntry pe; 1612 1613 if (s->root_scalable) { 1614 vtd_ce_get_rid2pasid_entry(s, ce, &pe, pasid); 1615 return VTD_SM_PASID_ENTRY_DID(pe.val[1]); 1616 } 1617 1618 return VTD_CONTEXT_ENTRY_DID(ce->hi); 1619 } 1620 1621 static int vtd_sync_shadow_page_table_range(VTDAddressSpace *vtd_as, 1622 VTDContextEntry *ce, 1623 hwaddr addr, hwaddr size) 1624 { 1625 IntelIOMMUState *s = vtd_as->iommu_state; 1626 vtd_page_walk_info info = { 1627 .hook_fn = vtd_sync_shadow_page_hook, 1628 .private = (void *)&vtd_as->iommu, 1629 .notify_unmap = true, 1630 .aw = s->aw_bits, 1631 .as = vtd_as, 1632 .domain_id = vtd_get_domain_id(s, ce, vtd_as->pasid), 1633 }; 1634 1635 return vtd_page_walk(s, ce, addr, addr + size, &info, vtd_as->pasid); 1636 } 1637 1638 static int vtd_address_space_sync(VTDAddressSpace *vtd_as) 1639 { 1640 int ret; 1641 VTDContextEntry ce; 1642 IOMMUNotifier *n; 1643 1644 /* If no MAP notifier registered, we simply invalidate all the cache */ 1645 if (!vtd_as_has_map_notifier(vtd_as)) { 1646 IOMMU_NOTIFIER_FOREACH(n, &vtd_as->iommu) { 1647 memory_region_unmap_iommu_notifier_range(n); 1648 } 1649 return 0; 1650 } 1651 1652 ret = vtd_dev_to_context_entry(vtd_as->iommu_state, 1653 pci_bus_num(vtd_as->bus), 1654 vtd_as->devfn, &ce); 1655 if (ret) { 1656 if (ret == -VTD_FR_CONTEXT_ENTRY_P) { 1657 /* 1658 * It's a valid scenario to have a context entry that is 1659 * not present. For example, when a device is removed 1660 * from an existing domain then the context entry will be 1661 * zeroed by the guest before it was put into another 1662 * domain. When this happens, instead of synchronizing 1663 * the shadow pages we should invalidate all existing 1664 * mappings and notify the backends. 1665 */ 1666 IOMMU_NOTIFIER_FOREACH(n, &vtd_as->iommu) { 1667 vtd_address_space_unmap(vtd_as, n); 1668 } 1669 ret = 0; 1670 } 1671 return ret; 1672 } 1673 1674 return vtd_sync_shadow_page_table_range(vtd_as, &ce, 0, UINT64_MAX); 1675 } 1676 1677 /* 1678 * Check if specific device is configured to bypass address 1679 * translation for DMA requests. In Scalable Mode, bypass 1680 * 1st-level translation or 2nd-level translation, it depends 1681 * on PGTT setting. 1682 */ 1683 static bool vtd_dev_pt_enabled(IntelIOMMUState *s, VTDContextEntry *ce, 1684 uint32_t pasid) 1685 { 1686 VTDPASIDEntry pe; 1687 int ret; 1688 1689 if (s->root_scalable) { 1690 ret = vtd_ce_get_rid2pasid_entry(s, ce, &pe, pasid); 1691 if (ret) { 1692 /* 1693 * This error is guest triggerable. We should assumt PT 1694 * not enabled for safety. 1695 */ 1696 return false; 1697 } 1698 return (VTD_PE_GET_TYPE(&pe) == VTD_SM_PASID_ENTRY_PT); 1699 } 1700 1701 return (vtd_ce_get_type(ce) == VTD_CONTEXT_TT_PASS_THROUGH); 1702 1703 } 1704 1705 static bool vtd_as_pt_enabled(VTDAddressSpace *as) 1706 { 1707 IntelIOMMUState *s; 1708 VTDContextEntry ce; 1709 1710 assert(as); 1711 1712 s = as->iommu_state; 1713 if (vtd_dev_to_context_entry(s, pci_bus_num(as->bus), as->devfn, 1714 &ce)) { 1715 /* 1716 * Possibly failed to parse the context entry for some reason 1717 * (e.g., during init, or any guest configuration errors on 1718 * context entries). We should assume PT not enabled for 1719 * safety. 1720 */ 1721 return false; 1722 } 1723 1724 return vtd_dev_pt_enabled(s, &ce, as->pasid); 1725 } 1726 1727 /* Return whether the device is using IOMMU translation. */ 1728 static bool vtd_switch_address_space(VTDAddressSpace *as) 1729 { 1730 bool use_iommu, pt; 1731 /* Whether we need to take the BQL on our own */ 1732 bool take_bql = !bql_locked(); 1733 1734 assert(as); 1735 1736 use_iommu = as->iommu_state->dmar_enabled && !vtd_as_pt_enabled(as); 1737 pt = as->iommu_state->dmar_enabled && vtd_as_pt_enabled(as); 1738 1739 trace_vtd_switch_address_space(pci_bus_num(as->bus), 1740 VTD_PCI_SLOT(as->devfn), 1741 VTD_PCI_FUNC(as->devfn), 1742 use_iommu); 1743 1744 /* 1745 * It's possible that we reach here without BQL, e.g., when called 1746 * from vtd_pt_enable_fast_path(). However the memory APIs need 1747 * it. We'd better make sure we have had it already, or, take it. 1748 */ 1749 if (take_bql) { 1750 bql_lock(); 1751 } 1752 1753 /* Turn off first then on the other */ 1754 if (use_iommu) { 1755 memory_region_set_enabled(&as->nodmar, false); 1756 memory_region_set_enabled(MEMORY_REGION(&as->iommu), true); 1757 /* 1758 * vt-d spec v3.4 3.14: 1759 * 1760 * """ 1761 * Requests-with-PASID with input address in range 0xFEEx_xxxx 1762 * are translated normally like any other request-with-PASID 1763 * through DMA-remapping hardware. 1764 * """ 1765 * 1766 * Need to disable ir for as with PASID. 1767 */ 1768 if (as->pasid != PCI_NO_PASID) { 1769 memory_region_set_enabled(&as->iommu_ir, false); 1770 } else { 1771 memory_region_set_enabled(&as->iommu_ir, true); 1772 } 1773 } else { 1774 memory_region_set_enabled(MEMORY_REGION(&as->iommu), false); 1775 memory_region_set_enabled(&as->nodmar, true); 1776 } 1777 1778 /* 1779 * vtd-spec v3.4 3.14: 1780 * 1781 * """ 1782 * Requests-with-PASID with input address in range 0xFEEx_xxxx are 1783 * translated normally like any other request-with-PASID through 1784 * DMA-remapping hardware. However, if such a request is processed 1785 * using pass-through translation, it will be blocked as described 1786 * in the paragraph below. 1787 * 1788 * Software must not program paging-structure entries to remap any 1789 * address to the interrupt address range. Untranslated requests 1790 * and translation requests that result in an address in the 1791 * interrupt range will be blocked with condition code LGN.4 or 1792 * SGN.8. 1793 * """ 1794 * 1795 * We enable per as memory region (iommu_ir_fault) for catching 1796 * the translation for interrupt range through PASID + PT. 1797 */ 1798 if (pt && as->pasid != PCI_NO_PASID) { 1799 memory_region_set_enabled(&as->iommu_ir_fault, true); 1800 } else { 1801 memory_region_set_enabled(&as->iommu_ir_fault, false); 1802 } 1803 1804 if (take_bql) { 1805 bql_unlock(); 1806 } 1807 1808 return use_iommu; 1809 } 1810 1811 static void vtd_switch_address_space_all(IntelIOMMUState *s) 1812 { 1813 VTDAddressSpace *vtd_as; 1814 GHashTableIter iter; 1815 1816 g_hash_table_iter_init(&iter, s->vtd_address_spaces); 1817 while (g_hash_table_iter_next(&iter, NULL, (void **)&vtd_as)) { 1818 vtd_switch_address_space(vtd_as); 1819 } 1820 } 1821 1822 static const bool vtd_qualified_faults[] = { 1823 [VTD_FR_RESERVED] = false, 1824 [VTD_FR_ROOT_ENTRY_P] = false, 1825 [VTD_FR_CONTEXT_ENTRY_P] = true, 1826 [VTD_FR_CONTEXT_ENTRY_INV] = true, 1827 [VTD_FR_ADDR_BEYOND_MGAW] = true, 1828 [VTD_FR_WRITE] = true, 1829 [VTD_FR_READ] = true, 1830 [VTD_FR_PAGING_ENTRY_INV] = true, 1831 [VTD_FR_ROOT_TABLE_INV] = false, 1832 [VTD_FR_CONTEXT_TABLE_INV] = false, 1833 [VTD_FR_INTERRUPT_ADDR] = true, 1834 [VTD_FR_ROOT_ENTRY_RSVD] = false, 1835 [VTD_FR_PAGING_ENTRY_RSVD] = true, 1836 [VTD_FR_CONTEXT_ENTRY_TT] = true, 1837 [VTD_FR_PASID_DIR_ACCESS_ERR] = false, 1838 [VTD_FR_PASID_DIR_ENTRY_P] = true, 1839 [VTD_FR_PASID_TABLE_ACCESS_ERR] = false, 1840 [VTD_FR_PASID_ENTRY_P] = true, 1841 [VTD_FR_PASID_TABLE_ENTRY_INV] = true, 1842 [VTD_FR_FS_PAGING_ENTRY_INV] = true, 1843 [VTD_FR_FS_PAGING_ENTRY_P] = true, 1844 [VTD_FR_FS_PAGING_ENTRY_RSVD] = true, 1845 [VTD_FR_PASID_ENTRY_FSPTPTR_INV] = true, 1846 [VTD_FR_FS_NON_CANONICAL] = true, 1847 [VTD_FR_FS_PAGING_ENTRY_US] = true, 1848 [VTD_FR_SM_WRITE] = true, 1849 [VTD_FR_SM_INTERRUPT_ADDR] = true, 1850 [VTD_FR_FS_BIT_UPDATE_FAILED] = true, 1851 [VTD_FR_MAX] = false, 1852 }; 1853 1854 /* To see if a fault condition is "qualified", which is reported to software 1855 * only if the FPD field in the context-entry used to process the faulting 1856 * request is 0. 1857 */ 1858 static inline bool vtd_is_qualified_fault(VTDFaultReason fault) 1859 { 1860 return vtd_qualified_faults[fault]; 1861 } 1862 1863 static inline bool vtd_is_interrupt_addr(hwaddr addr) 1864 { 1865 return VTD_INTERRUPT_ADDR_FIRST <= addr && addr <= VTD_INTERRUPT_ADDR_LAST; 1866 } 1867 1868 static gboolean vtd_find_as_by_sid_and_pasid(gpointer key, gpointer value, 1869 gpointer user_data) 1870 { 1871 struct vtd_as_key *as_key = (struct vtd_as_key *)key; 1872 struct vtd_as_raw_key *target = (struct vtd_as_raw_key *)user_data; 1873 uint16_t sid = PCI_BUILD_BDF(pci_bus_num(as_key->bus), as_key->devfn); 1874 1875 return (as_key->pasid == target->pasid) && (sid == target->sid); 1876 } 1877 1878 static VTDAddressSpace *vtd_get_as_by_sid_and_pasid(IntelIOMMUState *s, 1879 uint16_t sid, 1880 uint32_t pasid) 1881 { 1882 struct vtd_as_raw_key key = { 1883 .sid = sid, 1884 .pasid = pasid 1885 }; 1886 1887 return g_hash_table_find(s->vtd_address_spaces, 1888 vtd_find_as_by_sid_and_pasid, &key); 1889 } 1890 1891 static VTDAddressSpace *vtd_get_as_by_sid(IntelIOMMUState *s, uint16_t sid) 1892 { 1893 return vtd_get_as_by_sid_and_pasid(s, sid, PCI_NO_PASID); 1894 } 1895 1896 static void vtd_pt_enable_fast_path(IntelIOMMUState *s, uint16_t source_id) 1897 { 1898 VTDAddressSpace *vtd_as; 1899 bool success = false; 1900 1901 vtd_as = vtd_get_as_by_sid(s, source_id); 1902 if (!vtd_as) { 1903 goto out; 1904 } 1905 1906 if (vtd_switch_address_space(vtd_as) == false) { 1907 /* We switched off IOMMU region successfully. */ 1908 success = true; 1909 } 1910 1911 out: 1912 trace_vtd_pt_enable_fast_path(source_id, success); 1913 } 1914 1915 /* 1916 * Rsvd field masks for fpte: 1917 * vtd_fpte_rsvd 4k pages 1918 * vtd_fpte_rsvd_large large pages 1919 * 1920 * We support only 4-level page tables. 1921 */ 1922 #define VTD_FPTE_RSVD_LEN 5 1923 static uint64_t vtd_fpte_rsvd[VTD_FPTE_RSVD_LEN]; 1924 static uint64_t vtd_fpte_rsvd_large[VTD_FPTE_RSVD_LEN]; 1925 1926 static bool vtd_flpte_nonzero_rsvd(uint64_t flpte, uint32_t level) 1927 { 1928 uint64_t rsvd_mask; 1929 1930 /* 1931 * We should have caught a guest-mis-programmed level earlier, 1932 * via vtd_is_fl_level_supported. 1933 */ 1934 assert(level < VTD_FPTE_RSVD_LEN); 1935 /* 1936 * Zero level doesn't exist. The smallest level is VTD_PT_LEVEL=1 and 1937 * checked by vtd_is_last_pte(). 1938 */ 1939 assert(level); 1940 1941 if ((level == VTD_PD_LEVEL || level == VTD_PDP_LEVEL) && 1942 (flpte & VTD_PT_PAGE_SIZE_MASK)) { 1943 /* large page */ 1944 rsvd_mask = vtd_fpte_rsvd_large[level]; 1945 } else { 1946 rsvd_mask = vtd_fpte_rsvd[level]; 1947 } 1948 1949 return flpte & rsvd_mask; 1950 } 1951 1952 static inline bool vtd_flpte_present(uint64_t flpte) 1953 { 1954 return !!(flpte & VTD_FL_P); 1955 } 1956 1957 /* Return true if IOVA is canonical, otherwise false. */ 1958 static bool vtd_iova_fl_check_canonical(IntelIOMMUState *s, uint64_t iova, 1959 VTDContextEntry *ce, uint32_t pasid) 1960 { 1961 uint64_t iova_limit = vtd_iova_limit(s, ce, s->aw_bits, pasid); 1962 uint64_t upper_bits_mask = ~(iova_limit - 1); 1963 uint64_t upper_bits = iova & upper_bits_mask; 1964 bool msb = ((iova & (iova_limit >> 1)) != 0); 1965 1966 if (msb) { 1967 return upper_bits == upper_bits_mask; 1968 } else { 1969 return !upper_bits; 1970 } 1971 } 1972 1973 static MemTxResult vtd_set_flag_in_pte(dma_addr_t base_addr, uint32_t index, 1974 uint64_t pte, uint64_t flag) 1975 { 1976 if (pte & flag) { 1977 return MEMTX_OK; 1978 } 1979 pte |= flag; 1980 pte = cpu_to_le64(pte); 1981 return dma_memory_write(&address_space_memory, 1982 base_addr + index * sizeof(pte), 1983 &pte, sizeof(pte), 1984 MEMTXATTRS_UNSPECIFIED); 1985 } 1986 1987 /* 1988 * Given the @iova, get relevant @flptep. @flpte_level will be the last level 1989 * of the translation, can be used for deciding the size of large page. 1990 */ 1991 static int vtd_iova_to_flpte(IntelIOMMUState *s, VTDContextEntry *ce, 1992 uint64_t iova, bool is_write, 1993 uint64_t *flptep, uint32_t *flpte_level, 1994 bool *reads, bool *writes, uint8_t aw_bits, 1995 uint32_t pasid) 1996 { 1997 dma_addr_t addr = vtd_get_iova_pgtbl_base(s, ce, pasid); 1998 uint32_t level = vtd_get_iova_level(s, ce, pasid); 1999 uint32_t offset; 2000 uint64_t flpte, flag_ad = VTD_FL_A; 2001 2002 if (!vtd_iova_fl_check_canonical(s, iova, ce, pasid)) { 2003 error_report_once("%s: detected non canonical IOVA (iova=0x%" PRIx64 "," 2004 "pasid=0x%" PRIx32 ")", __func__, iova, pasid); 2005 return -VTD_FR_FS_NON_CANONICAL; 2006 } 2007 2008 while (true) { 2009 offset = vtd_iova_level_offset(iova, level); 2010 flpte = vtd_get_pte(addr, offset); 2011 2012 if (flpte == (uint64_t)-1) { 2013 if (level == vtd_get_iova_level(s, ce, pasid)) { 2014 /* Invalid programming of pasid-entry */ 2015 return -VTD_FR_PASID_ENTRY_FSPTPTR_INV; 2016 } else { 2017 return -VTD_FR_FS_PAGING_ENTRY_INV; 2018 } 2019 } 2020 2021 if (!vtd_flpte_present(flpte)) { 2022 *reads = false; 2023 *writes = false; 2024 return -VTD_FR_FS_PAGING_ENTRY_P; 2025 } 2026 2027 /* No emulated device supports supervisor privilege request yet */ 2028 if (!(flpte & VTD_FL_US)) { 2029 *reads = false; 2030 *writes = false; 2031 return -VTD_FR_FS_PAGING_ENTRY_US; 2032 } 2033 2034 *reads = true; 2035 *writes = (*writes) && (flpte & VTD_FL_RW); 2036 if (is_write && !(flpte & VTD_FL_RW)) { 2037 return -VTD_FR_SM_WRITE; 2038 } 2039 if (vtd_flpte_nonzero_rsvd(flpte, level)) { 2040 error_report_once("%s: detected flpte reserved non-zero " 2041 "iova=0x%" PRIx64 ", level=0x%" PRIx32 2042 "flpte=0x%" PRIx64 ", pasid=0x%" PRIX32 ")", 2043 __func__, iova, level, flpte, pasid); 2044 return -VTD_FR_FS_PAGING_ENTRY_RSVD; 2045 } 2046 2047 if (vtd_is_last_pte(flpte, level) && is_write) { 2048 flag_ad |= VTD_FL_D; 2049 } 2050 2051 if (vtd_set_flag_in_pte(addr, offset, flpte, flag_ad) != MEMTX_OK) { 2052 return -VTD_FR_FS_BIT_UPDATE_FAILED; 2053 } 2054 2055 if (vtd_is_last_pte(flpte, level)) { 2056 *flptep = flpte; 2057 *flpte_level = level; 2058 return 0; 2059 } 2060 2061 addr = vtd_get_pte_addr(flpte, aw_bits); 2062 level--; 2063 } 2064 } 2065 2066 static void vtd_report_fault(IntelIOMMUState *s, 2067 int err, bool is_fpd_set, 2068 uint16_t source_id, 2069 hwaddr addr, 2070 bool is_write, 2071 bool is_pasid, 2072 uint32_t pasid) 2073 { 2074 if (is_fpd_set && vtd_is_qualified_fault(err)) { 2075 trace_vtd_fault_disabled(); 2076 } else { 2077 vtd_report_dmar_fault(s, source_id, addr, err, is_write, 2078 is_pasid, pasid); 2079 } 2080 } 2081 2082 /* Map dev to context-entry then do a paging-structures walk to do a iommu 2083 * translation. 2084 * 2085 * Called from RCU critical section. 2086 * 2087 * @bus_num: The bus number 2088 * @devfn: The devfn, which is the combined of device and function number 2089 * @is_write: The access is a write operation 2090 * @entry: IOMMUTLBEntry that contain the addr to be translated and result 2091 * 2092 * Returns true if translation is successful, otherwise false. 2093 */ 2094 static bool vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus, 2095 uint8_t devfn, hwaddr addr, bool is_write, 2096 IOMMUTLBEntry *entry) 2097 { 2098 IntelIOMMUState *s = vtd_as->iommu_state; 2099 VTDContextEntry ce; 2100 uint8_t bus_num = pci_bus_num(bus); 2101 VTDContextCacheEntry *cc_entry; 2102 uint64_t pte, page_mask; 2103 uint32_t level, pasid = vtd_as->pasid; 2104 uint16_t source_id = PCI_BUILD_BDF(bus_num, devfn); 2105 int ret_fr; 2106 bool is_fpd_set = false; 2107 bool reads = true; 2108 bool writes = true; 2109 uint8_t access_flags, pgtt; 2110 bool rid2pasid = (pasid == PCI_NO_PASID) && s->root_scalable; 2111 VTDIOTLBEntry *iotlb_entry; 2112 uint64_t xlat, size; 2113 2114 /* 2115 * We have standalone memory region for interrupt addresses, we 2116 * should never receive translation requests in this region. 2117 */ 2118 assert(!vtd_is_interrupt_addr(addr)); 2119 2120 vtd_iommu_lock(s); 2121 2122 cc_entry = &vtd_as->context_cache_entry; 2123 2124 /* Try to fetch pte from IOTLB, we don't need RID2PASID logic */ 2125 if (!rid2pasid) { 2126 iotlb_entry = vtd_lookup_iotlb(s, source_id, pasid, addr); 2127 if (iotlb_entry) { 2128 trace_vtd_iotlb_page_hit(source_id, addr, iotlb_entry->pte, 2129 iotlb_entry->domain_id); 2130 pte = iotlb_entry->pte; 2131 access_flags = iotlb_entry->access_flags; 2132 page_mask = iotlb_entry->mask; 2133 goto out; 2134 } 2135 } 2136 2137 /* Try to fetch context-entry from cache first */ 2138 if (cc_entry->context_cache_gen == s->context_cache_gen) { 2139 trace_vtd_iotlb_cc_hit(bus_num, devfn, cc_entry->context_entry.hi, 2140 cc_entry->context_entry.lo, 2141 cc_entry->context_cache_gen); 2142 ce = cc_entry->context_entry; 2143 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 2144 if (!is_fpd_set && s->root_scalable) { 2145 ret_fr = vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set, pasid); 2146 if (ret_fr) { 2147 vtd_report_fault(s, -ret_fr, is_fpd_set, 2148 source_id, addr, is_write, 2149 false, 0); 2150 goto error; 2151 } 2152 } 2153 } else { 2154 ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce); 2155 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 2156 if (!ret_fr && !is_fpd_set && s->root_scalable) { 2157 ret_fr = vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set, pasid); 2158 } 2159 if (ret_fr) { 2160 vtd_report_fault(s, -ret_fr, is_fpd_set, 2161 source_id, addr, is_write, 2162 false, 0); 2163 goto error; 2164 } 2165 /* Update context-cache */ 2166 trace_vtd_iotlb_cc_update(bus_num, devfn, ce.hi, ce.lo, 2167 cc_entry->context_cache_gen, 2168 s->context_cache_gen); 2169 cc_entry->context_entry = ce; 2170 cc_entry->context_cache_gen = s->context_cache_gen; 2171 } 2172 2173 if (rid2pasid) { 2174 pasid = VTD_CE_GET_RID2PASID(&ce); 2175 } 2176 2177 /* 2178 * We don't need to translate for pass-through context entries. 2179 * Also, let's ignore IOTLB caching as well for PT devices. 2180 */ 2181 if (vtd_dev_pt_enabled(s, &ce, pasid)) { 2182 entry->iova = addr & VTD_PAGE_MASK_4K; 2183 entry->translated_addr = entry->iova; 2184 entry->addr_mask = ~VTD_PAGE_MASK_4K; 2185 entry->perm = IOMMU_RW; 2186 trace_vtd_translate_pt(source_id, entry->iova); 2187 2188 /* 2189 * When this happens, it means firstly caching-mode is not 2190 * enabled, and this is the first passthrough translation for 2191 * the device. Let's enable the fast path for passthrough. 2192 * 2193 * When passthrough is disabled again for the device, we can 2194 * capture it via the context entry invalidation, then the 2195 * IOMMU region can be swapped back. 2196 */ 2197 vtd_pt_enable_fast_path(s, source_id); 2198 vtd_iommu_unlock(s); 2199 return true; 2200 } 2201 2202 /* Try to fetch pte from IOTLB for RID2PASID slow path */ 2203 if (rid2pasid) { 2204 iotlb_entry = vtd_lookup_iotlb(s, source_id, pasid, addr); 2205 if (iotlb_entry) { 2206 trace_vtd_iotlb_page_hit(source_id, addr, iotlb_entry->pte, 2207 iotlb_entry->domain_id); 2208 pte = iotlb_entry->pte; 2209 access_flags = iotlb_entry->access_flags; 2210 page_mask = iotlb_entry->mask; 2211 goto out; 2212 } 2213 } 2214 2215 if (s->flts && s->root_scalable) { 2216 ret_fr = vtd_iova_to_flpte(s, &ce, addr, is_write, &pte, &level, 2217 &reads, &writes, s->aw_bits, pasid); 2218 pgtt = VTD_SM_PASID_ENTRY_FLT; 2219 } else { 2220 ret_fr = vtd_iova_to_slpte(s, &ce, addr, is_write, &pte, &level, 2221 &reads, &writes, s->aw_bits, pasid); 2222 pgtt = VTD_SM_PASID_ENTRY_SLT; 2223 } 2224 if (!ret_fr) { 2225 xlat = vtd_get_pte_addr(pte, s->aw_bits); 2226 size = ~vtd_pt_level_page_mask(level) + 1; 2227 2228 /* 2229 * Per VT-d spec 4.1 section 3.15: Untranslated requests and translation 2230 * requests that result in an address in the interrupt range will be 2231 * blocked with condition code LGN.4 or SGN.8. 2232 */ 2233 if ((xlat <= VTD_INTERRUPT_ADDR_LAST && 2234 xlat + size - 1 >= VTD_INTERRUPT_ADDR_FIRST)) { 2235 error_report_once("%s: xlat address is in interrupt range " 2236 "(iova=0x%" PRIx64 ", level=0x%" PRIx32 ", " 2237 "pte=0x%" PRIx64 ", write=%d, " 2238 "xlat=0x%" PRIx64 ", size=0x%" PRIx64 ", " 2239 "pasid=0x%" PRIx32 ")", 2240 __func__, addr, level, pte, is_write, 2241 xlat, size, pasid); 2242 ret_fr = s->scalable_mode ? -VTD_FR_SM_INTERRUPT_ADDR : 2243 -VTD_FR_INTERRUPT_ADDR; 2244 } 2245 } 2246 2247 if (ret_fr) { 2248 vtd_report_fault(s, -ret_fr, is_fpd_set, source_id, 2249 addr, is_write, pasid != PCI_NO_PASID, pasid); 2250 goto error; 2251 } 2252 2253 page_mask = vtd_pt_level_page_mask(level); 2254 access_flags = IOMMU_ACCESS_FLAG(reads, writes); 2255 vtd_update_iotlb(s, source_id, vtd_get_domain_id(s, &ce, pasid), 2256 addr, pte, access_flags, level, pasid, pgtt); 2257 out: 2258 vtd_iommu_unlock(s); 2259 entry->iova = addr & page_mask; 2260 entry->translated_addr = vtd_get_pte_addr(pte, s->aw_bits) & page_mask; 2261 entry->addr_mask = ~page_mask; 2262 entry->perm = access_flags; 2263 return true; 2264 2265 error: 2266 vtd_iommu_unlock(s); 2267 entry->iova = 0; 2268 entry->translated_addr = 0; 2269 entry->addr_mask = 0; 2270 entry->perm = IOMMU_NONE; 2271 return false; 2272 } 2273 2274 static void vtd_root_table_setup(IntelIOMMUState *s) 2275 { 2276 s->root = vtd_get_quad_raw(s, DMAR_RTADDR_REG); 2277 s->root &= VTD_RTADDR_ADDR_MASK(s->aw_bits); 2278 2279 vtd_update_scalable_state(s); 2280 2281 trace_vtd_reg_dmar_root(s->root, s->root_scalable); 2282 } 2283 2284 static void vtd_iec_notify_all(IntelIOMMUState *s, bool global, 2285 uint32_t index, uint32_t mask) 2286 { 2287 x86_iommu_iec_notify_all(X86_IOMMU_DEVICE(s), global, index, mask); 2288 } 2289 2290 static void vtd_interrupt_remap_table_setup(IntelIOMMUState *s) 2291 { 2292 uint64_t value = 0; 2293 value = vtd_get_quad_raw(s, DMAR_IRTA_REG); 2294 s->intr_size = 1UL << ((value & VTD_IRTA_SIZE_MASK) + 1); 2295 s->intr_root = value & VTD_IRTA_ADDR_MASK(s->aw_bits); 2296 s->intr_eime = value & VTD_IRTA_EIME; 2297 2298 /* Notify global invalidation */ 2299 vtd_iec_notify_all(s, true, 0, 0); 2300 2301 trace_vtd_reg_ir_root(s->intr_root, s->intr_size); 2302 } 2303 2304 static void vtd_iommu_replay_all(IntelIOMMUState *s) 2305 { 2306 VTDAddressSpace *vtd_as; 2307 2308 QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) { 2309 vtd_address_space_sync(vtd_as); 2310 } 2311 } 2312 2313 static void vtd_context_global_invalidate(IntelIOMMUState *s) 2314 { 2315 trace_vtd_inv_desc_cc_global(); 2316 /* Protects context cache */ 2317 vtd_iommu_lock(s); 2318 s->context_cache_gen++; 2319 if (s->context_cache_gen == VTD_CONTEXT_CACHE_GEN_MAX) { 2320 vtd_reset_context_cache_locked(s); 2321 } 2322 vtd_iommu_unlock(s); 2323 vtd_address_space_refresh_all(s); 2324 /* 2325 * From VT-d spec 6.5.2.1, a global context entry invalidation 2326 * should be followed by a IOTLB global invalidation, so we should 2327 * be safe even without this. Hoewever, let's replay the region as 2328 * well to be safer, and go back here when we need finer tunes for 2329 * VT-d emulation codes. 2330 */ 2331 vtd_iommu_replay_all(s); 2332 } 2333 2334 /* Do a context-cache device-selective invalidation. 2335 * @func_mask: FM field after shifting 2336 */ 2337 static void vtd_context_device_invalidate(IntelIOMMUState *s, 2338 uint16_t source_id, 2339 uint16_t func_mask) 2340 { 2341 GHashTableIter as_it; 2342 uint16_t mask; 2343 VTDAddressSpace *vtd_as; 2344 uint8_t bus_n, devfn; 2345 2346 trace_vtd_inv_desc_cc_devices(source_id, func_mask); 2347 2348 switch (func_mask & 3) { 2349 case 0: 2350 mask = 0; /* No bits in the SID field masked */ 2351 break; 2352 case 1: 2353 mask = 4; /* Mask bit 2 in the SID field */ 2354 break; 2355 case 2: 2356 mask = 6; /* Mask bit 2:1 in the SID field */ 2357 break; 2358 case 3: 2359 mask = 7; /* Mask bit 2:0 in the SID field */ 2360 break; 2361 default: 2362 g_assert_not_reached(); 2363 } 2364 mask = ~mask; 2365 2366 bus_n = VTD_SID_TO_BUS(source_id); 2367 devfn = VTD_SID_TO_DEVFN(source_id); 2368 2369 g_hash_table_iter_init(&as_it, s->vtd_address_spaces); 2370 while (g_hash_table_iter_next(&as_it, NULL, (void **)&vtd_as)) { 2371 if ((pci_bus_num(vtd_as->bus) == bus_n) && 2372 (vtd_as->devfn & mask) == (devfn & mask)) { 2373 trace_vtd_inv_desc_cc_device(bus_n, VTD_PCI_SLOT(vtd_as->devfn), 2374 VTD_PCI_FUNC(vtd_as->devfn)); 2375 vtd_iommu_lock(s); 2376 vtd_as->context_cache_entry.context_cache_gen = 0; 2377 vtd_iommu_unlock(s); 2378 /* 2379 * Do switch address space when needed, in case if the 2380 * device passthrough bit is switched. 2381 */ 2382 vtd_switch_address_space(vtd_as); 2383 /* 2384 * So a device is moving out of (or moving into) a 2385 * domain, resync the shadow page table. 2386 * This won't bring bad even if we have no such 2387 * notifier registered - the IOMMU notification 2388 * framework will skip MAP notifications if that 2389 * happened. 2390 */ 2391 vtd_address_space_sync(vtd_as); 2392 } 2393 } 2394 } 2395 2396 /* Context-cache invalidation 2397 * Returns the Context Actual Invalidation Granularity. 2398 * @val: the content of the CCMD_REG 2399 */ 2400 static uint64_t vtd_context_cache_invalidate(IntelIOMMUState *s, uint64_t val) 2401 { 2402 uint64_t caig; 2403 uint64_t type = val & VTD_CCMD_CIRG_MASK; 2404 2405 switch (type) { 2406 case VTD_CCMD_DOMAIN_INVL: 2407 /* Fall through */ 2408 case VTD_CCMD_GLOBAL_INVL: 2409 caig = VTD_CCMD_GLOBAL_INVL_A; 2410 vtd_context_global_invalidate(s); 2411 break; 2412 2413 case VTD_CCMD_DEVICE_INVL: 2414 caig = VTD_CCMD_DEVICE_INVL_A; 2415 vtd_context_device_invalidate(s, VTD_CCMD_SID(val), VTD_CCMD_FM(val)); 2416 break; 2417 2418 default: 2419 error_report_once("%s: invalid context: 0x%" PRIx64, 2420 __func__, val); 2421 caig = 0; 2422 } 2423 return caig; 2424 } 2425 2426 static void vtd_iotlb_global_invalidate(IntelIOMMUState *s) 2427 { 2428 trace_vtd_inv_desc_iotlb_global(); 2429 vtd_reset_iotlb(s); 2430 vtd_iommu_replay_all(s); 2431 } 2432 2433 static void vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id) 2434 { 2435 VTDContextEntry ce; 2436 VTDAddressSpace *vtd_as; 2437 2438 trace_vtd_inv_desc_iotlb_domain(domain_id); 2439 2440 vtd_iommu_lock(s); 2441 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_domain, 2442 &domain_id); 2443 vtd_iommu_unlock(s); 2444 2445 QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) { 2446 if (!vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus), 2447 vtd_as->devfn, &ce) && 2448 domain_id == vtd_get_domain_id(s, &ce, vtd_as->pasid)) { 2449 vtd_address_space_sync(vtd_as); 2450 } 2451 } 2452 } 2453 2454 /* 2455 * There is no pasid field in iotlb invalidation descriptor, so PCI_NO_PASID 2456 * is passed as parameter. Piotlb invalidation supports pasid, pasid in its 2457 * descriptor is passed which should not be PCI_NO_PASID. 2458 */ 2459 static void vtd_iotlb_page_invalidate_notify(IntelIOMMUState *s, 2460 uint16_t domain_id, hwaddr addr, 2461 uint8_t am, uint32_t pasid) 2462 { 2463 VTDAddressSpace *vtd_as; 2464 VTDContextEntry ce; 2465 int ret; 2466 hwaddr size = (1 << am) * VTD_PAGE_SIZE; 2467 2468 QLIST_FOREACH(vtd_as, &(s->vtd_as_with_notifiers), next) { 2469 ret = vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus), 2470 vtd_as->devfn, &ce); 2471 if (!ret && domain_id == vtd_get_domain_id(s, &ce, vtd_as->pasid)) { 2472 uint32_t rid2pasid = PCI_NO_PASID; 2473 2474 if (s->root_scalable) { 2475 rid2pasid = VTD_CE_GET_RID2PASID(&ce); 2476 } 2477 2478 /* 2479 * In legacy mode, vtd_as->pasid == pasid is always true. 2480 * In scalable mode, for vtd address space backing a PCI 2481 * device without pasid, needs to compare pasid with 2482 * rid2pasid of this device. 2483 */ 2484 if (!(vtd_as->pasid == pasid || 2485 (vtd_as->pasid == PCI_NO_PASID && pasid == rid2pasid))) { 2486 continue; 2487 } 2488 2489 if (vtd_as_has_map_notifier(vtd_as)) { 2490 /* 2491 * When stage-1 translation is off, as long as we have MAP 2492 * notifications registered in any of our IOMMU notifiers, 2493 * we need to sync the shadow page table. Otherwise VFIO 2494 * device attaches to nested page table instead of shadow 2495 * page table, so no need to sync. 2496 */ 2497 if (!s->flts || !s->root_scalable) { 2498 vtd_sync_shadow_page_table_range(vtd_as, &ce, addr, size); 2499 } 2500 } else { 2501 /* 2502 * For UNMAP-only notifiers, we don't need to walk the 2503 * page tables. We just deliver the PSI down to 2504 * invalidate caches. 2505 */ 2506 const IOMMUTLBEvent event = { 2507 .type = IOMMU_NOTIFIER_UNMAP, 2508 .entry = { 2509 .target_as = &address_space_memory, 2510 .iova = addr, 2511 .translated_addr = 0, 2512 .addr_mask = size - 1, 2513 .perm = IOMMU_NONE, 2514 }, 2515 }; 2516 memory_region_notify_iommu(&vtd_as->iommu, 0, event); 2517 } 2518 } 2519 } 2520 } 2521 2522 static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id, 2523 hwaddr addr, uint8_t am) 2524 { 2525 VTDIOTLBPageInvInfo info; 2526 2527 trace_vtd_inv_desc_iotlb_pages(domain_id, addr, am); 2528 2529 assert(am <= VTD_MAMV); 2530 info.domain_id = domain_id; 2531 info.addr = addr; 2532 info.mask = ~((1 << am) - 1); 2533 vtd_iommu_lock(s); 2534 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info); 2535 vtd_iommu_unlock(s); 2536 vtd_iotlb_page_invalidate_notify(s, domain_id, addr, am, PCI_NO_PASID); 2537 } 2538 2539 /* Flush IOTLB 2540 * Returns the IOTLB Actual Invalidation Granularity. 2541 * @val: the content of the IOTLB_REG 2542 */ 2543 static uint64_t vtd_iotlb_flush(IntelIOMMUState *s, uint64_t val) 2544 { 2545 uint64_t iaig; 2546 uint64_t type = val & VTD_TLB_FLUSH_GRANU_MASK; 2547 uint16_t domain_id; 2548 hwaddr addr; 2549 uint8_t am; 2550 2551 switch (type) { 2552 case VTD_TLB_GLOBAL_FLUSH: 2553 iaig = VTD_TLB_GLOBAL_FLUSH_A; 2554 vtd_iotlb_global_invalidate(s); 2555 break; 2556 2557 case VTD_TLB_DSI_FLUSH: 2558 domain_id = VTD_TLB_DID(val); 2559 iaig = VTD_TLB_DSI_FLUSH_A; 2560 vtd_iotlb_domain_invalidate(s, domain_id); 2561 break; 2562 2563 case VTD_TLB_PSI_FLUSH: 2564 domain_id = VTD_TLB_DID(val); 2565 addr = vtd_get_quad_raw(s, DMAR_IVA_REG); 2566 am = VTD_IVA_AM(addr); 2567 addr = VTD_IVA_ADDR(addr); 2568 if (am > VTD_MAMV) { 2569 error_report_once("%s: address mask overflow: 0x%" PRIx64, 2570 __func__, vtd_get_quad_raw(s, DMAR_IVA_REG)); 2571 iaig = 0; 2572 break; 2573 } 2574 iaig = VTD_TLB_PSI_FLUSH_A; 2575 vtd_iotlb_page_invalidate(s, domain_id, addr, am); 2576 break; 2577 2578 default: 2579 error_report_once("%s: invalid granularity: 0x%" PRIx64, 2580 __func__, val); 2581 iaig = 0; 2582 } 2583 return iaig; 2584 } 2585 2586 static void vtd_fetch_inv_desc(IntelIOMMUState *s); 2587 2588 static inline bool vtd_queued_inv_disable_check(IntelIOMMUState *s) 2589 { 2590 return s->qi_enabled && (s->iq_tail == s->iq_head) && 2591 (s->iq_last_desc_type == VTD_INV_DESC_WAIT); 2592 } 2593 2594 static void vtd_handle_gcmd_qie(IntelIOMMUState *s, bool en) 2595 { 2596 uint64_t iqa_val = vtd_get_quad_raw(s, DMAR_IQA_REG); 2597 2598 trace_vtd_inv_qi_enable(en); 2599 2600 if (en) { 2601 s->iq = iqa_val & VTD_IQA_IQA_MASK(s->aw_bits); 2602 /* 2^(x+8) entries */ 2603 s->iq_size = 1UL << ((iqa_val & VTD_IQA_QS) + 8 - (s->iq_dw ? 1 : 0)); 2604 s->qi_enabled = true; 2605 trace_vtd_inv_qi_setup(s->iq, s->iq_size); 2606 /* Ok - report back to driver */ 2607 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_QIES); 2608 2609 if (s->iq_tail != 0) { 2610 /* 2611 * This is a spec violation but Windows guests are known to set up 2612 * Queued Invalidation this way so we allow the write and process 2613 * Invalidation Descriptors right away. 2614 */ 2615 trace_vtd_warn_invalid_qi_tail(s->iq_tail); 2616 if (!(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) { 2617 vtd_fetch_inv_desc(s); 2618 } 2619 } 2620 } else { 2621 if (vtd_queued_inv_disable_check(s)) { 2622 /* disable Queued Invalidation */ 2623 vtd_set_quad_raw(s, DMAR_IQH_REG, 0); 2624 s->iq_head = 0; 2625 s->qi_enabled = false; 2626 /* Ok - report back to driver */ 2627 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_QIES, 0); 2628 } else { 2629 error_report_once("%s: detected improper state when disable QI " 2630 "(head=0x%x, tail=0x%x, last_type=%d)", 2631 __func__, 2632 s->iq_head, s->iq_tail, s->iq_last_desc_type); 2633 } 2634 } 2635 } 2636 2637 /* Set Root Table Pointer */ 2638 static void vtd_handle_gcmd_srtp(IntelIOMMUState *s) 2639 { 2640 vtd_root_table_setup(s); 2641 /* Ok - report back to driver */ 2642 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_RTPS); 2643 vtd_reset_caches(s); 2644 vtd_address_space_refresh_all(s); 2645 } 2646 2647 /* Set Interrupt Remap Table Pointer */ 2648 static void vtd_handle_gcmd_sirtp(IntelIOMMUState *s) 2649 { 2650 vtd_interrupt_remap_table_setup(s); 2651 /* Ok - report back to driver */ 2652 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRTPS); 2653 } 2654 2655 /* Handle Translation Enable/Disable */ 2656 static void vtd_handle_gcmd_te(IntelIOMMUState *s, bool en) 2657 { 2658 if (s->dmar_enabled == en) { 2659 return; 2660 } 2661 2662 trace_vtd_dmar_enable(en); 2663 2664 if (en) { 2665 s->dmar_enabled = true; 2666 /* Ok - report back to driver */ 2667 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_TES); 2668 } else { 2669 s->dmar_enabled = false; 2670 2671 /* Clear the index of Fault Recording Register */ 2672 s->next_frcd_reg = 0; 2673 /* Ok - report back to driver */ 2674 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_TES, 0); 2675 } 2676 2677 vtd_reset_caches(s); 2678 vtd_address_space_refresh_all(s); 2679 } 2680 2681 /* Handle Interrupt Remap Enable/Disable */ 2682 static void vtd_handle_gcmd_ire(IntelIOMMUState *s, bool en) 2683 { 2684 trace_vtd_ir_enable(en); 2685 2686 if (en) { 2687 s->intr_enabled = true; 2688 /* Ok - report back to driver */ 2689 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRES); 2690 } else { 2691 s->intr_enabled = false; 2692 /* Ok - report back to driver */ 2693 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_IRES, 0); 2694 } 2695 } 2696 2697 /* Handle write to Global Command Register */ 2698 static void vtd_handle_gcmd_write(IntelIOMMUState *s) 2699 { 2700 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 2701 uint32_t status = vtd_get_long_raw(s, DMAR_GSTS_REG); 2702 uint32_t val = vtd_get_long_raw(s, DMAR_GCMD_REG); 2703 uint32_t changed = status ^ val; 2704 2705 trace_vtd_reg_write_gcmd(status, val); 2706 if ((changed & VTD_GCMD_TE) && s->dma_translation) { 2707 /* Translation enable/disable */ 2708 vtd_handle_gcmd_te(s, val & VTD_GCMD_TE); 2709 } 2710 if (val & VTD_GCMD_SRTP) { 2711 /* Set/update the root-table pointer */ 2712 vtd_handle_gcmd_srtp(s); 2713 } 2714 if (changed & VTD_GCMD_QIE) { 2715 /* Queued Invalidation Enable */ 2716 vtd_handle_gcmd_qie(s, val & VTD_GCMD_QIE); 2717 } 2718 if (val & VTD_GCMD_SIRTP) { 2719 /* Set/update the interrupt remapping root-table pointer */ 2720 vtd_handle_gcmd_sirtp(s); 2721 } 2722 if ((changed & VTD_GCMD_IRE) && 2723 x86_iommu_ir_supported(x86_iommu)) { 2724 /* Interrupt remap enable/disable */ 2725 vtd_handle_gcmd_ire(s, val & VTD_GCMD_IRE); 2726 } 2727 } 2728 2729 /* Handle write to Context Command Register */ 2730 static void vtd_handle_ccmd_write(IntelIOMMUState *s) 2731 { 2732 uint64_t ret; 2733 uint64_t val = vtd_get_quad_raw(s, DMAR_CCMD_REG); 2734 2735 /* Context-cache invalidation request */ 2736 if (val & VTD_CCMD_ICC) { 2737 if (s->qi_enabled) { 2738 error_report_once("Queued Invalidation enabled, " 2739 "should not use register-based invalidation"); 2740 return; 2741 } 2742 ret = vtd_context_cache_invalidate(s, val); 2743 /* Invalidation completed. Change something to show */ 2744 vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_ICC, 0ULL); 2745 ret = vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_CAIG_MASK, 2746 ret); 2747 } 2748 } 2749 2750 /* Handle write to IOTLB Invalidation Register */ 2751 static void vtd_handle_iotlb_write(IntelIOMMUState *s) 2752 { 2753 uint64_t ret; 2754 uint64_t val = vtd_get_quad_raw(s, DMAR_IOTLB_REG); 2755 2756 /* IOTLB invalidation request */ 2757 if (val & VTD_TLB_IVT) { 2758 if (s->qi_enabled) { 2759 error_report_once("Queued Invalidation enabled, " 2760 "should not use register-based invalidation"); 2761 return; 2762 } 2763 ret = vtd_iotlb_flush(s, val); 2764 /* Invalidation completed. Change something to show */ 2765 vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, VTD_TLB_IVT, 0ULL); 2766 ret = vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, 2767 VTD_TLB_FLUSH_GRANU_MASK_A, ret); 2768 } 2769 } 2770 2771 /* Fetch an Invalidation Descriptor from the Invalidation Queue */ 2772 static bool vtd_get_inv_desc(IntelIOMMUState *s, 2773 VTDInvDesc *inv_desc) 2774 { 2775 dma_addr_t base_addr = s->iq; 2776 uint32_t offset = s->iq_head; 2777 uint32_t dw = s->iq_dw ? 32 : 16; 2778 dma_addr_t addr = base_addr + offset * dw; 2779 2780 if (dma_memory_read(&address_space_memory, addr, 2781 inv_desc, dw, MEMTXATTRS_UNSPECIFIED)) { 2782 error_report_once("Read INV DESC failed."); 2783 return false; 2784 } 2785 inv_desc->lo = le64_to_cpu(inv_desc->lo); 2786 inv_desc->hi = le64_to_cpu(inv_desc->hi); 2787 if (dw == 32) { 2788 inv_desc->val[2] = le64_to_cpu(inv_desc->val[2]); 2789 inv_desc->val[3] = le64_to_cpu(inv_desc->val[3]); 2790 } 2791 return true; 2792 } 2793 2794 static bool vtd_inv_desc_reserved_check(IntelIOMMUState *s, 2795 VTDInvDesc *inv_desc, 2796 uint64_t mask[4], bool dw, 2797 const char *func_name, 2798 const char *desc_type) 2799 { 2800 if (s->iq_dw) { 2801 if (inv_desc->val[0] & mask[0] || inv_desc->val[1] & mask[1] || 2802 inv_desc->val[2] & mask[2] || inv_desc->val[3] & mask[3]) { 2803 error_report("%s: invalid %s desc val[3]: 0x%"PRIx64 2804 " val[2]: 0x%"PRIx64" val[1]=0x%"PRIx64 2805 " val[0]=0x%"PRIx64" (reserved nonzero)", 2806 func_name, desc_type, inv_desc->val[3], 2807 inv_desc->val[2], inv_desc->val[1], 2808 inv_desc->val[0]); 2809 return false; 2810 } 2811 } else { 2812 if (dw) { 2813 error_report("%s: 256-bit %s desc in 128-bit invalidation queue", 2814 func_name, desc_type); 2815 return false; 2816 } 2817 2818 if (inv_desc->lo & mask[0] || inv_desc->hi & mask[1]) { 2819 error_report("%s: invalid %s desc: hi=%"PRIx64", lo=%"PRIx64 2820 " (reserved nonzero)", func_name, desc_type, 2821 inv_desc->hi, inv_desc->lo); 2822 return false; 2823 } 2824 } 2825 2826 return true; 2827 } 2828 2829 static bool vtd_process_wait_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc) 2830 { 2831 uint64_t mask[4] = {VTD_INV_DESC_WAIT_RSVD_LO, VTD_INV_DESC_WAIT_RSVD_HI, 2832 VTD_INV_DESC_ALL_ONE, VTD_INV_DESC_ALL_ONE}; 2833 2834 if (!vtd_inv_desc_reserved_check(s, inv_desc, mask, false, 2835 __func__, "wait")) { 2836 return false; 2837 } 2838 2839 if (inv_desc->lo & VTD_INV_DESC_WAIT_SW) { 2840 /* Status Write */ 2841 uint32_t status_data = (uint32_t)(inv_desc->lo >> 2842 VTD_INV_DESC_WAIT_DATA_SHIFT); 2843 2844 assert(!(inv_desc->lo & VTD_INV_DESC_WAIT_IF)); 2845 2846 /* FIXME: need to be masked with HAW? */ 2847 dma_addr_t status_addr = inv_desc->hi; 2848 trace_vtd_inv_desc_wait_sw(status_addr, status_data); 2849 status_data = cpu_to_le32(status_data); 2850 if (dma_memory_write(&address_space_memory, status_addr, 2851 &status_data, sizeof(status_data), 2852 MEMTXATTRS_UNSPECIFIED)) { 2853 trace_vtd_inv_desc_wait_write_fail(inv_desc->hi, inv_desc->lo); 2854 return false; 2855 } 2856 } else if (inv_desc->lo & VTD_INV_DESC_WAIT_IF) { 2857 /* Interrupt flag */ 2858 vtd_generate_completion_event(s); 2859 } else { 2860 error_report_once("%s: invalid wait desc: hi=%"PRIx64", lo=%"PRIx64 2861 " (unknown type)", __func__, inv_desc->hi, 2862 inv_desc->lo); 2863 return false; 2864 } 2865 return true; 2866 } 2867 2868 static bool vtd_process_context_cache_desc(IntelIOMMUState *s, 2869 VTDInvDesc *inv_desc) 2870 { 2871 uint16_t sid, fmask; 2872 uint64_t mask[4] = {VTD_INV_DESC_CC_RSVD, VTD_INV_DESC_ALL_ONE, 2873 VTD_INV_DESC_ALL_ONE, VTD_INV_DESC_ALL_ONE}; 2874 2875 if (!vtd_inv_desc_reserved_check(s, inv_desc, mask, false, 2876 __func__, "cc inv")) { 2877 return false; 2878 } 2879 2880 switch (inv_desc->lo & VTD_INV_DESC_CC_G) { 2881 case VTD_INV_DESC_CC_DOMAIN: 2882 trace_vtd_inv_desc_cc_domain( 2883 (uint16_t)VTD_INV_DESC_CC_DID(inv_desc->lo)); 2884 /* Fall through */ 2885 case VTD_INV_DESC_CC_GLOBAL: 2886 vtd_context_global_invalidate(s); 2887 break; 2888 2889 case VTD_INV_DESC_CC_DEVICE: 2890 sid = VTD_INV_DESC_CC_SID(inv_desc->lo); 2891 fmask = VTD_INV_DESC_CC_FM(inv_desc->lo); 2892 vtd_context_device_invalidate(s, sid, fmask); 2893 break; 2894 2895 default: 2896 error_report_once("%s: invalid cc inv desc: hi=%"PRIx64", lo=%"PRIx64 2897 " (invalid type)", __func__, inv_desc->hi, 2898 inv_desc->lo); 2899 return false; 2900 } 2901 return true; 2902 } 2903 2904 static bool vtd_process_iotlb_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc) 2905 { 2906 uint16_t domain_id; 2907 uint8_t am; 2908 hwaddr addr; 2909 uint64_t mask[4] = {VTD_INV_DESC_IOTLB_RSVD_LO, VTD_INV_DESC_IOTLB_RSVD_HI, 2910 VTD_INV_DESC_ALL_ONE, VTD_INV_DESC_ALL_ONE}; 2911 2912 if (!vtd_inv_desc_reserved_check(s, inv_desc, mask, false, 2913 __func__, "iotlb inv")) { 2914 return false; 2915 } 2916 2917 switch (inv_desc->lo & VTD_INV_DESC_IOTLB_G) { 2918 case VTD_INV_DESC_IOTLB_GLOBAL: 2919 vtd_iotlb_global_invalidate(s); 2920 break; 2921 2922 case VTD_INV_DESC_IOTLB_DOMAIN: 2923 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo); 2924 vtd_iotlb_domain_invalidate(s, domain_id); 2925 break; 2926 2927 case VTD_INV_DESC_IOTLB_PAGE: 2928 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo); 2929 addr = VTD_INV_DESC_IOTLB_ADDR(inv_desc->hi); 2930 am = VTD_INV_DESC_IOTLB_AM(inv_desc->hi); 2931 if (am > VTD_MAMV) { 2932 error_report_once("%s: invalid iotlb inv desc: hi=0x%"PRIx64 2933 ", lo=0x%"PRIx64" (am=%u > VTD_MAMV=%u)", 2934 __func__, inv_desc->hi, inv_desc->lo, 2935 am, (unsigned)VTD_MAMV); 2936 return false; 2937 } 2938 vtd_iotlb_page_invalidate(s, domain_id, addr, am); 2939 break; 2940 2941 default: 2942 error_report_once("%s: invalid iotlb inv desc: hi=0x%"PRIx64 2943 ", lo=0x%"PRIx64" (type mismatch: 0x%llx)", 2944 __func__, inv_desc->hi, inv_desc->lo, 2945 inv_desc->lo & VTD_INV_DESC_IOTLB_G); 2946 return false; 2947 } 2948 return true; 2949 } 2950 2951 static gboolean vtd_hash_remove_by_pasid(gpointer key, gpointer value, 2952 gpointer user_data) 2953 { 2954 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 2955 VTDIOTLBPageInvInfo *info = (VTDIOTLBPageInvInfo *)user_data; 2956 2957 return ((entry->domain_id == info->domain_id) && 2958 (entry->pasid == info->pasid)); 2959 } 2960 2961 static void vtd_piotlb_pasid_invalidate(IntelIOMMUState *s, 2962 uint16_t domain_id, uint32_t pasid) 2963 { 2964 VTDIOTLBPageInvInfo info; 2965 VTDAddressSpace *vtd_as; 2966 VTDContextEntry ce; 2967 2968 info.domain_id = domain_id; 2969 info.pasid = pasid; 2970 2971 vtd_iommu_lock(s); 2972 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_pasid, 2973 &info); 2974 vtd_iommu_unlock(s); 2975 2976 QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) { 2977 if (!vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus), 2978 vtd_as->devfn, &ce) && 2979 domain_id == vtd_get_domain_id(s, &ce, vtd_as->pasid)) { 2980 uint32_t rid2pasid = VTD_CE_GET_RID2PASID(&ce); 2981 2982 if ((vtd_as->pasid != PCI_NO_PASID || pasid != rid2pasid) && 2983 vtd_as->pasid != pasid) { 2984 continue; 2985 } 2986 2987 if (!s->flts || !vtd_as_has_map_notifier(vtd_as)) { 2988 vtd_address_space_sync(vtd_as); 2989 } 2990 } 2991 } 2992 } 2993 2994 static void vtd_piotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id, 2995 uint32_t pasid, hwaddr addr, uint8_t am) 2996 { 2997 VTDIOTLBPageInvInfo info; 2998 2999 info.domain_id = domain_id; 3000 info.pasid = pasid; 3001 info.addr = addr; 3002 info.mask = ~((1 << am) - 1); 3003 3004 vtd_iommu_lock(s); 3005 g_hash_table_foreach_remove(s->iotlb, 3006 vtd_hash_remove_by_page_piotlb, &info); 3007 vtd_iommu_unlock(s); 3008 3009 vtd_iotlb_page_invalidate_notify(s, domain_id, addr, am, pasid); 3010 } 3011 3012 static bool vtd_process_piotlb_desc(IntelIOMMUState *s, 3013 VTDInvDesc *inv_desc) 3014 { 3015 uint16_t domain_id; 3016 uint32_t pasid; 3017 hwaddr addr; 3018 uint8_t am; 3019 uint64_t mask[4] = {VTD_INV_DESC_PIOTLB_RSVD_VAL0, 3020 VTD_INV_DESC_PIOTLB_RSVD_VAL1, 3021 VTD_INV_DESC_ALL_ONE, VTD_INV_DESC_ALL_ONE}; 3022 3023 if (!vtd_inv_desc_reserved_check(s, inv_desc, mask, true, 3024 __func__, "piotlb inv")) { 3025 return false; 3026 } 3027 3028 domain_id = VTD_INV_DESC_PIOTLB_DID(inv_desc->val[0]); 3029 pasid = VTD_INV_DESC_PIOTLB_PASID(inv_desc->val[0]); 3030 switch (inv_desc->val[0] & VTD_INV_DESC_PIOTLB_G) { 3031 case VTD_INV_DESC_PIOTLB_ALL_IN_PASID: 3032 vtd_piotlb_pasid_invalidate(s, domain_id, pasid); 3033 break; 3034 3035 case VTD_INV_DESC_PIOTLB_PSI_IN_PASID: 3036 am = VTD_INV_DESC_PIOTLB_AM(inv_desc->val[1]); 3037 addr = (hwaddr) VTD_INV_DESC_PIOTLB_ADDR(inv_desc->val[1]); 3038 vtd_piotlb_page_invalidate(s, domain_id, pasid, addr, am); 3039 break; 3040 3041 default: 3042 error_report_once("%s: invalid piotlb inv desc: hi=0x%"PRIx64 3043 ", lo=0x%"PRIx64" (type mismatch: 0x%llx)", 3044 __func__, inv_desc->val[1], inv_desc->val[0], 3045 inv_desc->val[0] & VTD_INV_DESC_IOTLB_G); 3046 return false; 3047 } 3048 return true; 3049 } 3050 3051 static bool vtd_process_inv_iec_desc(IntelIOMMUState *s, 3052 VTDInvDesc *inv_desc) 3053 { 3054 uint64_t mask[4] = {VTD_INV_DESC_IEC_RSVD, VTD_INV_DESC_ALL_ONE, 3055 VTD_INV_DESC_ALL_ONE, VTD_INV_DESC_ALL_ONE}; 3056 3057 if (!vtd_inv_desc_reserved_check(s, inv_desc, mask, false, 3058 __func__, "iec inv")) { 3059 return false; 3060 } 3061 3062 trace_vtd_inv_desc_iec(inv_desc->iec.granularity, 3063 inv_desc->iec.index, 3064 inv_desc->iec.index_mask); 3065 3066 vtd_iec_notify_all(s, !inv_desc->iec.granularity, 3067 inv_desc->iec.index, 3068 inv_desc->iec.index_mask); 3069 return true; 3070 } 3071 3072 static void do_invalidate_device_tlb(VTDAddressSpace *vtd_dev_as, 3073 bool size, hwaddr addr) 3074 { 3075 /* 3076 * According to ATS spec table 2.4: 3077 * S = 0, bits 15:12 = xxxx range size: 4K 3078 * S = 1, bits 15:12 = xxx0 range size: 8K 3079 * S = 1, bits 15:12 = xx01 range size: 16K 3080 * S = 1, bits 15:12 = x011 range size: 32K 3081 * S = 1, bits 15:12 = 0111 range size: 64K 3082 * ... 3083 */ 3084 3085 IOMMUTLBEvent event; 3086 uint64_t sz; 3087 3088 if (size) { 3089 sz = (VTD_PAGE_SIZE * 2) << cto64(addr >> VTD_PAGE_SHIFT); 3090 addr &= ~(sz - 1); 3091 } else { 3092 sz = VTD_PAGE_SIZE; 3093 } 3094 3095 event.type = IOMMU_NOTIFIER_DEVIOTLB_UNMAP; 3096 event.entry.target_as = &vtd_dev_as->as; 3097 event.entry.addr_mask = sz - 1; 3098 event.entry.iova = addr; 3099 event.entry.perm = IOMMU_NONE; 3100 event.entry.translated_addr = 0; 3101 memory_region_notify_iommu(&vtd_dev_as->iommu, 0, event); 3102 } 3103 3104 static bool vtd_process_device_piotlb_desc(IntelIOMMUState *s, 3105 VTDInvDesc *inv_desc) 3106 { 3107 uint16_t sid; 3108 VTDAddressSpace *vtd_dev_as; 3109 bool size; 3110 bool global; 3111 hwaddr addr; 3112 uint32_t pasid; 3113 uint64_t mask[4] = {VTD_INV_DESC_PASID_DEVICE_IOTLB_RSVD_VAL0, 3114 VTD_INV_DESC_PASID_DEVICE_IOTLB_RSVD_VAL1, 3115 VTD_INV_DESC_ALL_ONE, VTD_INV_DESC_ALL_ONE}; 3116 3117 if (!vtd_inv_desc_reserved_check(s, inv_desc, mask, true, 3118 __func__, "device piotlb inv")) { 3119 return false; 3120 } 3121 3122 global = VTD_INV_DESC_PASID_DEVICE_IOTLB_GLOBAL(inv_desc->hi); 3123 size = VTD_INV_DESC_PASID_DEVICE_IOTLB_SIZE(inv_desc->hi); 3124 addr = VTD_INV_DESC_PASID_DEVICE_IOTLB_ADDR(inv_desc->hi); 3125 sid = VTD_INV_DESC_PASID_DEVICE_IOTLB_SID(inv_desc->lo); 3126 if (global) { 3127 QLIST_FOREACH(vtd_dev_as, &s->vtd_as_with_notifiers, next) { 3128 if ((vtd_dev_as->pasid != PCI_NO_PASID) && 3129 (PCI_BUILD_BDF(pci_bus_num(vtd_dev_as->bus), 3130 vtd_dev_as->devfn) == sid)) { 3131 do_invalidate_device_tlb(vtd_dev_as, size, addr); 3132 } 3133 } 3134 } else { 3135 pasid = VTD_INV_DESC_PASID_DEVICE_IOTLB_PASID(inv_desc->lo); 3136 vtd_dev_as = vtd_get_as_by_sid_and_pasid(s, sid, pasid); 3137 if (!vtd_dev_as) { 3138 return true; 3139 } 3140 3141 do_invalidate_device_tlb(vtd_dev_as, size, addr); 3142 } 3143 3144 return true; 3145 } 3146 3147 static bool vtd_process_device_iotlb_desc(IntelIOMMUState *s, 3148 VTDInvDesc *inv_desc) 3149 { 3150 VTDAddressSpace *vtd_dev_as; 3151 hwaddr addr; 3152 uint16_t sid; 3153 bool size; 3154 uint64_t mask[4] = {VTD_INV_DESC_DEVICE_IOTLB_RSVD_LO, 3155 VTD_INV_DESC_DEVICE_IOTLB_RSVD_HI, 3156 VTD_INV_DESC_ALL_ONE, VTD_INV_DESC_ALL_ONE}; 3157 3158 if (!vtd_inv_desc_reserved_check(s, inv_desc, mask, false, 3159 __func__, "dev-iotlb inv")) { 3160 return false; 3161 } 3162 3163 addr = VTD_INV_DESC_DEVICE_IOTLB_ADDR(inv_desc->hi); 3164 sid = VTD_INV_DESC_DEVICE_IOTLB_SID(inv_desc->lo); 3165 size = VTD_INV_DESC_DEVICE_IOTLB_SIZE(inv_desc->hi); 3166 3167 /* 3168 * Using sid is OK since the guest should have finished the 3169 * initialization of both the bus and device. 3170 */ 3171 vtd_dev_as = vtd_get_as_by_sid(s, sid); 3172 if (!vtd_dev_as) { 3173 goto done; 3174 } 3175 3176 do_invalidate_device_tlb(vtd_dev_as, size, addr); 3177 3178 done: 3179 return true; 3180 } 3181 3182 static bool vtd_process_inv_desc(IntelIOMMUState *s) 3183 { 3184 VTDInvDesc inv_desc; 3185 uint8_t desc_type; 3186 3187 trace_vtd_inv_qi_head(s->iq_head); 3188 if (!vtd_get_inv_desc(s, &inv_desc)) { 3189 s->iq_last_desc_type = VTD_INV_DESC_NONE; 3190 return false; 3191 } 3192 3193 desc_type = VTD_INV_DESC_TYPE(inv_desc.lo); 3194 /* FIXME: should update at first or at last? */ 3195 s->iq_last_desc_type = desc_type; 3196 3197 switch (desc_type) { 3198 case VTD_INV_DESC_CC: 3199 trace_vtd_inv_desc("context-cache", inv_desc.hi, inv_desc.lo); 3200 if (!vtd_process_context_cache_desc(s, &inv_desc)) { 3201 return false; 3202 } 3203 break; 3204 3205 case VTD_INV_DESC_IOTLB: 3206 trace_vtd_inv_desc("iotlb", inv_desc.hi, inv_desc.lo); 3207 if (!vtd_process_iotlb_desc(s, &inv_desc)) { 3208 return false; 3209 } 3210 break; 3211 3212 case VTD_INV_DESC_PIOTLB: 3213 trace_vtd_inv_desc("p-iotlb", inv_desc.val[1], inv_desc.val[0]); 3214 if (!vtd_process_piotlb_desc(s, &inv_desc)) { 3215 return false; 3216 } 3217 break; 3218 3219 case VTD_INV_DESC_WAIT: 3220 trace_vtd_inv_desc("wait", inv_desc.hi, inv_desc.lo); 3221 if (!vtd_process_wait_desc(s, &inv_desc)) { 3222 return false; 3223 } 3224 break; 3225 3226 case VTD_INV_DESC_IEC: 3227 trace_vtd_inv_desc("iec", inv_desc.hi, inv_desc.lo); 3228 if (!vtd_process_inv_iec_desc(s, &inv_desc)) { 3229 return false; 3230 } 3231 break; 3232 3233 case VTD_INV_DESC_DEV_PIOTLB: 3234 trace_vtd_inv_desc("device-piotlb", inv_desc.hi, inv_desc.lo); 3235 if (!vtd_process_device_piotlb_desc(s, &inv_desc)) { 3236 return false; 3237 } 3238 break; 3239 3240 case VTD_INV_DESC_DEVICE: 3241 trace_vtd_inv_desc("device", inv_desc.hi, inv_desc.lo); 3242 if (!vtd_process_device_iotlb_desc(s, &inv_desc)) { 3243 return false; 3244 } 3245 break; 3246 3247 /* 3248 * TODO: the entity of below two cases will be implemented in future series. 3249 * To make guest (which integrates scalable mode support patch set in 3250 * iommu driver) work, just return true is enough so far. 3251 */ 3252 case VTD_INV_DESC_PC: 3253 if (s->scalable_mode) { 3254 break; 3255 } 3256 /* fallthrough */ 3257 default: 3258 error_report_once("%s: invalid inv desc: hi=%"PRIx64", lo=%"PRIx64 3259 " (unknown type)", __func__, inv_desc.hi, 3260 inv_desc.lo); 3261 return false; 3262 } 3263 s->iq_head++; 3264 if (s->iq_head == s->iq_size) { 3265 s->iq_head = 0; 3266 } 3267 return true; 3268 } 3269 3270 /* Try to fetch and process more Invalidation Descriptors */ 3271 static void vtd_fetch_inv_desc(IntelIOMMUState *s) 3272 { 3273 int qi_shift; 3274 3275 /* Refer to 10.4.23 of VT-d spec 3.0 */ 3276 qi_shift = s->iq_dw ? VTD_IQH_QH_SHIFT_5 : VTD_IQH_QH_SHIFT_4; 3277 3278 trace_vtd_inv_qi_fetch(); 3279 3280 if (s->iq_tail >= s->iq_size) { 3281 /* Detects an invalid Tail pointer */ 3282 error_report_once("%s: detected invalid QI tail " 3283 "(tail=0x%x, size=0x%x)", 3284 __func__, s->iq_tail, s->iq_size); 3285 vtd_handle_inv_queue_error(s); 3286 return; 3287 } 3288 while (s->iq_head != s->iq_tail) { 3289 if (!vtd_process_inv_desc(s)) { 3290 /* Invalidation Queue Errors */ 3291 vtd_handle_inv_queue_error(s); 3292 break; 3293 } 3294 /* Must update the IQH_REG in time */ 3295 vtd_set_quad_raw(s, DMAR_IQH_REG, 3296 (((uint64_t)(s->iq_head)) << qi_shift) & 3297 VTD_IQH_QH_MASK); 3298 } 3299 } 3300 3301 /* Handle write to Invalidation Queue Tail Register */ 3302 static void vtd_handle_iqt_write(IntelIOMMUState *s) 3303 { 3304 uint64_t val = vtd_get_quad_raw(s, DMAR_IQT_REG); 3305 3306 if (s->iq_dw && (val & VTD_IQT_QT_256_RSV_BIT)) { 3307 error_report_once("%s: RSV bit is set: val=0x%"PRIx64, 3308 __func__, val); 3309 vtd_handle_inv_queue_error(s); 3310 return; 3311 } 3312 s->iq_tail = VTD_IQT_QT(s->iq_dw, val); 3313 trace_vtd_inv_qi_tail(s->iq_tail); 3314 3315 if (s->qi_enabled && !(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) { 3316 /* Process Invalidation Queue here */ 3317 vtd_fetch_inv_desc(s); 3318 } 3319 } 3320 3321 static void vtd_handle_fsts_write(IntelIOMMUState *s) 3322 { 3323 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 3324 uint32_t fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG); 3325 uint32_t status_fields = VTD_FSTS_PFO | VTD_FSTS_PPF | VTD_FSTS_IQE; 3326 3327 if ((fectl_reg & VTD_FECTL_IP) && !(fsts_reg & status_fields)) { 3328 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 3329 trace_vtd_fsts_clear_ip(); 3330 } 3331 /* FIXME: when IQE is Clear, should we try to fetch some Invalidation 3332 * Descriptors if there are any when Queued Invalidation is enabled? 3333 */ 3334 } 3335 3336 static void vtd_handle_fectl_write(IntelIOMMUState *s) 3337 { 3338 uint32_t fectl_reg; 3339 /* FIXME: when software clears the IM field, check the IP field. But do we 3340 * need to compare the old value and the new value to conclude that 3341 * software clears the IM field? Or just check if the IM field is zero? 3342 */ 3343 fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG); 3344 3345 trace_vtd_reg_write_fectl(fectl_reg); 3346 3347 if ((fectl_reg & VTD_FECTL_IP) && !(fectl_reg & VTD_FECTL_IM)) { 3348 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG); 3349 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 3350 } 3351 } 3352 3353 static void vtd_handle_ics_write(IntelIOMMUState *s) 3354 { 3355 uint32_t ics_reg = vtd_get_long_raw(s, DMAR_ICS_REG); 3356 uint32_t iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG); 3357 3358 if ((iectl_reg & VTD_IECTL_IP) && !(ics_reg & VTD_ICS_IWC)) { 3359 trace_vtd_reg_ics_clear_ip(); 3360 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 3361 } 3362 } 3363 3364 static void vtd_handle_iectl_write(IntelIOMMUState *s) 3365 { 3366 uint32_t iectl_reg; 3367 /* FIXME: when software clears the IM field, check the IP field. But do we 3368 * need to compare the old value and the new value to conclude that 3369 * software clears the IM field? Or just check if the IM field is zero? 3370 */ 3371 iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG); 3372 3373 trace_vtd_reg_write_iectl(iectl_reg); 3374 3375 if ((iectl_reg & VTD_IECTL_IP) && !(iectl_reg & VTD_IECTL_IM)) { 3376 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG); 3377 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 3378 } 3379 } 3380 3381 static uint64_t vtd_mem_read(void *opaque, hwaddr addr, unsigned size) 3382 { 3383 IntelIOMMUState *s = opaque; 3384 uint64_t val; 3385 3386 trace_vtd_reg_read(addr, size); 3387 3388 if (addr + size > DMAR_REG_SIZE) { 3389 error_report_once("%s: MMIO over range: addr=0x%" PRIx64 3390 " size=0x%x", __func__, addr, size); 3391 return (uint64_t)-1; 3392 } 3393 3394 switch (addr) { 3395 /* Root Table Address Register, 64-bit */ 3396 case DMAR_RTADDR_REG: 3397 val = vtd_get_quad_raw(s, DMAR_RTADDR_REG); 3398 if (size == 4) { 3399 val = val & ((1ULL << 32) - 1); 3400 } 3401 break; 3402 3403 case DMAR_RTADDR_REG_HI: 3404 assert(size == 4); 3405 val = vtd_get_quad_raw(s, DMAR_RTADDR_REG) >> 32; 3406 break; 3407 3408 /* Invalidation Queue Address Register, 64-bit */ 3409 case DMAR_IQA_REG: 3410 val = s->iq | 3411 (vtd_get_quad(s, DMAR_IQA_REG) & 3412 (VTD_IQA_QS | VTD_IQA_DW_MASK)); 3413 if (size == 4) { 3414 val = val & ((1ULL << 32) - 1); 3415 } 3416 break; 3417 3418 case DMAR_IQA_REG_HI: 3419 assert(size == 4); 3420 val = s->iq >> 32; 3421 break; 3422 3423 default: 3424 if (size == 4) { 3425 val = vtd_get_long(s, addr); 3426 } else { 3427 val = vtd_get_quad(s, addr); 3428 } 3429 } 3430 3431 return val; 3432 } 3433 3434 static void vtd_mem_write(void *opaque, hwaddr addr, 3435 uint64_t val, unsigned size) 3436 { 3437 IntelIOMMUState *s = opaque; 3438 3439 trace_vtd_reg_write(addr, size, val); 3440 3441 if (addr + size > DMAR_REG_SIZE) { 3442 error_report_once("%s: MMIO over range: addr=0x%" PRIx64 3443 " size=0x%x", __func__, addr, size); 3444 return; 3445 } 3446 3447 switch (addr) { 3448 /* Global Command Register, 32-bit */ 3449 case DMAR_GCMD_REG: 3450 vtd_set_long(s, addr, val); 3451 vtd_handle_gcmd_write(s); 3452 break; 3453 3454 /* Context Command Register, 64-bit */ 3455 case DMAR_CCMD_REG: 3456 if (size == 4) { 3457 vtd_set_long(s, addr, val); 3458 } else { 3459 vtd_set_quad(s, addr, val); 3460 vtd_handle_ccmd_write(s); 3461 } 3462 break; 3463 3464 case DMAR_CCMD_REG_HI: 3465 assert(size == 4); 3466 vtd_set_long(s, addr, val); 3467 vtd_handle_ccmd_write(s); 3468 break; 3469 3470 /* IOTLB Invalidation Register, 64-bit */ 3471 case DMAR_IOTLB_REG: 3472 if (size == 4) { 3473 vtd_set_long(s, addr, val); 3474 } else { 3475 vtd_set_quad(s, addr, val); 3476 vtd_handle_iotlb_write(s); 3477 } 3478 break; 3479 3480 case DMAR_IOTLB_REG_HI: 3481 assert(size == 4); 3482 vtd_set_long(s, addr, val); 3483 vtd_handle_iotlb_write(s); 3484 break; 3485 3486 /* Invalidate Address Register, 64-bit */ 3487 case DMAR_IVA_REG: 3488 if (size == 4) { 3489 vtd_set_long(s, addr, val); 3490 } else { 3491 vtd_set_quad(s, addr, val); 3492 } 3493 break; 3494 3495 case DMAR_IVA_REG_HI: 3496 assert(size == 4); 3497 vtd_set_long(s, addr, val); 3498 break; 3499 3500 /* Fault Status Register, 32-bit */ 3501 case DMAR_FSTS_REG: 3502 assert(size == 4); 3503 vtd_set_long(s, addr, val); 3504 vtd_handle_fsts_write(s); 3505 break; 3506 3507 /* Fault Event Control Register, 32-bit */ 3508 case DMAR_FECTL_REG: 3509 assert(size == 4); 3510 vtd_set_long(s, addr, val); 3511 vtd_handle_fectl_write(s); 3512 break; 3513 3514 /* Fault Event Data Register, 32-bit */ 3515 case DMAR_FEDATA_REG: 3516 assert(size == 4); 3517 vtd_set_long(s, addr, val); 3518 break; 3519 3520 /* Fault Event Address Register, 32-bit */ 3521 case DMAR_FEADDR_REG: 3522 if (size == 4) { 3523 vtd_set_long(s, addr, val); 3524 } else { 3525 /* 3526 * While the register is 32-bit only, some guests (Xen...) write to 3527 * it with 64-bit. 3528 */ 3529 vtd_set_quad(s, addr, val); 3530 } 3531 break; 3532 3533 /* Fault Event Upper Address Register, 32-bit */ 3534 case DMAR_FEUADDR_REG: 3535 assert(size == 4); 3536 vtd_set_long(s, addr, val); 3537 break; 3538 3539 /* Protected Memory Enable Register, 32-bit */ 3540 case DMAR_PMEN_REG: 3541 assert(size == 4); 3542 vtd_set_long(s, addr, val); 3543 break; 3544 3545 /* Root Table Address Register, 64-bit */ 3546 case DMAR_RTADDR_REG: 3547 if (size == 4) { 3548 vtd_set_long(s, addr, val); 3549 } else { 3550 vtd_set_quad(s, addr, val); 3551 } 3552 break; 3553 3554 case DMAR_RTADDR_REG_HI: 3555 assert(size == 4); 3556 vtd_set_long(s, addr, val); 3557 break; 3558 3559 /* Invalidation Queue Tail Register, 64-bit */ 3560 case DMAR_IQT_REG: 3561 if (size == 4) { 3562 vtd_set_long(s, addr, val); 3563 } else { 3564 vtd_set_quad(s, addr, val); 3565 } 3566 vtd_handle_iqt_write(s); 3567 break; 3568 3569 case DMAR_IQT_REG_HI: 3570 assert(size == 4); 3571 vtd_set_long(s, addr, val); 3572 /* 19:63 of IQT_REG is RsvdZ, do nothing here */ 3573 break; 3574 3575 /* Invalidation Queue Address Register, 64-bit */ 3576 case DMAR_IQA_REG: 3577 if (size == 4) { 3578 vtd_set_long(s, addr, val); 3579 } else { 3580 vtd_set_quad(s, addr, val); 3581 } 3582 vtd_update_iq_dw(s); 3583 break; 3584 3585 case DMAR_IQA_REG_HI: 3586 assert(size == 4); 3587 vtd_set_long(s, addr, val); 3588 break; 3589 3590 /* Invalidation Completion Status Register, 32-bit */ 3591 case DMAR_ICS_REG: 3592 assert(size == 4); 3593 vtd_set_long(s, addr, val); 3594 vtd_handle_ics_write(s); 3595 break; 3596 3597 /* Invalidation Event Control Register, 32-bit */ 3598 case DMAR_IECTL_REG: 3599 assert(size == 4); 3600 vtd_set_long(s, addr, val); 3601 vtd_handle_iectl_write(s); 3602 break; 3603 3604 /* Invalidation Event Data Register, 32-bit */ 3605 case DMAR_IEDATA_REG: 3606 assert(size == 4); 3607 vtd_set_long(s, addr, val); 3608 break; 3609 3610 /* Invalidation Event Address Register, 32-bit */ 3611 case DMAR_IEADDR_REG: 3612 assert(size == 4); 3613 vtd_set_long(s, addr, val); 3614 break; 3615 3616 /* Invalidation Event Upper Address Register, 32-bit */ 3617 case DMAR_IEUADDR_REG: 3618 assert(size == 4); 3619 vtd_set_long(s, addr, val); 3620 break; 3621 3622 /* Fault Recording Registers, 128-bit */ 3623 case DMAR_FRCD_REG_0_0: 3624 if (size == 4) { 3625 vtd_set_long(s, addr, val); 3626 } else { 3627 vtd_set_quad(s, addr, val); 3628 } 3629 break; 3630 3631 case DMAR_FRCD_REG_0_1: 3632 assert(size == 4); 3633 vtd_set_long(s, addr, val); 3634 break; 3635 3636 case DMAR_FRCD_REG_0_2: 3637 if (size == 4) { 3638 vtd_set_long(s, addr, val); 3639 } else { 3640 vtd_set_quad(s, addr, val); 3641 /* May clear bit 127 (Fault), update PPF */ 3642 vtd_update_fsts_ppf(s); 3643 } 3644 break; 3645 3646 case DMAR_FRCD_REG_0_3: 3647 assert(size == 4); 3648 vtd_set_long(s, addr, val); 3649 /* May clear bit 127 (Fault), update PPF */ 3650 vtd_update_fsts_ppf(s); 3651 break; 3652 3653 case DMAR_IRTA_REG: 3654 if (size == 4) { 3655 vtd_set_long(s, addr, val); 3656 } else { 3657 vtd_set_quad(s, addr, val); 3658 } 3659 break; 3660 3661 case DMAR_IRTA_REG_HI: 3662 assert(size == 4); 3663 vtd_set_long(s, addr, val); 3664 break; 3665 3666 default: 3667 if (size == 4) { 3668 vtd_set_long(s, addr, val); 3669 } else { 3670 vtd_set_quad(s, addr, val); 3671 } 3672 } 3673 } 3674 3675 static IOMMUTLBEntry vtd_iommu_translate(IOMMUMemoryRegion *iommu, hwaddr addr, 3676 IOMMUAccessFlags flag, int iommu_idx) 3677 { 3678 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu); 3679 IntelIOMMUState *s = vtd_as->iommu_state; 3680 IOMMUTLBEntry iotlb = { 3681 /* We'll fill in the rest later. */ 3682 .target_as = &address_space_memory, 3683 }; 3684 bool success; 3685 3686 if (likely(s->dmar_enabled)) { 3687 success = vtd_do_iommu_translate(vtd_as, vtd_as->bus, vtd_as->devfn, 3688 addr, flag & IOMMU_WO, &iotlb); 3689 } else { 3690 /* DMAR disabled, passthrough, use 4k-page*/ 3691 iotlb.iova = addr & VTD_PAGE_MASK_4K; 3692 iotlb.translated_addr = addr & VTD_PAGE_MASK_4K; 3693 iotlb.addr_mask = ~VTD_PAGE_MASK_4K; 3694 iotlb.perm = IOMMU_RW; 3695 success = true; 3696 } 3697 3698 if (likely(success)) { 3699 trace_vtd_dmar_translate(pci_bus_num(vtd_as->bus), 3700 VTD_PCI_SLOT(vtd_as->devfn), 3701 VTD_PCI_FUNC(vtd_as->devfn), 3702 iotlb.iova, iotlb.translated_addr, 3703 iotlb.addr_mask); 3704 } else { 3705 error_report_once("%s: detected translation failure " 3706 "(dev=%02x:%02x:%02x, iova=0x%" PRIx64 ")", 3707 __func__, pci_bus_num(vtd_as->bus), 3708 VTD_PCI_SLOT(vtd_as->devfn), 3709 VTD_PCI_FUNC(vtd_as->devfn), 3710 addr); 3711 } 3712 3713 return iotlb; 3714 } 3715 3716 static int vtd_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu, 3717 IOMMUNotifierFlag old, 3718 IOMMUNotifierFlag new, 3719 Error **errp) 3720 { 3721 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu); 3722 IntelIOMMUState *s = vtd_as->iommu_state; 3723 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 3724 3725 /* TODO: add support for VFIO and vhost users */ 3726 if (s->snoop_control) { 3727 error_setg_errno(errp, ENOTSUP, 3728 "Snoop Control with vhost or VFIO is not supported"); 3729 return -ENOTSUP; 3730 } 3731 if (!s->caching_mode && (new & IOMMU_NOTIFIER_MAP)) { 3732 error_setg_errno(errp, ENOTSUP, 3733 "device %02x.%02x.%x requires caching mode", 3734 pci_bus_num(vtd_as->bus), PCI_SLOT(vtd_as->devfn), 3735 PCI_FUNC(vtd_as->devfn)); 3736 return -ENOTSUP; 3737 } 3738 if (!x86_iommu->dt_supported && (new & IOMMU_NOTIFIER_DEVIOTLB_UNMAP)) { 3739 error_setg_errno(errp, ENOTSUP, 3740 "device %02x.%02x.%x requires device IOTLB mode", 3741 pci_bus_num(vtd_as->bus), PCI_SLOT(vtd_as->devfn), 3742 PCI_FUNC(vtd_as->devfn)); 3743 return -ENOTSUP; 3744 } 3745 3746 /* Update per-address-space notifier flags */ 3747 vtd_as->notifier_flags = new; 3748 3749 if (old == IOMMU_NOTIFIER_NONE) { 3750 QLIST_INSERT_HEAD(&s->vtd_as_with_notifiers, vtd_as, next); 3751 } else if (new == IOMMU_NOTIFIER_NONE) { 3752 QLIST_REMOVE(vtd_as, next); 3753 } 3754 return 0; 3755 } 3756 3757 static int vtd_post_load(void *opaque, int version_id) 3758 { 3759 IntelIOMMUState *iommu = opaque; 3760 3761 /* 3762 * We don't need to migrate the root_scalable because we can 3763 * simply do the calculation after the loading is complete. We 3764 * can actually do similar things with root, dmar_enabled, etc. 3765 * however since we've had them already so we'd better keep them 3766 * for compatibility of migration. 3767 */ 3768 vtd_update_scalable_state(iommu); 3769 3770 vtd_update_iq_dw(iommu); 3771 3772 /* 3773 * Memory regions are dynamically turned on/off depending on 3774 * context entry configurations from the guest. After migration, 3775 * we need to make sure the memory regions are still correct. 3776 */ 3777 vtd_switch_address_space_all(iommu); 3778 3779 return 0; 3780 } 3781 3782 static const VMStateDescription vtd_vmstate = { 3783 .name = "iommu-intel", 3784 .version_id = 1, 3785 .minimum_version_id = 1, 3786 .priority = MIG_PRI_IOMMU, 3787 .post_load = vtd_post_load, 3788 .fields = (const VMStateField[]) { 3789 VMSTATE_UINT64(root, IntelIOMMUState), 3790 VMSTATE_UINT64(intr_root, IntelIOMMUState), 3791 VMSTATE_UINT64(iq, IntelIOMMUState), 3792 VMSTATE_UINT32(intr_size, IntelIOMMUState), 3793 VMSTATE_UINT16(iq_head, IntelIOMMUState), 3794 VMSTATE_UINT16(iq_tail, IntelIOMMUState), 3795 VMSTATE_UINT16(iq_size, IntelIOMMUState), 3796 VMSTATE_UINT16(next_frcd_reg, IntelIOMMUState), 3797 VMSTATE_UINT8_ARRAY(csr, IntelIOMMUState, DMAR_REG_SIZE), 3798 VMSTATE_UINT8(iq_last_desc_type, IntelIOMMUState), 3799 VMSTATE_UNUSED(1), /* bool root_extended is obsolete by VT-d */ 3800 VMSTATE_BOOL(dmar_enabled, IntelIOMMUState), 3801 VMSTATE_BOOL(qi_enabled, IntelIOMMUState), 3802 VMSTATE_BOOL(intr_enabled, IntelIOMMUState), 3803 VMSTATE_BOOL(intr_eime, IntelIOMMUState), 3804 VMSTATE_END_OF_LIST() 3805 } 3806 }; 3807 3808 static const MemoryRegionOps vtd_mem_ops = { 3809 .read = vtd_mem_read, 3810 .write = vtd_mem_write, 3811 .endianness = DEVICE_LITTLE_ENDIAN, 3812 .impl = { 3813 .min_access_size = 4, 3814 .max_access_size = 8, 3815 }, 3816 .valid = { 3817 .min_access_size = 4, 3818 .max_access_size = 8, 3819 }, 3820 }; 3821 3822 static const Property vtd_properties[] = { 3823 DEFINE_PROP_UINT32("version", IntelIOMMUState, version, 0), 3824 DEFINE_PROP_ON_OFF_AUTO("eim", IntelIOMMUState, intr_eim, 3825 ON_OFF_AUTO_AUTO), 3826 DEFINE_PROP_BOOL("x-buggy-eim", IntelIOMMUState, buggy_eim, false), 3827 DEFINE_PROP_UINT8("aw-bits", IntelIOMMUState, aw_bits, 3828 VTD_HOST_ADDRESS_WIDTH), 3829 DEFINE_PROP_BOOL("caching-mode", IntelIOMMUState, caching_mode, FALSE), 3830 DEFINE_PROP_BOOL("x-scalable-mode", IntelIOMMUState, scalable_mode, FALSE), 3831 DEFINE_PROP_BOOL("x-flts", IntelIOMMUState, flts, FALSE), 3832 DEFINE_PROP_BOOL("snoop-control", IntelIOMMUState, snoop_control, false), 3833 DEFINE_PROP_BOOL("x-pasid-mode", IntelIOMMUState, pasid, false), 3834 DEFINE_PROP_BOOL("dma-drain", IntelIOMMUState, dma_drain, true), 3835 DEFINE_PROP_BOOL("dma-translation", IntelIOMMUState, dma_translation, true), 3836 DEFINE_PROP_BOOL("stale-tm", IntelIOMMUState, stale_tm, false), 3837 DEFINE_PROP_BOOL("fs1gp", IntelIOMMUState, fs1gp, true), 3838 }; 3839 3840 /* Read IRTE entry with specific index */ 3841 static bool vtd_irte_get(IntelIOMMUState *iommu, uint16_t index, 3842 VTD_IR_TableEntry *entry, uint16_t sid, 3843 bool do_fault) 3844 { 3845 static const uint16_t vtd_svt_mask[VTD_SQ_MAX] = \ 3846 {0xffff, 0xfffb, 0xfff9, 0xfff8}; 3847 dma_addr_t addr = 0x00; 3848 uint16_t mask, source_id; 3849 uint8_t bus, bus_max, bus_min; 3850 3851 if (index >= iommu->intr_size) { 3852 error_report_once("%s: index too large: ind=0x%x", 3853 __func__, index); 3854 if (do_fault) { 3855 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_INDEX_OVER, index); 3856 } 3857 return false; 3858 } 3859 3860 addr = iommu->intr_root + index * sizeof(*entry); 3861 if (dma_memory_read(&address_space_memory, addr, 3862 entry, sizeof(*entry), MEMTXATTRS_UNSPECIFIED)) { 3863 error_report_once("%s: read failed: ind=0x%x addr=0x%" PRIx64, 3864 __func__, index, addr); 3865 if (do_fault) { 3866 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_ROOT_INVAL, index); 3867 } 3868 return false; 3869 } 3870 3871 entry->data[0] = le64_to_cpu(entry->data[0]); 3872 entry->data[1] = le64_to_cpu(entry->data[1]); 3873 3874 trace_vtd_ir_irte_get(index, entry->data[1], entry->data[0]); 3875 3876 /* 3877 * The remaining potential fault conditions are "qualified" by the 3878 * Fault Processing Disable bit in the IRTE. Even "not present". 3879 * So just clear the do_fault flag if PFD is set, which will 3880 * prevent faults being raised. 3881 */ 3882 if (entry->irte.fault_disable) { 3883 do_fault = false; 3884 } 3885 3886 if (!entry->irte.present) { 3887 error_report_once("%s: detected non-present IRTE " 3888 "(index=%u, high=0x%" PRIx64 ", low=0x%" PRIx64 ")", 3889 __func__, index, entry->data[1], entry->data[0]); 3890 if (do_fault) { 3891 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_ENTRY_P, index); 3892 } 3893 return false; 3894 } 3895 3896 if (entry->irte.__reserved_0 || entry->irte.__reserved_1 || 3897 entry->irte.__reserved_2) { 3898 error_report_once("%s: detected non-zero reserved IRTE " 3899 "(index=%u, high=0x%" PRIx64 ", low=0x%" PRIx64 ")", 3900 __func__, index, entry->data[1], entry->data[0]); 3901 if (do_fault) { 3902 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_IRTE_RSVD, index); 3903 } 3904 return false; 3905 } 3906 3907 if (sid != X86_IOMMU_SID_INVALID) { 3908 /* Validate IRTE SID */ 3909 source_id = entry->irte.source_id; 3910 switch (entry->irte.sid_vtype) { 3911 case VTD_SVT_NONE: 3912 break; 3913 3914 case VTD_SVT_ALL: 3915 mask = vtd_svt_mask[entry->irte.sid_q]; 3916 if ((source_id & mask) != (sid & mask)) { 3917 error_report_once("%s: invalid IRTE SID " 3918 "(index=%u, sid=%u, source_id=%u)", 3919 __func__, index, sid, source_id); 3920 if (do_fault) { 3921 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_SID_ERR, index); 3922 } 3923 return false; 3924 } 3925 break; 3926 3927 case VTD_SVT_BUS: 3928 bus_max = source_id >> 8; 3929 bus_min = source_id & 0xff; 3930 bus = sid >> 8; 3931 if (bus > bus_max || bus < bus_min) { 3932 error_report_once("%s: invalid SVT_BUS " 3933 "(index=%u, bus=%u, min=%u, max=%u)", 3934 __func__, index, bus, bus_min, bus_max); 3935 if (do_fault) { 3936 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_SID_ERR, index); 3937 } 3938 return false; 3939 } 3940 break; 3941 3942 default: 3943 error_report_once("%s: detected invalid IRTE SVT " 3944 "(index=%u, type=%d)", __func__, 3945 index, entry->irte.sid_vtype); 3946 /* Take this as verification failure. */ 3947 if (do_fault) { 3948 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_SID_ERR, index); 3949 } 3950 return false; 3951 } 3952 } 3953 3954 return true; 3955 } 3956 3957 /* Fetch IRQ information of specific IR index */ 3958 static bool vtd_remap_irq_get(IntelIOMMUState *iommu, uint16_t index, 3959 X86IOMMUIrq *irq, uint16_t sid, bool do_fault) 3960 { 3961 VTD_IR_TableEntry irte = {}; 3962 3963 if (!vtd_irte_get(iommu, index, &irte, sid, do_fault)) { 3964 return false; 3965 } 3966 3967 irq->trigger_mode = irte.irte.trigger_mode; 3968 irq->vector = irte.irte.vector; 3969 irq->delivery_mode = irte.irte.delivery_mode; 3970 irq->dest = irte.irte.dest_id; 3971 if (!iommu->intr_eime) { 3972 #define VTD_IR_APIC_DEST_MASK (0xff00ULL) 3973 #define VTD_IR_APIC_DEST_SHIFT (8) 3974 irq->dest = (irq->dest & VTD_IR_APIC_DEST_MASK) >> 3975 VTD_IR_APIC_DEST_SHIFT; 3976 } 3977 irq->dest_mode = irte.irte.dest_mode; 3978 irq->redir_hint = irte.irte.redir_hint; 3979 3980 trace_vtd_ir_remap(index, irq->trigger_mode, irq->vector, 3981 irq->delivery_mode, irq->dest, irq->dest_mode); 3982 3983 return true; 3984 } 3985 3986 /* Interrupt remapping for MSI/MSI-X entry */ 3987 static int vtd_interrupt_remap_msi(IntelIOMMUState *iommu, 3988 MSIMessage *origin, 3989 MSIMessage *translated, 3990 uint16_t sid, bool do_fault) 3991 { 3992 VTD_IR_MSIAddress addr; 3993 uint16_t index; 3994 X86IOMMUIrq irq = {}; 3995 3996 assert(origin && translated); 3997 3998 trace_vtd_ir_remap_msi_req(origin->address, origin->data); 3999 4000 if (!iommu || !iommu->intr_enabled) { 4001 memcpy(translated, origin, sizeof(*origin)); 4002 goto out; 4003 } 4004 4005 if (origin->address & VTD_MSI_ADDR_HI_MASK) { 4006 error_report_once("%s: MSI address high 32 bits non-zero detected: " 4007 "address=0x%" PRIx64, __func__, origin->address); 4008 if (do_fault) { 4009 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_REQ_RSVD, 0); 4010 } 4011 return -EINVAL; 4012 } 4013 4014 addr.data = origin->address & VTD_MSI_ADDR_LO_MASK; 4015 if (addr.addr.__head != 0xfee) { 4016 error_report_once("%s: MSI address low 32 bit invalid: 0x%" PRIx32, 4017 __func__, addr.data); 4018 if (do_fault) { 4019 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_REQ_RSVD, 0); 4020 } 4021 return -EINVAL; 4022 } 4023 4024 /* This is compatible mode. */ 4025 if (addr.addr.int_mode != VTD_IR_INT_FORMAT_REMAP) { 4026 memcpy(translated, origin, sizeof(*origin)); 4027 goto out; 4028 } 4029 4030 index = addr.addr.index_h << 15 | addr.addr.index_l; 4031 4032 #define VTD_IR_MSI_DATA_SUBHANDLE (0x0000ffff) 4033 #define VTD_IR_MSI_DATA_RESERVED (0xffff0000) 4034 4035 if (addr.addr.sub_valid) { 4036 /* See VT-d spec 5.1.2.2 and 5.1.3 on subhandle */ 4037 index += origin->data & VTD_IR_MSI_DATA_SUBHANDLE; 4038 } 4039 4040 if (!vtd_remap_irq_get(iommu, index, &irq, sid, do_fault)) { 4041 return -EINVAL; 4042 } 4043 4044 if (addr.addr.sub_valid) { 4045 trace_vtd_ir_remap_type("MSI"); 4046 if (origin->data & VTD_IR_MSI_DATA_RESERVED) { 4047 error_report_once("%s: invalid IR MSI " 4048 "(sid=%u, address=0x%" PRIx64 4049 ", data=0x%" PRIx32 ")", 4050 __func__, sid, origin->address, origin->data); 4051 if (do_fault) { 4052 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_REQ_RSVD, 0); 4053 } 4054 return -EINVAL; 4055 } 4056 } else { 4057 uint8_t vector = origin->data & 0xff; 4058 uint8_t trigger_mode = (origin->data >> MSI_DATA_TRIGGER_SHIFT) & 0x1; 4059 4060 trace_vtd_ir_remap_type("IOAPIC"); 4061 /* IOAPIC entry vector should be aligned with IRTE vector 4062 * (see vt-d spec 5.1.5.1). */ 4063 if (vector != irq.vector) { 4064 trace_vtd_warn_ir_vector(sid, index, vector, irq.vector); 4065 } 4066 4067 /* The Trigger Mode field must match the Trigger Mode in the IRTE. 4068 * (see vt-d spec 5.1.5.1). */ 4069 if (trigger_mode != irq.trigger_mode) { 4070 trace_vtd_warn_ir_trigger(sid, index, trigger_mode, 4071 irq.trigger_mode); 4072 } 4073 } 4074 4075 /* 4076 * We'd better keep the last two bits, assuming that guest OS 4077 * might modify it. Keep it does not hurt after all. 4078 */ 4079 irq.msi_addr_last_bits = addr.addr.__not_care; 4080 4081 /* Translate X86IOMMUIrq to MSI message */ 4082 x86_iommu_irq_to_msi_message(&irq, translated); 4083 4084 out: 4085 trace_vtd_ir_remap_msi(origin->address, origin->data, 4086 translated->address, translated->data); 4087 return 0; 4088 } 4089 4090 static int vtd_int_remap(X86IOMMUState *iommu, MSIMessage *src, 4091 MSIMessage *dst, uint16_t sid) 4092 { 4093 return vtd_interrupt_remap_msi(INTEL_IOMMU_DEVICE(iommu), 4094 src, dst, sid, false); 4095 } 4096 4097 static MemTxResult vtd_mem_ir_read(void *opaque, hwaddr addr, 4098 uint64_t *data, unsigned size, 4099 MemTxAttrs attrs) 4100 { 4101 return MEMTX_OK; 4102 } 4103 4104 static MemTxResult vtd_mem_ir_write(void *opaque, hwaddr addr, 4105 uint64_t value, unsigned size, 4106 MemTxAttrs attrs) 4107 { 4108 int ret = 0; 4109 MSIMessage from = {}, to = {}; 4110 uint16_t sid = X86_IOMMU_SID_INVALID; 4111 4112 from.address = (uint64_t) addr + VTD_INTERRUPT_ADDR_FIRST; 4113 from.data = (uint32_t) value; 4114 4115 if (!attrs.unspecified) { 4116 /* We have explicit Source ID */ 4117 sid = attrs.requester_id; 4118 } 4119 4120 ret = vtd_interrupt_remap_msi(opaque, &from, &to, sid, true); 4121 if (ret) { 4122 /* Drop this interrupt */ 4123 return MEMTX_ERROR; 4124 } 4125 4126 apic_get_class(NULL)->send_msi(&to); 4127 4128 return MEMTX_OK; 4129 } 4130 4131 static const MemoryRegionOps vtd_mem_ir_ops = { 4132 .read_with_attrs = vtd_mem_ir_read, 4133 .write_with_attrs = vtd_mem_ir_write, 4134 .endianness = DEVICE_LITTLE_ENDIAN, 4135 .impl = { 4136 .min_access_size = 4, 4137 .max_access_size = 4, 4138 }, 4139 .valid = { 4140 .min_access_size = 4, 4141 .max_access_size = 4, 4142 }, 4143 }; 4144 4145 static void vtd_report_ir_illegal_access(VTDAddressSpace *vtd_as, 4146 hwaddr addr, bool is_write) 4147 { 4148 IntelIOMMUState *s = vtd_as->iommu_state; 4149 uint8_t bus_n = pci_bus_num(vtd_as->bus); 4150 uint16_t sid = PCI_BUILD_BDF(bus_n, vtd_as->devfn); 4151 bool is_fpd_set = false; 4152 VTDContextEntry ce; 4153 4154 assert(vtd_as->pasid != PCI_NO_PASID); 4155 4156 /* Try out best to fetch FPD, we can't do anything more */ 4157 if (vtd_dev_to_context_entry(s, bus_n, vtd_as->devfn, &ce) == 0) { 4158 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 4159 if (!is_fpd_set && s->root_scalable) { 4160 vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set, vtd_as->pasid); 4161 } 4162 } 4163 4164 vtd_report_fault(s, VTD_FR_SM_INTERRUPT_ADDR, 4165 is_fpd_set, sid, addr, is_write, 4166 true, vtd_as->pasid); 4167 } 4168 4169 static MemTxResult vtd_mem_ir_fault_read(void *opaque, hwaddr addr, 4170 uint64_t *data, unsigned size, 4171 MemTxAttrs attrs) 4172 { 4173 vtd_report_ir_illegal_access(opaque, addr, false); 4174 4175 return MEMTX_ERROR; 4176 } 4177 4178 static MemTxResult vtd_mem_ir_fault_write(void *opaque, hwaddr addr, 4179 uint64_t value, unsigned size, 4180 MemTxAttrs attrs) 4181 { 4182 vtd_report_ir_illegal_access(opaque, addr, true); 4183 4184 return MEMTX_ERROR; 4185 } 4186 4187 static const MemoryRegionOps vtd_mem_ir_fault_ops = { 4188 .read_with_attrs = vtd_mem_ir_fault_read, 4189 .write_with_attrs = vtd_mem_ir_fault_write, 4190 .endianness = DEVICE_LITTLE_ENDIAN, 4191 .impl = { 4192 .min_access_size = 1, 4193 .max_access_size = 8, 4194 }, 4195 .valid = { 4196 .min_access_size = 1, 4197 .max_access_size = 8, 4198 }, 4199 }; 4200 4201 VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, 4202 int devfn, unsigned int pasid) 4203 { 4204 /* 4205 * We can't simply use sid here since the bus number might not be 4206 * initialized by the guest. 4207 */ 4208 struct vtd_as_key key = { 4209 .bus = bus, 4210 .devfn = devfn, 4211 .pasid = pasid, 4212 }; 4213 VTDAddressSpace *vtd_dev_as; 4214 char name[128]; 4215 4216 vtd_dev_as = g_hash_table_lookup(s->vtd_address_spaces, &key); 4217 if (!vtd_dev_as) { 4218 struct vtd_as_key *new_key = g_malloc(sizeof(*new_key)); 4219 4220 new_key->bus = bus; 4221 new_key->devfn = devfn; 4222 new_key->pasid = pasid; 4223 4224 if (pasid == PCI_NO_PASID) { 4225 snprintf(name, sizeof(name), "vtd-%02x.%x", PCI_SLOT(devfn), 4226 PCI_FUNC(devfn)); 4227 } else { 4228 snprintf(name, sizeof(name), "vtd-%02x.%x-pasid-%x", PCI_SLOT(devfn), 4229 PCI_FUNC(devfn), pasid); 4230 } 4231 4232 vtd_dev_as = g_new0(VTDAddressSpace, 1); 4233 4234 vtd_dev_as->bus = bus; 4235 vtd_dev_as->devfn = (uint8_t)devfn; 4236 vtd_dev_as->pasid = pasid; 4237 vtd_dev_as->iommu_state = s; 4238 vtd_dev_as->context_cache_entry.context_cache_gen = 0; 4239 vtd_dev_as->iova_tree = iova_tree_new(); 4240 4241 memory_region_init(&vtd_dev_as->root, OBJECT(s), name, UINT64_MAX); 4242 address_space_init(&vtd_dev_as->as, &vtd_dev_as->root, "vtd-root"); 4243 4244 /* 4245 * Build the DMAR-disabled container with aliases to the 4246 * shared MRs. Note that aliasing to a shared memory region 4247 * could help the memory API to detect same FlatViews so we 4248 * can have devices to share the same FlatView when DMAR is 4249 * disabled (either by not providing "intel_iommu=on" or with 4250 * "iommu=pt"). It will greatly reduce the total number of 4251 * FlatViews of the system hence VM runs faster. 4252 */ 4253 memory_region_init_alias(&vtd_dev_as->nodmar, OBJECT(s), 4254 "vtd-nodmar", &s->mr_nodmar, 0, 4255 memory_region_size(&s->mr_nodmar)); 4256 4257 /* 4258 * Build the per-device DMAR-enabled container. 4259 * 4260 * TODO: currently we have per-device IOMMU memory region only 4261 * because we have per-device IOMMU notifiers for devices. If 4262 * one day we can abstract the IOMMU notifiers out of the 4263 * memory regions then we can also share the same memory 4264 * region here just like what we've done above with the nodmar 4265 * region. 4266 */ 4267 strcat(name, "-dmar"); 4268 memory_region_init_iommu(&vtd_dev_as->iommu, sizeof(vtd_dev_as->iommu), 4269 TYPE_INTEL_IOMMU_MEMORY_REGION, OBJECT(s), 4270 name, UINT64_MAX); 4271 memory_region_init_alias(&vtd_dev_as->iommu_ir, OBJECT(s), "vtd-ir", 4272 &s->mr_ir, 0, memory_region_size(&s->mr_ir)); 4273 memory_region_add_subregion_overlap(MEMORY_REGION(&vtd_dev_as->iommu), 4274 VTD_INTERRUPT_ADDR_FIRST, 4275 &vtd_dev_as->iommu_ir, 1); 4276 4277 /* 4278 * This region is used for catching fault to access interrupt 4279 * range via passthrough + PASID. See also 4280 * vtd_switch_address_space(). We can't use alias since we 4281 * need to know the sid which is valid for MSI who uses 4282 * bus_master_as (see msi_send_message()). 4283 */ 4284 memory_region_init_io(&vtd_dev_as->iommu_ir_fault, OBJECT(s), 4285 &vtd_mem_ir_fault_ops, vtd_dev_as, "vtd-no-ir", 4286 VTD_INTERRUPT_ADDR_SIZE); 4287 /* 4288 * Hook to root since when PT is enabled vtd_dev_as->iommu 4289 * will be disabled. 4290 */ 4291 memory_region_add_subregion_overlap(MEMORY_REGION(&vtd_dev_as->root), 4292 VTD_INTERRUPT_ADDR_FIRST, 4293 &vtd_dev_as->iommu_ir_fault, 2); 4294 4295 /* 4296 * Hook both the containers under the root container, we 4297 * switch between DMAR & noDMAR by enable/disable 4298 * corresponding sub-containers 4299 */ 4300 memory_region_add_subregion_overlap(&vtd_dev_as->root, 0, 4301 MEMORY_REGION(&vtd_dev_as->iommu), 4302 0); 4303 memory_region_add_subregion_overlap(&vtd_dev_as->root, 0, 4304 &vtd_dev_as->nodmar, 0); 4305 4306 vtd_switch_address_space(vtd_dev_as); 4307 4308 g_hash_table_insert(s->vtd_address_spaces, new_key, vtd_dev_as); 4309 } 4310 return vtd_dev_as; 4311 } 4312 4313 static bool vtd_check_hiod(IntelIOMMUState *s, HostIOMMUDevice *hiod, 4314 Error **errp) 4315 { 4316 HostIOMMUDeviceClass *hiodc = HOST_IOMMU_DEVICE_GET_CLASS(hiod); 4317 int ret; 4318 4319 if (!hiodc->get_cap) { 4320 error_setg(errp, ".get_cap() not implemented"); 4321 return false; 4322 } 4323 4324 /* Common checks */ 4325 ret = hiodc->get_cap(hiod, HOST_IOMMU_DEVICE_CAP_AW_BITS, errp); 4326 if (ret < 0) { 4327 return false; 4328 } 4329 if (s->aw_bits > ret) { 4330 error_setg(errp, "aw-bits %d > host aw-bits %d", s->aw_bits, ret); 4331 return false; 4332 } 4333 4334 if (!s->flts) { 4335 /* All checks requested by VTD stage-2 translation pass */ 4336 return true; 4337 } 4338 4339 error_setg(errp, "host device is uncompatible with stage-1 translation"); 4340 return false; 4341 } 4342 4343 static bool vtd_dev_set_iommu_device(PCIBus *bus, void *opaque, int devfn, 4344 HostIOMMUDevice *hiod, Error **errp) 4345 { 4346 IntelIOMMUState *s = opaque; 4347 struct vtd_as_key key = { 4348 .bus = bus, 4349 .devfn = devfn, 4350 }; 4351 struct vtd_as_key *new_key; 4352 4353 assert(hiod); 4354 4355 vtd_iommu_lock(s); 4356 4357 if (g_hash_table_lookup(s->vtd_host_iommu_dev, &key)) { 4358 error_setg(errp, "Host IOMMU device already exist"); 4359 vtd_iommu_unlock(s); 4360 return false; 4361 } 4362 4363 if (!vtd_check_hiod(s, hiod, errp)) { 4364 vtd_iommu_unlock(s); 4365 return false; 4366 } 4367 4368 new_key = g_malloc(sizeof(*new_key)); 4369 new_key->bus = bus; 4370 new_key->devfn = devfn; 4371 4372 object_ref(hiod); 4373 g_hash_table_insert(s->vtd_host_iommu_dev, new_key, hiod); 4374 4375 vtd_iommu_unlock(s); 4376 4377 return true; 4378 } 4379 4380 static void vtd_dev_unset_iommu_device(PCIBus *bus, void *opaque, int devfn) 4381 { 4382 IntelIOMMUState *s = opaque; 4383 struct vtd_as_key key = { 4384 .bus = bus, 4385 .devfn = devfn, 4386 }; 4387 4388 vtd_iommu_lock(s); 4389 4390 if (!g_hash_table_lookup(s->vtd_host_iommu_dev, &key)) { 4391 vtd_iommu_unlock(s); 4392 return; 4393 } 4394 4395 g_hash_table_remove(s->vtd_host_iommu_dev, &key); 4396 4397 vtd_iommu_unlock(s); 4398 } 4399 4400 /* Unmap the whole range in the notifier's scope. */ 4401 static void vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier *n) 4402 { 4403 hwaddr total, remain; 4404 hwaddr start = n->start; 4405 hwaddr end = n->end; 4406 IntelIOMMUState *s = as->iommu_state; 4407 DMAMap map; 4408 4409 /* 4410 * Note: all the codes in this function has a assumption that IOVA 4411 * bits are no more than VTD_MGAW bits (which is restricted by 4412 * VT-d spec), otherwise we need to consider overflow of 64 bits. 4413 */ 4414 4415 if (end > VTD_ADDRESS_SIZE(s->aw_bits) - 1) { 4416 /* 4417 * Don't need to unmap regions that is bigger than the whole 4418 * VT-d supported address space size 4419 */ 4420 end = VTD_ADDRESS_SIZE(s->aw_bits) - 1; 4421 } 4422 4423 assert(start <= end); 4424 total = remain = end - start + 1; 4425 4426 while (remain >= VTD_PAGE_SIZE) { 4427 IOMMUTLBEvent event; 4428 uint64_t mask = dma_aligned_pow2_mask(start, end, s->aw_bits); 4429 uint64_t size = mask + 1; 4430 4431 assert(size); 4432 4433 event.type = IOMMU_NOTIFIER_UNMAP; 4434 event.entry.iova = start; 4435 event.entry.addr_mask = mask; 4436 event.entry.target_as = &address_space_memory; 4437 event.entry.perm = IOMMU_NONE; 4438 /* This field is meaningless for unmap */ 4439 event.entry.translated_addr = 0; 4440 4441 memory_region_notify_iommu_one(n, &event); 4442 4443 start += size; 4444 remain -= size; 4445 } 4446 4447 assert(!remain); 4448 4449 trace_vtd_as_unmap_whole(pci_bus_num(as->bus), 4450 VTD_PCI_SLOT(as->devfn), 4451 VTD_PCI_FUNC(as->devfn), 4452 n->start, total); 4453 4454 map.iova = n->start; 4455 map.size = total - 1; /* Inclusive */ 4456 iova_tree_remove(as->iova_tree, map); 4457 } 4458 4459 static void vtd_address_space_unmap_all(IntelIOMMUState *s) 4460 { 4461 VTDAddressSpace *vtd_as; 4462 IOMMUNotifier *n; 4463 4464 QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) { 4465 IOMMU_NOTIFIER_FOREACH(n, &vtd_as->iommu) { 4466 vtd_address_space_unmap(vtd_as, n); 4467 } 4468 } 4469 } 4470 4471 static void vtd_address_space_refresh_all(IntelIOMMUState *s) 4472 { 4473 vtd_address_space_unmap_all(s); 4474 vtd_switch_address_space_all(s); 4475 } 4476 4477 static int vtd_replay_hook(const IOMMUTLBEvent *event, void *private) 4478 { 4479 memory_region_notify_iommu_one(private, event); 4480 return 0; 4481 } 4482 4483 static void vtd_iommu_replay(IOMMUMemoryRegion *iommu_mr, IOMMUNotifier *n) 4484 { 4485 VTDAddressSpace *vtd_as = container_of(iommu_mr, VTDAddressSpace, iommu); 4486 IntelIOMMUState *s = vtd_as->iommu_state; 4487 uint8_t bus_n = pci_bus_num(vtd_as->bus); 4488 VTDContextEntry ce; 4489 DMAMap map = { .iova = 0, .size = HWADDR_MAX }; 4490 4491 /* replay is protected by BQL, page walk will re-setup it safely */ 4492 iova_tree_remove(vtd_as->iova_tree, map); 4493 4494 if (vtd_dev_to_context_entry(s, bus_n, vtd_as->devfn, &ce) == 0) { 4495 trace_vtd_replay_ce_valid(s->root_scalable ? "scalable mode" : 4496 "legacy mode", 4497 bus_n, PCI_SLOT(vtd_as->devfn), 4498 PCI_FUNC(vtd_as->devfn), 4499 vtd_get_domain_id(s, &ce, vtd_as->pasid), 4500 ce.hi, ce.lo); 4501 if (n->notifier_flags & IOMMU_NOTIFIER_MAP) { 4502 /* This is required only for MAP typed notifiers */ 4503 vtd_page_walk_info info = { 4504 .hook_fn = vtd_replay_hook, 4505 .private = (void *)n, 4506 .notify_unmap = false, 4507 .aw = s->aw_bits, 4508 .as = vtd_as, 4509 .domain_id = vtd_get_domain_id(s, &ce, vtd_as->pasid), 4510 }; 4511 4512 vtd_page_walk(s, &ce, 0, ~0ULL, &info, vtd_as->pasid); 4513 } 4514 } else { 4515 trace_vtd_replay_ce_invalid(bus_n, PCI_SLOT(vtd_as->devfn), 4516 PCI_FUNC(vtd_as->devfn)); 4517 } 4518 4519 return; 4520 } 4521 4522 static void vtd_cap_init(IntelIOMMUState *s) 4523 { 4524 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 4525 4526 s->cap = VTD_CAP_FRO | VTD_CAP_NFR | VTD_CAP_ND | 4527 VTD_CAP_MAMV | VTD_CAP_PSI | VTD_CAP_SLLPS | 4528 VTD_CAP_MGAW(s->aw_bits); 4529 if (s->dma_drain) { 4530 s->cap |= VTD_CAP_DRAIN; 4531 } 4532 if (s->dma_translation) { 4533 if (s->aw_bits >= VTD_HOST_AW_39BIT) { 4534 s->cap |= VTD_CAP_SAGAW_39bit; 4535 } 4536 if (s->aw_bits >= VTD_HOST_AW_48BIT) { 4537 s->cap |= VTD_CAP_SAGAW_48bit; 4538 } 4539 } 4540 s->ecap = VTD_ECAP_QI | VTD_ECAP_IRO; 4541 4542 if (x86_iommu_ir_supported(x86_iommu)) { 4543 s->ecap |= VTD_ECAP_IR | VTD_ECAP_MHMV; 4544 if (s->intr_eim == ON_OFF_AUTO_ON) { 4545 s->ecap |= VTD_ECAP_EIM; 4546 } 4547 assert(s->intr_eim != ON_OFF_AUTO_AUTO); 4548 } 4549 4550 if (x86_iommu->dt_supported) { 4551 s->ecap |= VTD_ECAP_DT; 4552 } 4553 4554 if (x86_iommu->pt_supported) { 4555 s->ecap |= VTD_ECAP_PT; 4556 } 4557 4558 if (s->caching_mode) { 4559 s->cap |= VTD_CAP_CM; 4560 } 4561 4562 /* TODO: read cap/ecap from host to decide which cap to be exposed. */ 4563 if (s->flts) { 4564 s->ecap |= VTD_ECAP_SMTS | VTD_ECAP_FLTS; 4565 if (s->fs1gp) { 4566 s->cap |= VTD_CAP_FS1GP; 4567 } 4568 } else if (s->scalable_mode) { 4569 s->ecap |= VTD_ECAP_SMTS | VTD_ECAP_SRS | VTD_ECAP_SLTS; 4570 } 4571 4572 if (s->snoop_control) { 4573 s->ecap |= VTD_ECAP_SC; 4574 } 4575 4576 if (s->pasid) { 4577 s->ecap |= VTD_ECAP_PASID; 4578 } 4579 } 4580 4581 /* 4582 * Do the initialization. It will also be called when reset, so pay 4583 * attention when adding new initialization stuff. 4584 */ 4585 static void vtd_init(IntelIOMMUState *s) 4586 { 4587 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 4588 4589 memset(s->csr, 0, DMAR_REG_SIZE); 4590 memset(s->wmask, 0, DMAR_REG_SIZE); 4591 memset(s->w1cmask, 0, DMAR_REG_SIZE); 4592 memset(s->womask, 0, DMAR_REG_SIZE); 4593 4594 s->root = 0; 4595 s->root_scalable = false; 4596 s->dmar_enabled = false; 4597 s->intr_enabled = false; 4598 s->iq_head = 0; 4599 s->iq_tail = 0; 4600 s->iq = 0; 4601 s->iq_size = 0; 4602 s->qi_enabled = false; 4603 s->iq_last_desc_type = VTD_INV_DESC_NONE; 4604 s->iq_dw = false; 4605 s->next_frcd_reg = 0; 4606 4607 vtd_cap_init(s); 4608 4609 /* 4610 * Rsvd field masks for spte 4611 */ 4612 vtd_spte_rsvd[0] = ~0ULL; 4613 vtd_spte_rsvd[1] = VTD_SPTE_PAGE_L1_RSVD_MASK(s->aw_bits, 4614 x86_iommu->dt_supported && s->stale_tm); 4615 vtd_spte_rsvd[2] = VTD_SPTE_PAGE_L2_RSVD_MASK(s->aw_bits); 4616 vtd_spte_rsvd[3] = VTD_SPTE_PAGE_L3_RSVD_MASK(s->aw_bits); 4617 vtd_spte_rsvd[4] = VTD_SPTE_PAGE_L4_RSVD_MASK(s->aw_bits); 4618 4619 vtd_spte_rsvd_large[2] = VTD_SPTE_LPAGE_L2_RSVD_MASK(s->aw_bits, 4620 x86_iommu->dt_supported && s->stale_tm); 4621 vtd_spte_rsvd_large[3] = VTD_SPTE_LPAGE_L3_RSVD_MASK(s->aw_bits, 4622 x86_iommu->dt_supported && s->stale_tm); 4623 4624 /* 4625 * Rsvd field masks for fpte 4626 */ 4627 vtd_fpte_rsvd[0] = ~0ULL; 4628 vtd_fpte_rsvd[1] = VTD_FPTE_PAGE_L1_RSVD_MASK(s->aw_bits); 4629 vtd_fpte_rsvd[2] = VTD_FPTE_PAGE_L2_RSVD_MASK(s->aw_bits); 4630 vtd_fpte_rsvd[3] = VTD_FPTE_PAGE_L3_RSVD_MASK(s->aw_bits); 4631 vtd_fpte_rsvd[4] = VTD_FPTE_PAGE_L4_RSVD_MASK(s->aw_bits); 4632 4633 vtd_fpte_rsvd_large[2] = VTD_FPTE_LPAGE_L2_RSVD_MASK(s->aw_bits); 4634 vtd_fpte_rsvd_large[3] = VTD_FPTE_LPAGE_L3_RSVD_MASK(s->aw_bits); 4635 4636 if (s->scalable_mode || s->snoop_control) { 4637 vtd_spte_rsvd[1] &= ~VTD_SPTE_SNP; 4638 vtd_spte_rsvd_large[2] &= ~VTD_SPTE_SNP; 4639 vtd_spte_rsvd_large[3] &= ~VTD_SPTE_SNP; 4640 } 4641 4642 vtd_reset_caches(s); 4643 4644 /* Define registers with default values and bit semantics */ 4645 vtd_define_long(s, DMAR_VER_REG, 0x10UL, 0, 0); 4646 vtd_define_quad(s, DMAR_CAP_REG, s->cap, 0, 0); 4647 vtd_define_quad(s, DMAR_ECAP_REG, s->ecap, 0, 0); 4648 vtd_define_long(s, DMAR_GCMD_REG, 0, 0xff800000UL, 0); 4649 vtd_define_long_wo(s, DMAR_GCMD_REG, 0xff800000UL); 4650 vtd_define_long(s, DMAR_GSTS_REG, 0, 0, 0); 4651 vtd_define_quad(s, DMAR_RTADDR_REG, 0, 0xfffffffffffffc00ULL, 0); 4652 vtd_define_quad(s, DMAR_CCMD_REG, 0, 0xe0000003ffffffffULL, 0); 4653 vtd_define_quad_wo(s, DMAR_CCMD_REG, 0x3ffff0000ULL); 4654 4655 /* Advanced Fault Logging not supported */ 4656 vtd_define_long(s, DMAR_FSTS_REG, 0, 0, 0x11UL); 4657 vtd_define_long(s, DMAR_FECTL_REG, 0x80000000UL, 0x80000000UL, 0); 4658 vtd_define_long(s, DMAR_FEDATA_REG, 0, 0x0000ffffUL, 0); 4659 vtd_define_long(s, DMAR_FEADDR_REG, 0, 0xfffffffcUL, 0); 4660 4661 /* Treated as RsvdZ when EIM in ECAP_REG is not supported 4662 * vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0xffffffffUL, 0); 4663 */ 4664 vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0, 0); 4665 4666 /* Treated as RO for implementations that PLMR and PHMR fields reported 4667 * as Clear in the CAP_REG. 4668 * vtd_define_long(s, DMAR_PMEN_REG, 0, 0x80000000UL, 0); 4669 */ 4670 vtd_define_long(s, DMAR_PMEN_REG, 0, 0, 0); 4671 4672 vtd_define_quad(s, DMAR_IQH_REG, 0, 0, 0); 4673 vtd_define_quad(s, DMAR_IQT_REG, 0, 0x7fff0ULL, 0); 4674 vtd_define_quad(s, DMAR_IQA_REG, 0, 0xfffffffffffff807ULL, 0); 4675 vtd_define_long(s, DMAR_ICS_REG, 0, 0, 0x1UL); 4676 vtd_define_long(s, DMAR_IECTL_REG, 0x80000000UL, 0x80000000UL, 0); 4677 vtd_define_long(s, DMAR_IEDATA_REG, 0, 0xffffffffUL, 0); 4678 vtd_define_long(s, DMAR_IEADDR_REG, 0, 0xfffffffcUL, 0); 4679 /* Treadted as RsvdZ when EIM in ECAP_REG is not supported */ 4680 vtd_define_long(s, DMAR_IEUADDR_REG, 0, 0, 0); 4681 4682 /* IOTLB registers */ 4683 vtd_define_quad(s, DMAR_IOTLB_REG, 0, 0Xb003ffff00000000ULL, 0); 4684 vtd_define_quad(s, DMAR_IVA_REG, 0, 0xfffffffffffff07fULL, 0); 4685 vtd_define_quad_wo(s, DMAR_IVA_REG, 0xfffffffffffff07fULL); 4686 4687 /* Fault Recording Registers, 128-bit */ 4688 vtd_define_quad(s, DMAR_FRCD_REG_0_0, 0, 0, 0); 4689 vtd_define_quad(s, DMAR_FRCD_REG_0_2, 0, 0, 0x8000000000000000ULL); 4690 4691 /* 4692 * Interrupt remapping registers. 4693 */ 4694 vtd_define_quad(s, DMAR_IRTA_REG, 0, 0xfffffffffffff80fULL, 0); 4695 } 4696 4697 /* Should not reset address_spaces when reset because devices will still use 4698 * the address space they got at first (won't ask the bus again). 4699 */ 4700 static void vtd_reset(DeviceState *dev) 4701 { 4702 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev); 4703 4704 vtd_init(s); 4705 vtd_address_space_refresh_all(s); 4706 } 4707 4708 static AddressSpace *vtd_host_dma_iommu(PCIBus *bus, void *opaque, int devfn) 4709 { 4710 IntelIOMMUState *s = opaque; 4711 VTDAddressSpace *vtd_as; 4712 4713 assert(0 <= devfn && devfn < PCI_DEVFN_MAX); 4714 4715 vtd_as = vtd_find_add_as(s, bus, devfn, PCI_NO_PASID); 4716 return &vtd_as->as; 4717 } 4718 4719 static PCIIOMMUOps vtd_iommu_ops = { 4720 .get_address_space = vtd_host_dma_iommu, 4721 .set_iommu_device = vtd_dev_set_iommu_device, 4722 .unset_iommu_device = vtd_dev_unset_iommu_device, 4723 }; 4724 4725 static bool vtd_decide_config(IntelIOMMUState *s, Error **errp) 4726 { 4727 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 4728 4729 if (s->intr_eim == ON_OFF_AUTO_ON && !x86_iommu_ir_supported(x86_iommu)) { 4730 error_setg(errp, "eim=on cannot be selected without intremap=on"); 4731 return false; 4732 } 4733 4734 if (s->intr_eim == ON_OFF_AUTO_AUTO) { 4735 s->intr_eim = (kvm_irqchip_in_kernel() || s->buggy_eim) 4736 && x86_iommu_ir_supported(x86_iommu) ? 4737 ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; 4738 } 4739 if (s->intr_eim == ON_OFF_AUTO_ON && !s->buggy_eim) { 4740 if (kvm_irqchip_is_split() && !kvm_enable_x2apic()) { 4741 error_setg(errp, "eim=on requires support on the KVM side" 4742 "(X2APIC_API, first shipped in v4.7)"); 4743 return false; 4744 } 4745 } 4746 4747 if (!s->scalable_mode && s->flts) { 4748 error_setg(errp, "x-flts is only available in scalable mode"); 4749 return false; 4750 } 4751 4752 if (!s->flts && s->aw_bits != VTD_HOST_AW_39BIT && 4753 s->aw_bits != VTD_HOST_AW_48BIT) { 4754 error_setg(errp, "%s: supported values for aw-bits are: %d, %d", 4755 s->scalable_mode ? "Scalable mode(flts=off)" : "Legacy mode", 4756 VTD_HOST_AW_39BIT, VTD_HOST_AW_48BIT); 4757 return false; 4758 } 4759 4760 if (s->flts && s->aw_bits != VTD_HOST_AW_48BIT) { 4761 error_setg(errp, 4762 "Scalable mode(flts=on): supported value for aw-bits is: %d", 4763 VTD_HOST_AW_48BIT); 4764 return false; 4765 } 4766 4767 if (s->scalable_mode && !s->dma_drain) { 4768 error_setg(errp, "Need to set dma_drain for scalable mode"); 4769 return false; 4770 } 4771 4772 if (s->pasid && !s->scalable_mode) { 4773 error_setg(errp, "Need to set scalable mode for PASID"); 4774 return false; 4775 } 4776 4777 return true; 4778 } 4779 4780 static int vtd_machine_done_notify_one(Object *child, void *unused) 4781 { 4782 IntelIOMMUState *iommu = INTEL_IOMMU_DEVICE(x86_iommu_get_default()); 4783 4784 /* 4785 * We hard-coded here because vfio-pci is the only special case 4786 * here. Let's be more elegant in the future when we can, but so 4787 * far there seems to be no better way. 4788 */ 4789 if (object_dynamic_cast(child, "vfio-pci") && !iommu->caching_mode) { 4790 vtd_panic_require_caching_mode(); 4791 } 4792 4793 return 0; 4794 } 4795 4796 static void vtd_machine_done_hook(Notifier *notifier, void *unused) 4797 { 4798 object_child_foreach_recursive(object_get_root(), 4799 vtd_machine_done_notify_one, NULL); 4800 } 4801 4802 static Notifier vtd_machine_done_notify = { 4803 .notify = vtd_machine_done_hook, 4804 }; 4805 4806 static void vtd_realize(DeviceState *dev, Error **errp) 4807 { 4808 MachineState *ms = MACHINE(qdev_get_machine()); 4809 PCMachineState *pcms = PC_MACHINE(ms); 4810 X86MachineState *x86ms = X86_MACHINE(ms); 4811 PCIBus *bus = pcms->pcibus; 4812 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev); 4813 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 4814 4815 if (s->pasid && x86_iommu->dt_supported) { 4816 /* 4817 * PASID-based-Device-TLB Invalidate Descriptor is not 4818 * implemented and it requires support from vhost layer which 4819 * needs to be implemented in the future. 4820 */ 4821 error_setg(errp, "PASID based device IOTLB is not supported"); 4822 return; 4823 } 4824 4825 if (!vtd_decide_config(s, errp)) { 4826 return; 4827 } 4828 4829 QLIST_INIT(&s->vtd_as_with_notifiers); 4830 qemu_mutex_init(&s->iommu_lock); 4831 memory_region_init_io(&s->csrmem, OBJECT(s), &vtd_mem_ops, s, 4832 "intel_iommu", DMAR_REG_SIZE); 4833 memory_region_add_subregion(get_system_memory(), 4834 Q35_HOST_BRIDGE_IOMMU_ADDR, &s->csrmem); 4835 4836 /* Create the shared memory regions by all devices */ 4837 memory_region_init(&s->mr_nodmar, OBJECT(s), "vtd-nodmar", 4838 UINT64_MAX); 4839 memory_region_init_io(&s->mr_ir, OBJECT(s), &vtd_mem_ir_ops, 4840 s, "vtd-ir", VTD_INTERRUPT_ADDR_SIZE); 4841 memory_region_init_alias(&s->mr_sys_alias, OBJECT(s), 4842 "vtd-sys-alias", get_system_memory(), 0, 4843 memory_region_size(get_system_memory())); 4844 memory_region_add_subregion_overlap(&s->mr_nodmar, 0, 4845 &s->mr_sys_alias, 0); 4846 memory_region_add_subregion_overlap(&s->mr_nodmar, 4847 VTD_INTERRUPT_ADDR_FIRST, 4848 &s->mr_ir, 1); 4849 /* No corresponding destroy */ 4850 s->iotlb = g_hash_table_new_full(vtd_iotlb_hash, vtd_iotlb_equal, 4851 g_free, g_free); 4852 s->vtd_address_spaces = g_hash_table_new_full(vtd_as_hash, vtd_as_equal, 4853 g_free, g_free); 4854 s->vtd_host_iommu_dev = g_hash_table_new_full(vtd_hiod_hash, vtd_hiod_equal, 4855 g_free, vtd_hiod_destroy); 4856 vtd_init(s); 4857 pci_setup_iommu(bus, &vtd_iommu_ops, dev); 4858 /* Pseudo address space under root PCI bus. */ 4859 x86ms->ioapic_as = vtd_host_dma_iommu(bus, s, Q35_PSEUDO_DEVFN_IOAPIC); 4860 qemu_add_machine_init_done_notifier(&vtd_machine_done_notify); 4861 } 4862 4863 static void vtd_class_init(ObjectClass *klass, void *data) 4864 { 4865 DeviceClass *dc = DEVICE_CLASS(klass); 4866 X86IOMMUClass *x86_class = X86_IOMMU_DEVICE_CLASS(klass); 4867 4868 device_class_set_legacy_reset(dc, vtd_reset); 4869 dc->vmsd = &vtd_vmstate; 4870 device_class_set_props(dc, vtd_properties); 4871 dc->hotpluggable = false; 4872 x86_class->realize = vtd_realize; 4873 x86_class->int_remap = vtd_int_remap; 4874 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 4875 dc->desc = "Intel IOMMU (VT-d) DMA Remapping device"; 4876 } 4877 4878 static const TypeInfo vtd_info = { 4879 .name = TYPE_INTEL_IOMMU_DEVICE, 4880 .parent = TYPE_X86_IOMMU_DEVICE, 4881 .instance_size = sizeof(IntelIOMMUState), 4882 .class_init = vtd_class_init, 4883 }; 4884 4885 static void vtd_iommu_memory_region_class_init(ObjectClass *klass, 4886 void *data) 4887 { 4888 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass); 4889 4890 imrc->translate = vtd_iommu_translate; 4891 imrc->notify_flag_changed = vtd_iommu_notify_flag_changed; 4892 imrc->replay = vtd_iommu_replay; 4893 } 4894 4895 static const TypeInfo vtd_iommu_memory_region_info = { 4896 .parent = TYPE_IOMMU_MEMORY_REGION, 4897 .name = TYPE_INTEL_IOMMU_MEMORY_REGION, 4898 .class_init = vtd_iommu_memory_region_class_init, 4899 }; 4900 4901 static void vtd_register_types(void) 4902 { 4903 type_register_static(&vtd_info); 4904 type_register_static(&vtd_iommu_memory_region_info); 4905 } 4906 4907 type_init(vtd_register_types) 4908