1 /* 2 * Copyright (C) 2011 Citrix Ltd. 3 * 4 * This work is licensed under the terms of the GNU GPL, version 2. See 5 * the COPYING file in the top-level directory. 6 * 7 * Contributions after 2012-01-13 are licensed under the terms of the 8 * GNU GPL, version 2 or (at your option) any later version. 9 */ 10 11 #include "qemu/osdep.h" 12 #include "qemu/units.h" 13 #include "qemu/error-report.h" 14 15 #include <sys/resource.h> 16 17 #include "hw/xen/xen-hvm-common.h" 18 #include "hw/xen/xen_native.h" 19 #include "qemu/bitmap.h" 20 21 #include "system/runstate.h" 22 #include "system/xen-mapcache.h" 23 #include "trace.h" 24 25 #include <xenevtchn.h> 26 #include <xengnttab.h> 27 28 #if HOST_LONG_BITS == 32 29 # define MCACHE_MAX_SIZE (1UL<<31) /* 2GB Cap */ 30 #else 31 # define MCACHE_MAX_SIZE (1UL<<35) /* 32GB Cap */ 32 #endif 33 34 /* This is the size of the virtual address space reserve to QEMU that will not 35 * be use by MapCache. 36 * From empirical tests I observed that qemu use 75MB more than the 37 * max_mcache_size. 38 */ 39 #define NON_MCACHE_MEMORY_SIZE (80 * MiB) 40 41 typedef struct MapCacheEntry { 42 hwaddr paddr_index; 43 uint8_t *vaddr_base; 44 unsigned long *valid_mapping; 45 uint32_t lock; 46 #define XEN_MAPCACHE_ENTRY_DUMMY (1 << 0) 47 #define XEN_MAPCACHE_ENTRY_GRANT (1 << 1) 48 uint8_t flags; 49 hwaddr size; 50 struct MapCacheEntry *next; 51 } MapCacheEntry; 52 53 typedef struct MapCacheRev { 54 uint8_t *vaddr_req; 55 hwaddr paddr_index; 56 hwaddr size; 57 QTAILQ_ENTRY(MapCacheRev) next; 58 bool dma; 59 } MapCacheRev; 60 61 typedef struct MapCache { 62 MapCacheEntry *entry; 63 unsigned long nr_buckets; 64 QTAILQ_HEAD(, MapCacheRev) locked_entries; 65 66 /* For most cases (>99.9%), the page address is the same. */ 67 MapCacheEntry *last_entry; 68 unsigned long max_mcache_size; 69 unsigned int bucket_shift; 70 unsigned long bucket_size; 71 72 phys_offset_to_gaddr_t phys_offset_to_gaddr; 73 QemuMutex lock; 74 void *opaque; 75 } MapCache; 76 77 static MapCache *mapcache; 78 static MapCache *mapcache_grants_ro; 79 static MapCache *mapcache_grants_rw; 80 static xengnttab_handle *xen_region_gnttabdev; 81 82 static inline void mapcache_lock(MapCache *mc) 83 { 84 qemu_mutex_lock(&mc->lock); 85 } 86 87 static inline void mapcache_unlock(MapCache *mc) 88 { 89 qemu_mutex_unlock(&mc->lock); 90 } 91 92 static inline int test_bits(int nr, int size, const unsigned long *addr) 93 { 94 unsigned long res = find_next_zero_bit(addr, size + nr, nr); 95 if (res >= nr + size) 96 return 1; 97 else 98 return 0; 99 } 100 101 static MapCache *xen_map_cache_init_single(phys_offset_to_gaddr_t f, 102 void *opaque, 103 unsigned int bucket_shift, 104 unsigned long max_size) 105 { 106 unsigned long size; 107 MapCache *mc; 108 109 assert(bucket_shift >= XC_PAGE_SHIFT); 110 111 mc = g_new0(MapCache, 1); 112 113 mc->phys_offset_to_gaddr = f; 114 mc->opaque = opaque; 115 qemu_mutex_init(&mc->lock); 116 117 QTAILQ_INIT(&mc->locked_entries); 118 119 mc->bucket_shift = bucket_shift; 120 mc->bucket_size = 1UL << bucket_shift; 121 mc->max_mcache_size = max_size; 122 123 mc->nr_buckets = 124 (((mc->max_mcache_size >> XC_PAGE_SHIFT) + 125 (1UL << (bucket_shift - XC_PAGE_SHIFT)) - 1) >> 126 (bucket_shift - XC_PAGE_SHIFT)); 127 128 size = mc->nr_buckets * sizeof(MapCacheEntry); 129 size = (size + XC_PAGE_SIZE - 1) & ~(XC_PAGE_SIZE - 1); 130 trace_xen_map_cache_init(mc->nr_buckets, size); 131 mc->entry = g_malloc0(size); 132 return mc; 133 } 134 135 void xen_map_cache_init(phys_offset_to_gaddr_t f, void *opaque) 136 { 137 struct rlimit rlimit_as; 138 unsigned long max_mcache_size; 139 unsigned int bucket_shift; 140 141 xen_region_gnttabdev = xengnttab_open(NULL, 0); 142 if (xen_region_gnttabdev == NULL) { 143 error_report("mapcache: Failed to open gnttab device"); 144 exit(EXIT_FAILURE); 145 } 146 147 if (HOST_LONG_BITS == 32) { 148 bucket_shift = 16; 149 } else { 150 bucket_shift = 20; 151 } 152 153 if (geteuid() == 0) { 154 rlimit_as.rlim_cur = RLIM_INFINITY; 155 rlimit_as.rlim_max = RLIM_INFINITY; 156 max_mcache_size = MCACHE_MAX_SIZE; 157 } else { 158 getrlimit(RLIMIT_AS, &rlimit_as); 159 rlimit_as.rlim_cur = rlimit_as.rlim_max; 160 161 if (rlimit_as.rlim_max != RLIM_INFINITY) { 162 warn_report("QEMU's maximum size of virtual" 163 " memory is not infinity"); 164 } 165 if (rlimit_as.rlim_max < MCACHE_MAX_SIZE + NON_MCACHE_MEMORY_SIZE) { 166 max_mcache_size = rlimit_as.rlim_max - NON_MCACHE_MEMORY_SIZE; 167 } else { 168 max_mcache_size = MCACHE_MAX_SIZE; 169 } 170 } 171 172 mapcache = xen_map_cache_init_single(f, opaque, 173 bucket_shift, 174 max_mcache_size); 175 176 /* 177 * Grant mappings must use XC_PAGE_SIZE granularity since we can't 178 * map anything beyond the number of pages granted to us. 179 */ 180 mapcache_grants_ro = xen_map_cache_init_single(f, opaque, 181 XC_PAGE_SHIFT, 182 max_mcache_size); 183 mapcache_grants_rw = xen_map_cache_init_single(f, opaque, 184 XC_PAGE_SHIFT, 185 max_mcache_size); 186 187 setrlimit(RLIMIT_AS, &rlimit_as); 188 } 189 190 static void xen_remap_bucket(MapCache *mc, 191 MapCacheEntry *entry, 192 void *vaddr, 193 hwaddr size, 194 hwaddr address_index, 195 bool dummy, 196 bool grant, 197 bool is_write, 198 ram_addr_t ram_offset) 199 { 200 uint8_t *vaddr_base; 201 g_autofree uint32_t *refs = NULL; 202 g_autofree xen_pfn_t *pfns = NULL; 203 g_autofree int *err; 204 unsigned int i; 205 hwaddr nb_pfn = size >> XC_PAGE_SHIFT; 206 207 trace_xen_remap_bucket(address_index); 208 209 if (grant) { 210 refs = g_new0(uint32_t, nb_pfn); 211 } else { 212 pfns = g_new0(xen_pfn_t, nb_pfn); 213 } 214 err = g_new0(int, nb_pfn); 215 216 if (entry->vaddr_base != NULL) { 217 if (!(entry->flags & XEN_MAPCACHE_ENTRY_DUMMY)) { 218 ram_block_notify_remove(entry->vaddr_base, entry->size, 219 entry->size); 220 } 221 222 /* 223 * If an entry is being replaced by another mapping and we're using 224 * MAP_FIXED flag for it - there is possibility of a race for vaddr 225 * address with another thread doing an mmap call itself 226 * (see man 2 mmap). To avoid that we skip explicit unmapping here 227 * and allow the kernel to destroy the previous mappings by replacing 228 * them in mmap call later. 229 * 230 * Non-identical replacements are not allowed therefore. 231 */ 232 assert(!vaddr || (entry->vaddr_base == vaddr && entry->size == size)); 233 234 if (!vaddr && munmap(entry->vaddr_base, entry->size) != 0) { 235 perror("unmap fails"); 236 exit(-1); 237 } 238 } 239 g_free(entry->valid_mapping); 240 entry->valid_mapping = NULL; 241 242 if (grant) { 243 hwaddr grant_base = address_index - (ram_offset >> XC_PAGE_SHIFT); 244 245 for (i = 0; i < nb_pfn; i++) { 246 refs[i] = grant_base + i; 247 } 248 } else { 249 for (i = 0; i < nb_pfn; i++) { 250 pfns[i] = (address_index << (mc->bucket_shift - XC_PAGE_SHIFT)) + i; 251 } 252 } 253 254 entry->flags &= ~XEN_MAPCACHE_ENTRY_GRANT; 255 256 if (!dummy) { 257 if (grant) { 258 int prot = PROT_READ; 259 260 if (is_write) { 261 prot |= PROT_WRITE; 262 } 263 264 entry->flags |= XEN_MAPCACHE_ENTRY_GRANT; 265 assert(vaddr == NULL); 266 vaddr_base = xengnttab_map_domain_grant_refs(xen_region_gnttabdev, 267 nb_pfn, 268 xen_domid, refs, 269 prot); 270 } else { 271 /* 272 * If the caller has requested the mapping at a specific address use 273 * MAP_FIXED to make sure it's honored. 274 * 275 * We don't yet support upgrading mappings from RO to RW, to handle 276 * models using ordinary address_space_rw(), foreign mappings ignore 277 * is_write and are always mapped RW. 278 */ 279 vaddr_base = xenforeignmemory_map2(xen_fmem, xen_domid, vaddr, 280 PROT_READ | PROT_WRITE, 281 vaddr ? MAP_FIXED : 0, 282 nb_pfn, pfns, err); 283 } 284 if (vaddr_base == NULL) { 285 perror(grant ? "xengnttab_map_domain_grant_refs" 286 : "xenforeignmemory_map2"); 287 exit(-1); 288 } 289 } else { 290 /* 291 * We create dummy mappings where we are unable to create a foreign 292 * mapping immediately due to certain circumstances (i.e. on resume now) 293 */ 294 vaddr_base = mmap(vaddr, size, PROT_READ | PROT_WRITE, 295 MAP_ANON | MAP_SHARED | (vaddr ? MAP_FIXED : 0), 296 -1, 0); 297 if (vaddr_base == MAP_FAILED) { 298 perror("mmap"); 299 exit(-1); 300 } 301 } 302 303 if (!(entry->flags & XEN_MAPCACHE_ENTRY_DUMMY)) { 304 ram_block_notify_add(vaddr_base, size, size); 305 } 306 307 entry->vaddr_base = vaddr_base; 308 entry->paddr_index = address_index; 309 entry->size = size; 310 entry->valid_mapping = g_new0(unsigned long, 311 BITS_TO_LONGS(size >> XC_PAGE_SHIFT)); 312 313 if (dummy) { 314 entry->flags |= XEN_MAPCACHE_ENTRY_DUMMY; 315 } else { 316 entry->flags &= ~(XEN_MAPCACHE_ENTRY_DUMMY); 317 } 318 319 bitmap_zero(entry->valid_mapping, nb_pfn); 320 for (i = 0; i < nb_pfn; i++) { 321 if (!err[i]) { 322 bitmap_set(entry->valid_mapping, i, 1); 323 } 324 } 325 } 326 327 static uint8_t *xen_map_cache_unlocked(MapCache *mc, 328 hwaddr phys_addr, hwaddr size, 329 ram_addr_t ram_offset, 330 uint8_t lock, bool dma, 331 bool grant, bool is_write) 332 { 333 MapCacheEntry *entry, *pentry = NULL, 334 *free_entry = NULL, *free_pentry = NULL; 335 hwaddr address_index; 336 hwaddr address_offset; 337 hwaddr cache_size = size; 338 hwaddr test_bit_size; 339 bool translated G_GNUC_UNUSED = false; 340 bool dummy = false; 341 342 tryagain: 343 address_index = phys_addr >> mc->bucket_shift; 344 address_offset = phys_addr & (mc->bucket_size - 1); 345 346 trace_xen_map_cache(phys_addr); 347 348 /* test_bit_size is always a multiple of XC_PAGE_SIZE */ 349 if (size) { 350 test_bit_size = size + (phys_addr & (XC_PAGE_SIZE - 1)); 351 352 if (test_bit_size % XC_PAGE_SIZE) { 353 test_bit_size += XC_PAGE_SIZE - (test_bit_size % XC_PAGE_SIZE); 354 } 355 } else { 356 test_bit_size = XC_PAGE_SIZE; 357 } 358 359 if (mc->last_entry != NULL && 360 mc->last_entry->paddr_index == address_index && 361 !lock && !size && 362 test_bits(address_offset >> XC_PAGE_SHIFT, 363 test_bit_size >> XC_PAGE_SHIFT, 364 mc->last_entry->valid_mapping)) { 365 trace_xen_map_cache_return( 366 mc->last_entry->vaddr_base + address_offset 367 ); 368 return mc->last_entry->vaddr_base + address_offset; 369 } 370 371 /* size is always a multiple of mc->bucket_size */ 372 if (size) { 373 cache_size = size + address_offset; 374 if (cache_size % mc->bucket_size) { 375 cache_size += mc->bucket_size - (cache_size % mc->bucket_size); 376 } 377 } else { 378 cache_size = mc->bucket_size; 379 } 380 381 entry = &mc->entry[address_index % mc->nr_buckets]; 382 383 while (entry && (!entry->vaddr_base || 384 entry->paddr_index != address_index || entry->size != cache_size || 385 !test_bits(address_offset >> XC_PAGE_SHIFT, 386 test_bit_size >> XC_PAGE_SHIFT, 387 entry->valid_mapping))) { 388 if (!free_entry && (!entry->lock || !entry->vaddr_base)) { 389 free_entry = entry; 390 free_pentry = pentry; 391 } 392 pentry = entry; 393 entry = entry->next; 394 } 395 if (!entry && free_entry) { 396 entry = free_entry; 397 pentry = free_pentry; 398 } 399 if (!entry) { 400 entry = g_new0(MapCacheEntry, 1); 401 pentry->next = entry; 402 xen_remap_bucket(mc, entry, NULL, cache_size, address_index, dummy, 403 grant, is_write, ram_offset); 404 } else if (!entry->lock) { 405 if (!entry->vaddr_base || entry->paddr_index != address_index || 406 entry->size != cache_size || 407 !test_bits(address_offset >> XC_PAGE_SHIFT, 408 test_bit_size >> XC_PAGE_SHIFT, 409 entry->valid_mapping)) { 410 xen_remap_bucket(mc, entry, NULL, cache_size, address_index, dummy, 411 grant, is_write, ram_offset); 412 } 413 } 414 415 if(!test_bits(address_offset >> XC_PAGE_SHIFT, 416 test_bit_size >> XC_PAGE_SHIFT, 417 entry->valid_mapping)) { 418 mc->last_entry = NULL; 419 #ifdef XEN_COMPAT_PHYSMAP 420 if (!translated && mc->phys_offset_to_gaddr) { 421 phys_addr = mc->phys_offset_to_gaddr(phys_addr, size); 422 translated = true; 423 goto tryagain; 424 } 425 #endif 426 if (!dummy && runstate_check(RUN_STATE_INMIGRATE)) { 427 dummy = true; 428 goto tryagain; 429 } 430 trace_xen_map_cache_return(NULL); 431 return NULL; 432 } 433 434 mc->last_entry = entry; 435 if (lock) { 436 MapCacheRev *reventry = g_new0(MapCacheRev, 1); 437 entry->lock++; 438 if (entry->lock == 0) { 439 error_report("mapcache entry lock overflow: "HWADDR_FMT_plx" -> %p", 440 entry->paddr_index, entry->vaddr_base); 441 abort(); 442 } 443 reventry->dma = dma; 444 reventry->vaddr_req = mc->last_entry->vaddr_base + address_offset; 445 reventry->paddr_index = mc->last_entry->paddr_index; 446 reventry->size = entry->size; 447 QTAILQ_INSERT_HEAD(&mc->locked_entries, reventry, next); 448 } 449 450 trace_xen_map_cache_return( 451 mc->last_entry->vaddr_base + address_offset 452 ); 453 return mc->last_entry->vaddr_base + address_offset; 454 } 455 456 uint8_t *xen_map_cache(MemoryRegion *mr, 457 hwaddr phys_addr, hwaddr size, 458 ram_addr_t ram_addr_offset, 459 uint8_t lock, bool dma, 460 bool is_write) 461 { 462 bool grant = xen_mr_is_grants(mr); 463 MapCache *mc = mapcache; 464 uint8_t *p; 465 466 if (grant) { 467 mc = is_write ? mapcache_grants_rw : mapcache_grants_ro; 468 } 469 470 if (grant && !lock) { 471 /* 472 * Grants are only supported via address_space_map(). Anything 473 * else is considered a user/guest error. 474 * 475 * QEMU generally doesn't expect these mappings to ever fail, so 476 * if this happens we report an error message and abort(). 477 */ 478 error_report("Tried to access a grant reference without mapping it."); 479 abort(); 480 } 481 482 mapcache_lock(mc); 483 p = xen_map_cache_unlocked(mc, phys_addr, size, ram_addr_offset, 484 lock, dma, grant, is_write); 485 mapcache_unlock(mc); 486 return p; 487 } 488 489 static ram_addr_t xen_ram_addr_from_mapcache_single(MapCache *mc, void *ptr) 490 { 491 MapCacheEntry *entry = NULL; 492 MapCacheRev *reventry; 493 hwaddr paddr_index; 494 hwaddr size; 495 ram_addr_t raddr; 496 int found = 0; 497 498 mapcache_lock(mc); 499 QTAILQ_FOREACH(reventry, &mc->locked_entries, next) { 500 if (reventry->vaddr_req == ptr) { 501 paddr_index = reventry->paddr_index; 502 size = reventry->size; 503 found = 1; 504 break; 505 } 506 } 507 if (!found) { 508 trace_xen_ram_addr_from_mapcache_not_found(ptr); 509 mapcache_unlock(mc); 510 return RAM_ADDR_INVALID; 511 } 512 513 entry = &mc->entry[paddr_index % mc->nr_buckets]; 514 while (entry && (entry->paddr_index != paddr_index || entry->size != size)) { 515 entry = entry->next; 516 } 517 if (!entry) { 518 trace_xen_ram_addr_from_mapcache_not_in_cache(ptr); 519 raddr = RAM_ADDR_INVALID; 520 } else { 521 raddr = (reventry->paddr_index << mc->bucket_shift) + 522 ((unsigned long) ptr - (unsigned long) entry->vaddr_base); 523 } 524 mapcache_unlock(mc); 525 return raddr; 526 } 527 528 ram_addr_t xen_ram_addr_from_mapcache(void *ptr) 529 { 530 ram_addr_t addr; 531 532 addr = xen_ram_addr_from_mapcache_single(mapcache, ptr); 533 if (addr == RAM_ADDR_INVALID) { 534 addr = xen_ram_addr_from_mapcache_single(mapcache_grants_ro, ptr); 535 } 536 if (addr == RAM_ADDR_INVALID) { 537 addr = xen_ram_addr_from_mapcache_single(mapcache_grants_rw, ptr); 538 } 539 540 return addr; 541 } 542 543 static void xen_invalidate_map_cache_entry_unlocked(MapCache *mc, 544 uint8_t *buffer) 545 { 546 MapCacheEntry *entry = NULL, *pentry = NULL; 547 MapCacheRev *reventry; 548 hwaddr paddr_index; 549 hwaddr size; 550 int found = 0; 551 int rc; 552 553 QTAILQ_FOREACH(reventry, &mc->locked_entries, next) { 554 if (reventry->vaddr_req == buffer) { 555 paddr_index = reventry->paddr_index; 556 size = reventry->size; 557 found = 1; 558 break; 559 } 560 } 561 if (!found) { 562 trace_xen_invalidate_map_cache_entry_unlocked_not_found(buffer); 563 QTAILQ_FOREACH(reventry, &mc->locked_entries, next) { 564 trace_xen_invalidate_map_cache_entry_unlocked_found( 565 reventry->paddr_index, 566 reventry->vaddr_req 567 ); 568 } 569 return; 570 } 571 QTAILQ_REMOVE(&mc->locked_entries, reventry, next); 572 g_free(reventry); 573 574 if (mc->last_entry != NULL && 575 mc->last_entry->paddr_index == paddr_index) { 576 mc->last_entry = NULL; 577 } 578 579 entry = &mc->entry[paddr_index % mc->nr_buckets]; 580 while (entry && (entry->paddr_index != paddr_index || entry->size != size)) { 581 pentry = entry; 582 entry = entry->next; 583 } 584 if (!entry) { 585 trace_xen_invalidate_map_cache_entry_unlocked_miss(buffer); 586 return; 587 } 588 entry->lock--; 589 if (entry->lock > 0) { 590 return; 591 } 592 593 ram_block_notify_remove(entry->vaddr_base, entry->size, entry->size); 594 if (entry->flags & XEN_MAPCACHE_ENTRY_GRANT) { 595 rc = xengnttab_unmap(xen_region_gnttabdev, entry->vaddr_base, 596 entry->size >> mc->bucket_shift); 597 } else { 598 rc = munmap(entry->vaddr_base, entry->size); 599 } 600 601 if (rc) { 602 perror("unmap fails"); 603 exit(-1); 604 } 605 606 g_free(entry->valid_mapping); 607 if (pentry) { 608 pentry->next = entry->next; 609 g_free(entry); 610 } else { 611 /* 612 * Invalidate mapping but keep entry->next pointing to the rest 613 * of the list. 614 * 615 * Note that lock is already zero here, otherwise we don't unmap. 616 */ 617 entry->paddr_index = 0; 618 entry->vaddr_base = NULL; 619 entry->valid_mapping = NULL; 620 entry->flags = 0; 621 entry->size = 0; 622 } 623 } 624 625 typedef struct XenMapCacheData { 626 Coroutine *co; 627 uint8_t *buffer; 628 } XenMapCacheData; 629 630 static void xen_invalidate_map_cache_entry_single(MapCache *mc, uint8_t *buffer) 631 { 632 mapcache_lock(mc); 633 xen_invalidate_map_cache_entry_unlocked(mc, buffer); 634 mapcache_unlock(mc); 635 } 636 637 static void xen_invalidate_map_cache_entry_all(uint8_t *buffer) 638 { 639 xen_invalidate_map_cache_entry_single(mapcache, buffer); 640 xen_invalidate_map_cache_entry_single(mapcache_grants_ro, buffer); 641 xen_invalidate_map_cache_entry_single(mapcache_grants_rw, buffer); 642 } 643 644 static void xen_invalidate_map_cache_entry_bh(void *opaque) 645 { 646 XenMapCacheData *data = opaque; 647 648 xen_invalidate_map_cache_entry_all(data->buffer); 649 aio_co_wake(data->co); 650 } 651 652 void coroutine_mixed_fn xen_invalidate_map_cache_entry(uint8_t *buffer) 653 { 654 if (qemu_in_coroutine()) { 655 XenMapCacheData data = { 656 .co = qemu_coroutine_self(), 657 .buffer = buffer, 658 }; 659 aio_bh_schedule_oneshot(qemu_get_current_aio_context(), 660 xen_invalidate_map_cache_entry_bh, &data); 661 qemu_coroutine_yield(); 662 } else { 663 xen_invalidate_map_cache_entry_all(buffer); 664 } 665 } 666 667 static void xen_invalidate_map_cache_single(MapCache *mc) 668 { 669 unsigned long i; 670 MapCacheRev *reventry; 671 672 mapcache_lock(mc); 673 674 QTAILQ_FOREACH(reventry, &mc->locked_entries, next) { 675 if (!reventry->dma) { 676 continue; 677 } 678 trace_xen_invalidate_map_cache(reventry->paddr_index, 679 reventry->vaddr_req); 680 } 681 682 for (i = 0; i < mc->nr_buckets; i++) { 683 MapCacheEntry *entry = &mc->entry[i]; 684 685 if (entry->vaddr_base == NULL) { 686 continue; 687 } 688 if (entry->lock > 0) { 689 continue; 690 } 691 692 if (munmap(entry->vaddr_base, entry->size) != 0) { 693 perror("unmap fails"); 694 exit(-1); 695 } 696 697 entry->paddr_index = 0; 698 entry->vaddr_base = NULL; 699 entry->size = 0; 700 g_free(entry->valid_mapping); 701 entry->valid_mapping = NULL; 702 } 703 704 mc->last_entry = NULL; 705 706 mapcache_unlock(mc); 707 } 708 709 void xen_invalidate_map_cache(void) 710 { 711 /* Flush pending AIO before destroying the mapcache */ 712 bdrv_drain_all(); 713 714 xen_invalidate_map_cache_single(mapcache); 715 } 716 717 static uint8_t *xen_replace_cache_entry_unlocked(MapCache *mc, 718 hwaddr old_phys_addr, 719 hwaddr new_phys_addr, 720 hwaddr size) 721 { 722 MapCacheEntry *entry; 723 hwaddr address_index, address_offset; 724 hwaddr test_bit_size, cache_size = size; 725 726 address_index = old_phys_addr >> mc->bucket_shift; 727 address_offset = old_phys_addr & (mc->bucket_size - 1); 728 729 assert(size); 730 /* test_bit_size is always a multiple of XC_PAGE_SIZE */ 731 test_bit_size = size + (old_phys_addr & (XC_PAGE_SIZE - 1)); 732 if (test_bit_size % XC_PAGE_SIZE) { 733 test_bit_size += XC_PAGE_SIZE - (test_bit_size % XC_PAGE_SIZE); 734 } 735 cache_size = size + address_offset; 736 if (cache_size % mc->bucket_size) { 737 cache_size += mc->bucket_size - (cache_size % mc->bucket_size); 738 } 739 740 entry = &mc->entry[address_index % mc->nr_buckets]; 741 while (entry && !(entry->paddr_index == address_index && 742 entry->size == cache_size)) { 743 entry = entry->next; 744 } 745 if (!entry) { 746 trace_xen_replace_cache_entry_unlocked(old_phys_addr); 747 return NULL; 748 } 749 750 assert((entry->flags & XEN_MAPCACHE_ENTRY_GRANT) == 0); 751 752 address_index = new_phys_addr >> mc->bucket_shift; 753 address_offset = new_phys_addr & (mc->bucket_size - 1); 754 755 trace_xen_replace_cache_entry_dummy(old_phys_addr, new_phys_addr); 756 757 xen_remap_bucket(mc, entry, entry->vaddr_base, 758 cache_size, address_index, false, 759 false, false, old_phys_addr); 760 if (!test_bits(address_offset >> XC_PAGE_SHIFT, 761 test_bit_size >> XC_PAGE_SHIFT, 762 entry->valid_mapping)) { 763 trace_xen_replace_cache_entry_unlocked_could_not_update_entry( 764 old_phys_addr 765 ); 766 return NULL; 767 } 768 769 return entry->vaddr_base + address_offset; 770 } 771 772 uint8_t *xen_replace_cache_entry(hwaddr old_phys_addr, 773 hwaddr new_phys_addr, 774 hwaddr size) 775 { 776 uint8_t *p; 777 778 mapcache_lock(mapcache); 779 p = xen_replace_cache_entry_unlocked(mapcache, old_phys_addr, 780 new_phys_addr, size); 781 mapcache_unlock(mapcache); 782 return p; 783 } 784