Lines Matching full:sg
252 struct gmap *sg, *next; in gmap_remove() local
258 list_for_each_entry_safe(sg, next, &gmap->children, list) { in gmap_remove()
259 list_del(&sg->list); in gmap_remove()
260 gmap_put(sg); in gmap_remove()
973 * Expected to be called with sg->mm->mmap_lock in read and
1019 * Expected to be called with sg->mm->mmap_lock in read
1055 * Called with sg->mm->mmap_lock in read.
1188 * @sg: pointer to the shadow guest address space structure
1192 * Called with the sg->guest_table_lock
1194 static inline void gmap_insert_rmap(struct gmap *sg, unsigned long vmaddr, in gmap_insert_rmap() argument
1200 BUG_ON(!gmap_is_shadow(sg)); in gmap_insert_rmap()
1201 slot = radix_tree_lookup_slot(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT); in gmap_insert_rmap()
1204 &sg->guest_table_lock); in gmap_insert_rmap()
1211 radix_tree_replace_slot(&sg->host_to_rmap, slot, rmap); in gmap_insert_rmap()
1214 radix_tree_insert(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT, in gmap_insert_rmap()
1221 * @sg: pointer to the shadow guest address space structure
1229 static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr, in gmap_protect_rmap() argument
1239 BUG_ON(!gmap_is_shadow(sg)); in gmap_protect_rmap()
1240 parent = sg->parent; in gmap_protect_rmap()
1257 spin_lock(&sg->guest_table_lock); in gmap_protect_rmap()
1261 gmap_insert_rmap(sg, vmaddr, rmap); in gmap_protect_rmap()
1262 spin_unlock(&sg->guest_table_lock); in gmap_protect_rmap()
1305 * @sg: pointer to the shadow guest address space structure
1308 * Called with the sg->guest_table_lock
1310 static void gmap_unshadow_page(struct gmap *sg, unsigned long raddr) in gmap_unshadow_page() argument
1314 BUG_ON(!gmap_is_shadow(sg)); in gmap_unshadow_page()
1315 table = gmap_table_walk(sg, raddr, 0); /* get page table pointer */ in gmap_unshadow_page()
1318 gmap_call_notifier(sg, raddr, raddr + _PAGE_SIZE - 1); in gmap_unshadow_page()
1319 ptep_unshadow_pte(sg->mm, raddr, (pte_t *) table); in gmap_unshadow_page()
1324 * @sg: pointer to the shadow guest address space structure
1328 * Called with the sg->guest_table_lock
1330 static void __gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_pgt() argument
1335 BUG_ON(!gmap_is_shadow(sg)); in __gmap_unshadow_pgt()
1342 * @sg: pointer to the shadow guest address space structure
1345 * Called with the sg->guest_table_lock
1347 static void gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr) in gmap_unshadow_pgt() argument
1353 BUG_ON(!gmap_is_shadow(sg)); in gmap_unshadow_pgt()
1354 ste = gmap_table_walk(sg, raddr, 1); /* get segment pointer */ in gmap_unshadow_pgt()
1357 gmap_call_notifier(sg, raddr, raddr + _SEGMENT_SIZE - 1); in gmap_unshadow_pgt()
1362 __gmap_unshadow_pgt(sg, raddr, __va(pgt)); in gmap_unshadow_pgt()
1371 * @sg: pointer to the shadow guest address space structure
1375 * Called with the sg->guest_table_lock
1377 static void __gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_sgt() argument
1384 BUG_ON(!gmap_is_shadow(sg)); in __gmap_unshadow_sgt()
1390 __gmap_unshadow_pgt(sg, raddr, __va(pgt)); in __gmap_unshadow_sgt()
1400 * @sg: pointer to the shadow guest address space structure
1405 static void gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr) in gmap_unshadow_sgt() argument
1411 BUG_ON(!gmap_is_shadow(sg)); in gmap_unshadow_sgt()
1412 r3e = gmap_table_walk(sg, raddr, 2); /* get region-3 pointer */ in gmap_unshadow_sgt()
1415 gmap_call_notifier(sg, raddr, raddr + _REGION3_SIZE - 1); in gmap_unshadow_sgt()
1420 __gmap_unshadow_sgt(sg, raddr, __va(sgt)); in gmap_unshadow_sgt()
1429 * @sg: pointer to the shadow guest address space structure
1433 * Called with the sg->guest_table_lock
1435 static void __gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_r3t() argument
1442 BUG_ON(!gmap_is_shadow(sg)); in __gmap_unshadow_r3t()
1448 __gmap_unshadow_sgt(sg, raddr, __va(sgt)); in __gmap_unshadow_r3t()
1458 * @sg: pointer to the shadow guest address space structure
1461 * Called with the sg->guest_table_lock
1463 static void gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr) in gmap_unshadow_r3t() argument
1469 BUG_ON(!gmap_is_shadow(sg)); in gmap_unshadow_r3t()
1470 r2e = gmap_table_walk(sg, raddr, 3); /* get region-2 pointer */ in gmap_unshadow_r3t()
1473 gmap_call_notifier(sg, raddr, raddr + _REGION2_SIZE - 1); in gmap_unshadow_r3t()
1478 __gmap_unshadow_r3t(sg, raddr, __va(r3t)); in gmap_unshadow_r3t()
1487 * @sg: pointer to the shadow guest address space structure
1491 * Called with the sg->guest_table_lock
1493 static void __gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_r2t() argument
1500 BUG_ON(!gmap_is_shadow(sg)); in __gmap_unshadow_r2t()
1506 __gmap_unshadow_r3t(sg, raddr, __va(r3t)); in __gmap_unshadow_r2t()
1516 * @sg: pointer to the shadow guest address space structure
1519 * Called with the sg->guest_table_lock
1521 static void gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr) in gmap_unshadow_r2t() argument
1527 BUG_ON(!gmap_is_shadow(sg)); in gmap_unshadow_r2t()
1528 r1e = gmap_table_walk(sg, raddr, 4); /* get region-1 pointer */ in gmap_unshadow_r2t()
1531 gmap_call_notifier(sg, raddr, raddr + _REGION1_SIZE - 1); in gmap_unshadow_r2t()
1536 __gmap_unshadow_r2t(sg, raddr, __va(r2t)); in gmap_unshadow_r2t()
1545 * @sg: pointer to the shadow guest address space structure
1551 static void __gmap_unshadow_r1t(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_r1t() argument
1559 BUG_ON(!gmap_is_shadow(sg)); in __gmap_unshadow_r1t()
1565 __gmap_unshadow_r2t(sg, raddr, __va(r2t)); in __gmap_unshadow_r1t()
1578 * @sg: pointer to the shadow guest address space structure
1580 * Called with sg->guest_table_lock
1582 static void gmap_unshadow(struct gmap *sg) in gmap_unshadow() argument
1586 BUG_ON(!gmap_is_shadow(sg)); in gmap_unshadow()
1587 if (sg->removed) in gmap_unshadow()
1589 sg->removed = 1; in gmap_unshadow()
1590 gmap_call_notifier(sg, 0, -1UL); in gmap_unshadow()
1591 gmap_flush_tlb(sg); in gmap_unshadow()
1592 table = __va(sg->asce & _ASCE_ORIGIN); in gmap_unshadow()
1593 switch (sg->asce & _ASCE_TYPE_MASK) { in gmap_unshadow()
1595 __gmap_unshadow_r1t(sg, 0, table); in gmap_unshadow()
1598 __gmap_unshadow_r2t(sg, 0, table); in gmap_unshadow()
1601 __gmap_unshadow_r3t(sg, 0, table); in gmap_unshadow()
1604 __gmap_unshadow_sgt(sg, 0, table); in gmap_unshadow()
1622 struct gmap *sg; in gmap_find_shadow() local
1624 list_for_each_entry(sg, &parent->children, list) { in gmap_find_shadow()
1625 if (sg->orig_asce != asce || sg->edat_level != edat_level || in gmap_find_shadow()
1626 sg->removed) in gmap_find_shadow()
1628 if (!sg->initialized) in gmap_find_shadow()
1630 refcount_inc(&sg->ref_count); in gmap_find_shadow()
1631 return sg; in gmap_find_shadow()
1639 * @sg: pointer to the shadow guest address space structure
1648 int gmap_shadow_valid(struct gmap *sg, unsigned long asce, int edat_level) in gmap_shadow_valid() argument
1650 if (sg->removed) in gmap_shadow_valid()
1652 return sg->orig_asce == asce && sg->edat_level == edat_level; in gmap_shadow_valid()
1674 struct gmap *sg, *new; in gmap_shadow() local
1681 sg = gmap_find_shadow(parent, asce, edat_level); in gmap_shadow()
1683 if (sg) in gmap_shadow()
1684 return sg; in gmap_shadow()
1700 sg = gmap_find_shadow(parent, asce, edat_level); in gmap_shadow()
1701 if (sg) { in gmap_shadow()
1704 return sg; in gmap_shadow()
1708 list_for_each_entry(sg, &parent->children, list) { in gmap_shadow()
1709 if (sg->orig_asce & _ASCE_REAL_SPACE) { in gmap_shadow()
1710 spin_lock(&sg->guest_table_lock); in gmap_shadow()
1711 gmap_unshadow(sg); in gmap_shadow()
1712 spin_unlock(&sg->guest_table_lock); in gmap_shadow()
1713 list_del(&sg->list); in gmap_shadow()
1714 gmap_put(sg); in gmap_shadow()
1748 * @sg: pointer to the shadow guest address space structure
1762 * Called with sg->mm->mmap_lock in read.
1764 int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t, in gmap_shadow_r2t() argument
1773 BUG_ON(!gmap_is_shadow(sg)); in gmap_shadow_r2t()
1783 spin_lock(&sg->guest_table_lock); in gmap_shadow_r2t()
1784 table = gmap_table_walk(sg, saddr, 4); /* get region-1 pointer */ in gmap_shadow_r2t()
1800 if (sg->edat_level >= 1) in gmap_shadow_r2t()
1802 list_add(&page->lru, &sg->crst_list); in gmap_shadow_r2t()
1806 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r2t()
1809 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r2t()
1815 rc = gmap_protect_rmap(sg, raddr, origin + offset, len); in gmap_shadow_r2t()
1816 spin_lock(&sg->guest_table_lock); in gmap_shadow_r2t()
1818 table = gmap_table_walk(sg, saddr, 4); in gmap_shadow_r2t()
1824 gmap_unshadow_r2t(sg, raddr); in gmap_shadow_r2t()
1826 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r2t()
1829 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r2t()
1837 * @sg: pointer to the shadow guest address space structure
1846 * Called with sg->mm->mmap_lock in read.
1848 int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t, in gmap_shadow_r3t() argument
1857 BUG_ON(!gmap_is_shadow(sg)); in gmap_shadow_r3t()
1867 spin_lock(&sg->guest_table_lock); in gmap_shadow_r3t()
1868 table = gmap_table_walk(sg, saddr, 3); /* get region-2 pointer */ in gmap_shadow_r3t()
1884 if (sg->edat_level >= 1) in gmap_shadow_r3t()
1886 list_add(&page->lru, &sg->crst_list); in gmap_shadow_r3t()
1890 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r3t()
1893 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r3t()
1899 rc = gmap_protect_rmap(sg, raddr, origin + offset, len); in gmap_shadow_r3t()
1900 spin_lock(&sg->guest_table_lock); in gmap_shadow_r3t()
1902 table = gmap_table_walk(sg, saddr, 3); in gmap_shadow_r3t()
1908 gmap_unshadow_r3t(sg, raddr); in gmap_shadow_r3t()
1910 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r3t()
1913 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r3t()
1921 * @sg: pointer to the shadow guest address space structure
1930 * Called with sg->mm->mmap_lock in read.
1932 int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt, in gmap_shadow_sgt() argument
1941 BUG_ON(!gmap_is_shadow(sg) || (sgt & _REGION3_ENTRY_LARGE)); in gmap_shadow_sgt()
1951 spin_lock(&sg->guest_table_lock); in gmap_shadow_sgt()
1952 table = gmap_table_walk(sg, saddr, 2); /* get region-3 pointer */ in gmap_shadow_sgt()
1968 if (sg->edat_level >= 1) in gmap_shadow_sgt()
1970 list_add(&page->lru, &sg->crst_list); in gmap_shadow_sgt()
1974 spin_unlock(&sg->guest_table_lock); in gmap_shadow_sgt()
1977 spin_unlock(&sg->guest_table_lock); in gmap_shadow_sgt()
1983 rc = gmap_protect_rmap(sg, raddr, origin + offset, len); in gmap_shadow_sgt()
1984 spin_lock(&sg->guest_table_lock); in gmap_shadow_sgt()
1986 table = gmap_table_walk(sg, saddr, 2); in gmap_shadow_sgt()
1992 gmap_unshadow_sgt(sg, raddr); in gmap_shadow_sgt()
1994 spin_unlock(&sg->guest_table_lock); in gmap_shadow_sgt()
1997 spin_unlock(&sg->guest_table_lock); in gmap_shadow_sgt()
2005 * @sg: pointer to the shadow guest address space structure
2014 * Called with sg->mm->mmap_lock in read.
2016 int gmap_shadow_pgt_lookup(struct gmap *sg, unsigned long saddr, in gmap_shadow_pgt_lookup() argument
2024 BUG_ON(!gmap_is_shadow(sg)); in gmap_shadow_pgt_lookup()
2025 spin_lock(&sg->guest_table_lock); in gmap_shadow_pgt_lookup()
2026 table = gmap_table_walk(sg, saddr, 1); /* get segment pointer */ in gmap_shadow_pgt_lookup()
2037 spin_unlock(&sg->guest_table_lock); in gmap_shadow_pgt_lookup()
2045 * @sg: pointer to the shadow guest address space structure
2056 int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt, in gmap_shadow_pgt() argument
2065 BUG_ON(!gmap_is_shadow(sg) || (pgt & _SEGMENT_ENTRY_LARGE)); in gmap_shadow_pgt()
2067 page = page_table_alloc_pgste(sg->mm); in gmap_shadow_pgt()
2075 spin_lock(&sg->guest_table_lock); in gmap_shadow_pgt()
2076 table = gmap_table_walk(sg, saddr, 1); /* get segment pointer */ in gmap_shadow_pgt()
2091 list_add(&page->lru, &sg->pt_list); in gmap_shadow_pgt()
2095 spin_unlock(&sg->guest_table_lock); in gmap_shadow_pgt()
2098 spin_unlock(&sg->guest_table_lock); in gmap_shadow_pgt()
2102 rc = gmap_protect_rmap(sg, raddr, origin, PAGE_SIZE); in gmap_shadow_pgt()
2103 spin_lock(&sg->guest_table_lock); in gmap_shadow_pgt()
2105 table = gmap_table_walk(sg, saddr, 1); in gmap_shadow_pgt()
2111 gmap_unshadow_pgt(sg, raddr); in gmap_shadow_pgt()
2113 spin_unlock(&sg->guest_table_lock); in gmap_shadow_pgt()
2116 spin_unlock(&sg->guest_table_lock); in gmap_shadow_pgt()
2125 * @sg: pointer to the shadow guest address space structure
2133 * Called with sg->mm->mmap_lock in read.
2135 int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte) in gmap_shadow_page() argument
2145 BUG_ON(!gmap_is_shadow(sg)); in gmap_shadow_page()
2146 parent = sg->parent; in gmap_shadow_page()
2167 spin_lock(&sg->guest_table_lock); in gmap_shadow_page()
2169 tptep = (pte_t *) gmap_table_walk(sg, saddr, 0); in gmap_shadow_page()
2171 spin_unlock(&sg->guest_table_lock); in gmap_shadow_page()
2176 rc = ptep_shadow_pte(sg->mm, saddr, sptep, tptep, pte); in gmap_shadow_page()
2179 gmap_insert_rmap(sg, vmaddr, rmap); in gmap_shadow_page()
2184 spin_unlock(&sg->guest_table_lock); in gmap_shadow_page()
2201 * Called with sg->parent->shadow_lock.
2203 static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr, in gmap_shadow_notify() argument
2209 BUG_ON(!gmap_is_shadow(sg)); in gmap_shadow_notify()
2211 spin_lock(&sg->guest_table_lock); in gmap_shadow_notify()
2212 if (sg->removed) { in gmap_shadow_notify()
2213 spin_unlock(&sg->guest_table_lock); in gmap_shadow_notify()
2217 start = sg->orig_asce & _ASCE_ORIGIN; in gmap_shadow_notify()
2218 end = start + ((sg->orig_asce & _ASCE_TABLE_LENGTH) + 1) * PAGE_SIZE; in gmap_shadow_notify()
2219 if (!(sg->orig_asce & _ASCE_REAL_SPACE) && gaddr >= start && in gmap_shadow_notify()
2222 gmap_unshadow(sg); in gmap_shadow_notify()
2223 spin_unlock(&sg->guest_table_lock); in gmap_shadow_notify()
2224 list_del(&sg->list); in gmap_shadow_notify()
2225 gmap_put(sg); in gmap_shadow_notify()
2229 head = radix_tree_delete(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT); in gmap_shadow_notify()
2235 gmap_unshadow_r2t(sg, raddr); in gmap_shadow_notify()
2238 gmap_unshadow_r3t(sg, raddr); in gmap_shadow_notify()
2241 gmap_unshadow_sgt(sg, raddr); in gmap_shadow_notify()
2244 gmap_unshadow_pgt(sg, raddr); in gmap_shadow_notify()
2247 gmap_unshadow_page(sg, raddr); in gmap_shadow_notify()
2252 spin_unlock(&sg->guest_table_lock); in gmap_shadow_notify()
2270 struct gmap *gmap, *sg, *next; in ptep_notify() local
2287 list_for_each_entry_safe(sg, next, in ptep_notify()
2289 gmap_shadow_notify(sg, vmaddr, gaddr); in ptep_notify()