Lines Matching refs:class
263 unsigned int class:CLASS_BITS + 1; member
490 return pool->size_class[zspage->class]; in zspage_class()
511 static inline void class_stat_add(struct size_class *class, int type, in class_stat_add() argument
514 class->stats.objs[type] += cnt; in class_stat_add()
517 static inline void class_stat_sub(struct size_class *class, int type, in class_stat_sub() argument
520 class->stats.objs[type] -= cnt; in class_stat_sub()
523 static inline unsigned long class_stat_read(struct size_class *class, int type) in class_stat_read() argument
525 return class->stats.objs[type]; in class_stat_read()
545 static unsigned long zs_can_compact(struct size_class *class);
551 struct size_class *class; in zs_stats_size_show() local
566 class = pool->size_class[i]; in zs_stats_size_show()
568 if (class->index != i) in zs_stats_size_show()
571 spin_lock(&class->lock); in zs_stats_size_show()
573 seq_printf(s, " %5u %5u ", i, class->size); in zs_stats_size_show()
575 inuse_totals[fg] += class_stat_read(class, fg); in zs_stats_size_show()
576 seq_printf(s, "%9lu ", class_stat_read(class, fg)); in zs_stats_size_show()
579 obj_allocated = class_stat_read(class, ZS_OBJS_ALLOCATED); in zs_stats_size_show()
580 obj_used = class_stat_read(class, ZS_OBJS_INUSE); in zs_stats_size_show()
581 freeable = zs_can_compact(class); in zs_stats_size_show()
582 spin_unlock(&class->lock); in zs_stats_size_show()
584 objs_per_zspage = class->objs_per_zspage; in zs_stats_size_show()
586 class->pages_per_zspage; in zs_stats_size_show()
590 class->pages_per_zspage, freeable); in zs_stats_size_show()
654 static int get_fullness_group(struct size_class *class, struct zspage *zspage) in get_fullness_group() argument
659 objs_per_zspage = class->objs_per_zspage; in get_fullness_group()
681 static void insert_zspage(struct size_class *class, in insert_zspage() argument
685 class_stat_add(class, fullness, 1); in insert_zspage()
686 list_add(&zspage->list, &class->fullness_list[fullness]); in insert_zspage()
694 static void remove_zspage(struct size_class *class, struct zspage *zspage) in remove_zspage() argument
698 VM_BUG_ON(list_empty(&class->fullness_list[fullness])); in remove_zspage()
701 class_stat_sub(class, fullness, 1); in remove_zspage()
713 static int fix_fullness_group(struct size_class *class, struct zspage *zspage) in fix_fullness_group() argument
717 newfg = get_fullness_group(class, zspage); in fix_fullness_group()
721 remove_zspage(class, zspage); in fix_fullness_group()
722 insert_zspage(class, zspage, newfg); in fix_fullness_group()
834 static void __free_zspage(struct zs_pool *pool, struct size_class *class, in __free_zspage() argument
839 assert_spin_locked(&class->lock); in __free_zspage()
857 class_stat_sub(class, ZS_OBJS_ALLOCATED, class->objs_per_zspage); in __free_zspage()
858 atomic_long_sub(class->pages_per_zspage, &pool->pages_allocated); in __free_zspage()
861 static void free_zspage(struct zs_pool *pool, struct size_class *class, in free_zspage() argument
877 remove_zspage(class, zspage); in free_zspage()
878 __free_zspage(pool, class, zspage); in free_zspage()
882 static void init_zspage(struct size_class *class, struct zspage *zspage) in init_zspage() argument
898 while ((off += class->size) < PAGE_SIZE) { in init_zspage()
900 link += class->size / sizeof(*link); in init_zspage()
926 static void create_page_chain(struct size_class *class, struct zspage *zspage, in create_page_chain() argument
932 int nr_zpdescs = class->pages_per_zspage; in create_page_chain()
949 if (unlikely(class->objs_per_zspage == 1 && in create_page_chain()
950 class->pages_per_zspage == 1)) in create_page_chain()
963 struct size_class *class, in alloc_zspage() argument
978 zspage->class = class->index; in alloc_zspage()
981 for (i = 0; i < class->pages_per_zspage; i++) { in alloc_zspage()
999 create_page_chain(class, zspage, zpdescs); in alloc_zspage()
1000 init_zspage(class, zspage); in alloc_zspage()
1005 static struct zspage *find_get_zspage(struct size_class *class) in find_get_zspage() argument
1011 zspage = list_first_entry_or_null(&class->fullness_list[i], in find_get_zspage()
1030 static bool zspage_full(struct size_class *class, struct zspage *zspage) in zspage_full() argument
1032 return get_zspage_inuse(zspage) == class->objs_per_zspage; in zspage_full()
1053 struct size_class *class; in zs_lookup_class_index() local
1055 class = pool->size_class[get_size_class_index(size)]; in zs_lookup_class_index()
1057 return class->index; in zs_lookup_class_index()
1074 struct size_class *class; in zs_obj_read_begin() local
1087 class = zspage_class(pool, zspage); in zs_obj_read_begin()
1088 off = offset_in_page(class->size * obj_idx); in zs_obj_read_begin()
1090 if (off + class->size <= PAGE_SIZE) { in zs_obj_read_begin()
1099 sizes[1] = class->size - sizes[0]; in zs_obj_read_begin()
1124 struct size_class *class; in zs_obj_read_end() local
1129 class = zspage_class(pool, zspage); in zs_obj_read_end()
1130 off = offset_in_page(class->size * obj_idx); in zs_obj_read_end()
1132 if (off + class->size <= PAGE_SIZE) { in zs_obj_read_end()
1150 struct size_class *class; in zs_obj_write() local
1162 class = zspage_class(pool, zspage); in zs_obj_write()
1163 off = offset_in_page(class->size * obj_idx); in zs_obj_write()
1217 struct size_class *class; in obj_malloc() local
1223 class = pool->size_class[zspage->class]; in obj_malloc()
1226 offset = obj * class->size; in obj_malloc()
1268 struct size_class *class; in zs_malloc() local
1284 class = pool->size_class[get_size_class_index(size)]; in zs_malloc()
1287 spin_lock(&class->lock); in zs_malloc()
1288 zspage = find_get_zspage(class); in zs_malloc()
1292 fix_fullness_group(class, zspage); in zs_malloc()
1293 class_stat_add(class, ZS_OBJS_INUSE, 1); in zs_malloc()
1298 spin_unlock(&class->lock); in zs_malloc()
1300 zspage = alloc_zspage(pool, class, gfp, nid); in zs_malloc()
1306 spin_lock(&class->lock); in zs_malloc()
1308 newfg = get_fullness_group(class, zspage); in zs_malloc()
1309 insert_zspage(class, zspage, newfg); in zs_malloc()
1310 atomic_long_add(class->pages_per_zspage, &pool->pages_allocated); in zs_malloc()
1311 class_stat_add(class, ZS_OBJS_ALLOCATED, class->objs_per_zspage); in zs_malloc()
1312 class_stat_add(class, ZS_OBJS_INUSE, 1); in zs_malloc()
1317 spin_unlock(&class->lock); in zs_malloc()
1356 struct size_class *class; in zs_free() local
1370 class = zspage_class(pool, zspage); in zs_free()
1371 spin_lock(&class->lock); in zs_free()
1374 class_stat_sub(class, ZS_OBJS_INUSE, 1); in zs_free()
1375 obj_free(class->size, obj); in zs_free()
1377 fullness = fix_fullness_group(class, zspage); in zs_free()
1379 free_zspage(pool, class, zspage); in zs_free()
1381 spin_unlock(&class->lock); in zs_free()
1386 static void zs_object_copy(struct size_class *class, unsigned long dst, in zs_object_copy() argument
1396 s_size = d_size = class->size; in zs_object_copy()
1401 s_off = offset_in_page(class->size * s_objidx); in zs_object_copy()
1402 d_off = offset_in_page(class->size * d_objidx); in zs_object_copy()
1404 if (s_off + class->size > PAGE_SIZE) in zs_object_copy()
1407 if (d_off + class->size > PAGE_SIZE) in zs_object_copy()
1418 if (written == class->size) in zs_object_copy()
1439 s_size = class->size - written; in zs_object_copy()
1447 d_size = class->size - written; in zs_object_copy()
1460 static unsigned long find_alloced_obj(struct size_class *class, in find_alloced_obj() argument
1469 offset += class->size * index; in find_alloced_obj()
1475 offset += class->size; in find_alloced_obj()
1493 struct size_class *class = pool->size_class[src_zspage->class]; in migrate_zspage() local
1496 handle = find_alloced_obj(class, s_zpdesc, &obj_idx); in migrate_zspage()
1507 zs_object_copy(class, free_obj, used_obj); in migrate_zspage()
1509 obj_free(class->size, used_obj); in migrate_zspage()
1512 if (zspage_full(class, dst_zspage)) in migrate_zspage()
1521 static struct zspage *isolate_src_zspage(struct size_class *class) in isolate_src_zspage() argument
1527 zspage = list_first_entry_or_null(&class->fullness_list[fg], in isolate_src_zspage()
1530 remove_zspage(class, zspage); in isolate_src_zspage()
1538 static struct zspage *isolate_dst_zspage(struct size_class *class) in isolate_dst_zspage() argument
1544 zspage = list_first_entry_or_null(&class->fullness_list[fg], in isolate_dst_zspage()
1547 remove_zspage(class, zspage); in isolate_dst_zspage()
1562 static int putback_zspage(struct size_class *class, struct zspage *zspage) in putback_zspage() argument
1566 fullness = get_fullness_group(class, zspage); in putback_zspage()
1567 insert_zspage(class, zspage, fullness); in putback_zspage()
1618 static void replace_sub_page(struct size_class *class, struct zspage *zspage, in replace_sub_page() argument
1635 create_page_chain(class, zspage, zpdescs); in replace_sub_page()
1657 struct size_class *class; in zs_page_migrate() local
1687 class = zspage_class(pool, zspage); in zs_page_migrate()
1692 spin_lock(&class->lock); in zs_page_migrate()
1695 spin_unlock(&class->lock); in zs_page_migrate()
1714 addr += class->size) { in zs_page_migrate()
1725 replace_sub_page(class, zspage, newzpdesc, zpdesc); in zs_page_migrate()
1731 spin_unlock(&class->lock); in zs_page_migrate()
1763 struct size_class *class; in async_free_zspage() local
1770 class = pool->size_class[i]; in async_free_zspage()
1771 if (class->index != i) in async_free_zspage()
1774 spin_lock(&class->lock); in async_free_zspage()
1775 list_splice_init(&class->fullness_list[ZS_INUSE_RATIO_0], in async_free_zspage()
1777 spin_unlock(&class->lock); in async_free_zspage()
1784 class = zspage_class(pool, zspage); in async_free_zspage()
1785 spin_lock(&class->lock); in async_free_zspage()
1786 class_stat_sub(class, ZS_INUSE_RATIO_0, 1); in async_free_zspage()
1787 __free_zspage(pool, class, zspage); in async_free_zspage()
1788 spin_unlock(&class->lock); in async_free_zspage()
1826 static unsigned long zs_can_compact(struct size_class *class) in zs_can_compact() argument
1829 unsigned long obj_allocated = class_stat_read(class, ZS_OBJS_ALLOCATED); in zs_can_compact()
1830 unsigned long obj_used = class_stat_read(class, ZS_OBJS_INUSE); in zs_can_compact()
1836 obj_wasted /= class->objs_per_zspage; in zs_can_compact()
1838 return obj_wasted * class->pages_per_zspage; in zs_can_compact()
1842 struct size_class *class) in __zs_compact() argument
1853 spin_lock(&class->lock); in __zs_compact()
1854 while (zs_can_compact(class)) { in __zs_compact()
1858 dst_zspage = isolate_dst_zspage(class); in __zs_compact()
1863 src_zspage = isolate_src_zspage(class); in __zs_compact()
1873 fg = putback_zspage(class, src_zspage); in __zs_compact()
1875 free_zspage(pool, class, src_zspage); in __zs_compact()
1876 pages_freed += class->pages_per_zspage; in __zs_compact()
1880 if (get_fullness_group(class, dst_zspage) == ZS_INUSE_RATIO_100 in __zs_compact()
1882 putback_zspage(class, dst_zspage); in __zs_compact()
1885 spin_unlock(&class->lock); in __zs_compact()
1889 spin_lock(&class->lock); in __zs_compact()
1894 putback_zspage(class, src_zspage); in __zs_compact()
1897 putback_zspage(class, dst_zspage); in __zs_compact()
1899 spin_unlock(&class->lock); in __zs_compact()
1908 struct size_class *class; in zs_compact() local
1921 class = pool->size_class[i]; in zs_compact()
1922 if (class->index != i) in zs_compact()
1924 pages_freed += __zs_compact(pool, class); in zs_compact()
1959 struct size_class *class; in zs_shrinker_count() local
1964 class = pool->size_class[i]; in zs_shrinker_count()
1965 if (class->index != i) in zs_shrinker_count()
1968 pages_to_free += zs_can_compact(class); in zs_shrinker_count()
2055 struct size_class *class; in zs_create_pool() local
2101 class = kzalloc(sizeof(struct size_class), GFP_KERNEL); in zs_create_pool()
2102 if (!class) in zs_create_pool()
2105 class->size = size; in zs_create_pool()
2106 class->index = i; in zs_create_pool()
2107 class->pages_per_zspage = pages_per_zspage; in zs_create_pool()
2108 class->objs_per_zspage = objs_per_zspage; in zs_create_pool()
2109 spin_lock_init(&class->lock); in zs_create_pool()
2110 pool->size_class[i] = class; in zs_create_pool()
2114 INIT_LIST_HEAD(&class->fullness_list[fullness]); in zs_create_pool()
2118 prev_class = class; in zs_create_pool()
2150 struct size_class *class = pool->size_class[i]; in zs_destroy_pool() local
2152 if (!class) in zs_destroy_pool()
2155 if (class->index != i) in zs_destroy_pool()
2159 if (list_empty(&class->fullness_list[fg])) in zs_destroy_pool()
2163 class->size, fg); in zs_destroy_pool()
2165 kfree(class); in zs_destroy_pool()