261 struct zspage {  struct
276 static void zspage_lock_init(struct zspage *zspage)  in zspage_lock_init()  argument
279 	struct zspage_lock *zsl = &zspage->zsl;  in zspage_lock_init()
304 static void zspage_read_lock(struct zspage *zspage)  in zspage_read_lock()  argument
306 	struct zspage_lock *zsl = &zspage->zsl;  in zspage_read_lock()
317 static void zspage_read_unlock(struct zspage *zspage)  in zspage_read_unlock()  argument
319 	struct zspage_lock *zsl = &zspage->zsl;  in zspage_read_unlock()
328 static __must_check bool zspage_write_trylock(struct zspage *zspage)  in zspage_write_trylock()  argument
330 	struct zspage_lock *zsl = &zspage->zsl;  in zspage_write_trylock()
344 static void zspage_write_unlock(struct zspage *zspage)  in zspage_write_unlock()  argument
346 	struct zspage_lock *zsl = &zspage->zsl;  in zspage_write_unlock()
355 static void SetZsHugePage(struct zspage *zspage)  in SetZsHugePage()  argument
357 	zspage->huge = 1;  in SetZsHugePage()
360 static bool ZsHugePage(struct zspage *zspage)  in ZsHugePage()  argument
362 	return zspage->huge;  in ZsHugePage()
368 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage);
372 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage) {}  in SetZsPageMovable()  argument
387 static struct zspage *cache_alloc_zspage(gfp_t gfp)  in cache_alloc_zspage()
394 static void cache_free_zspage(struct zspage *zspage)  in cache_free_zspage()  argument
396 	kmem_cache_free(zspage_cachep, zspage);  in cache_free_zspage()
411 static inline int get_zspage_inuse(struct zspage *zspage)  in get_zspage_inuse()  argument
413 	return zspage->inuse;  in get_zspage_inuse()
416 static inline void mod_zspage_inuse(struct zspage *zspage, int val)  in mod_zspage_inuse()  argument
418 	zspage->inuse += val;  in mod_zspage_inuse()
421 static struct zpdesc *get_first_zpdesc(struct zspage *zspage)  in get_first_zpdesc()  argument
423 	struct zpdesc *first_zpdesc = zspage->first_zpdesc;  in get_first_zpdesc()
447 static inline unsigned int get_freeobj(struct zspage *zspage)  in get_freeobj()  argument
449 	return zspage->freeobj;  in get_freeobj()
452 static inline void set_freeobj(struct zspage *zspage, unsigned int obj)  in set_freeobj()  argument
454 	zspage->freeobj = obj;  in set_freeobj()
458 				       struct zspage *zspage)  in zspage_class()  argument
460 	return pool->size_class[zspage->class];  in zspage_class()
624 static int get_fullness_group(struct size_class *class, struct zspage *zspage)  in get_fullness_group()  argument
628 	inuse = get_zspage_inuse(zspage);  in get_fullness_group()
652 				struct zspage *zspage,  in insert_zspage()  argument
656 	list_add(&zspage->list, &class->fullness_list[fullness]);  in insert_zspage()
657 	zspage->fullness = fullness;  in insert_zspage()
664 static void remove_zspage(struct size_class *class, struct zspage *zspage)  in remove_zspage()  argument
666 	int fullness = zspage->fullness;  in remove_zspage()
670 	list_del_init(&zspage->list);  in remove_zspage()
683 static int fix_fullness_group(struct size_class *class, struct zspage *zspage)  in fix_fullness_group()  argument
687 	newfg = get_fullness_group(class, zspage);  in fix_fullness_group()
688 	if (newfg == zspage->fullness)  in fix_fullness_group()
691 	remove_zspage(class, zspage);  in fix_fullness_group()
692 	insert_zspage(class, zspage, newfg);  in fix_fullness_group()
697 static struct zspage *get_zspage(struct zpdesc *zpdesc)  in get_zspage()
699 	struct zspage *zspage = zpdesc->zspage;  in get_zspage()  local
701 	BUG_ON(zspage->magic != ZSPAGE_MAGIC);  in get_zspage()
702 	return zspage;  in get_zspage()
707 	struct zspage *zspage = get_zspage(zpdesc);  in get_next_zpdesc()  local
709 	if (unlikely(ZsHugePage(zspage)))  in get_next_zpdesc()
757 	struct zspage *zspage = get_zspage(zpdesc);  in obj_allocated()  local
759 	if (unlikely(ZsHugePage(zspage))) {  in obj_allocated()
778 	zpdesc->zspage = NULL;  in reset_zpdesc()
783 static int trylock_zspage(struct zspage *zspage)  in trylock_zspage()  argument
787 	for (cursor = get_first_zpdesc(zspage); cursor != NULL; cursor =  in trylock_zspage()
797 	for (cursor = get_first_zpdesc(zspage); cursor != fail; cursor =  in trylock_zspage()
805 				struct zspage *zspage)  in __free_zspage()  argument
811 	VM_BUG_ON(get_zspage_inuse(zspage));  in __free_zspage()
812 	VM_BUG_ON(zspage->fullness != ZS_INUSE_RATIO_0);  in __free_zspage()
814 	next = zpdesc = get_first_zpdesc(zspage);  in __free_zspage()
825 	cache_free_zspage(zspage);  in __free_zspage()
832 				struct zspage *zspage)  in free_zspage()  argument
834 	VM_BUG_ON(get_zspage_inuse(zspage));  in free_zspage()
835 	VM_BUG_ON(list_empty(&zspage->list));  in free_zspage()
842 	if (!trylock_zspage(zspage)) {  in free_zspage()
847 	remove_zspage(class, zspage);  in free_zspage()
848 	__free_zspage(pool, class, zspage);  in free_zspage()
852 static void init_zspage(struct size_class *class, struct zspage *zspage)  in init_zspage()  argument
856 	struct zpdesc *zpdesc = get_first_zpdesc(zspage);  in init_zspage()
893 	set_freeobj(zspage, 0);  in init_zspage()
896 static void create_page_chain(struct size_class *class, struct zspage *zspage,  in create_page_chain()  argument
914 		zpdesc->zspage = zspage;  in create_page_chain()
917 			zspage->first_zpdesc = zpdesc;  in create_page_chain()
921 				SetZsHugePage(zspage);  in create_page_chain()
932 static struct zspage *alloc_zspage(struct zs_pool *pool,  in alloc_zspage()
938 	struct zspage *zspage = cache_alloc_zspage(gfp);  in alloc_zspage()  local
940 	if (!zspage)  in alloc_zspage()
946 	zspage->magic = ZSPAGE_MAGIC;  in alloc_zspage()
947 	zspage->pool = pool;  in alloc_zspage()
948 	zspage->class = class->index;  in alloc_zspage()
949 	zspage_lock_init(zspage);  in alloc_zspage()
960 			cache_free_zspage(zspage);  in alloc_zspage()
969 	create_page_chain(class, zspage, zpdescs);  in alloc_zspage()
970 	init_zspage(class, zspage);  in alloc_zspage()
972 	return zspage;  in alloc_zspage()
975 static struct zspage *find_get_zspage(struct size_class *class)  in find_get_zspage()
978 	struct zspage *zspage;  in find_get_zspage()  local
981 		zspage = list_first_entry_or_null(&class->fullness_list[i],  in find_get_zspage()
982 						  struct zspage, list);  in find_get_zspage()
983 		if (zspage)  in find_get_zspage()
987 	return zspage;  in find_get_zspage()
1000 static bool zspage_full(struct size_class *class, struct zspage *zspage)  in zspage_full()  argument
1002 	return get_zspage_inuse(zspage) == class->objs_per_zspage;  in zspage_full()
1005 static bool zspage_empty(struct zspage *zspage)  in zspage_empty()  argument
1007 	return get_zspage_inuse(zspage) == 0;  in zspage_empty()
1040 	struct zspage *zspage;  in zs_obj_read_begin()  local
1051 	zspage = get_zspage(zpdesc);  in zs_obj_read_begin()
1054 	zspage_read_lock(zspage);  in zs_obj_read_begin()
1057 	class = zspage_class(pool, zspage);  in zs_obj_read_begin()
1060 	if (!ZsHugePage(zspage))  in zs_obj_read_begin()
1090 	struct zspage *zspage;  in zs_obj_read_end()  local
1098 	zspage = get_zspage(zpdesc);  in zs_obj_read_end()
1099 	class = zspage_class(pool, zspage);  in zs_obj_read_end()
1102 	if (!ZsHugePage(zspage))  in zs_obj_read_end()
1110 	zspage_read_unlock(zspage);  in zs_obj_read_end()
1117 	struct zspage *zspage;  in zs_obj_read_sg_begin()  local
1127 	zspage = get_zspage(zpdesc);  in zs_obj_read_sg_begin()
1130 	zspage_read_lock(zspage);  in zs_obj_read_sg_begin()
1133 	class = zspage_class(pool, zspage);  in zs_obj_read_sg_begin()
1136 	if (!ZsHugePage(zspage))  in zs_obj_read_sg_begin()
1163 	struct zspage *zspage;  in zs_obj_read_sg_end()  local
1170 	zspage = get_zspage(zpdesc);  in zs_obj_read_sg_end()
1172 	zspage_read_unlock(zspage);  in zs_obj_read_sg_end()
1179 	struct zspage *zspage;  in zs_obj_write()  local
1189 	zspage = get_zspage(zpdesc);  in zs_obj_write()
1192 	zspage_read_lock(zspage);  in zs_obj_write()
1195 	class = zspage_class(pool, zspage);  in zs_obj_write()
1198 	if (!ZsHugePage(zspage))  in zs_obj_write()
1221 	zspage_read_unlock(zspage);  in zs_obj_write()
1245 				struct zspage *zspage, unsigned long handle)  in obj_malloc()  argument
1256 	class = pool->size_class[zspage->class];  in obj_malloc()
1257 	obj = get_freeobj(zspage);  in obj_malloc()
1262 	m_zpdesc = get_first_zpdesc(zspage);  in obj_malloc()
1269 	set_freeobj(zspage, link->next >> OBJ_TAG_BITS);  in obj_malloc()
1270 	if (likely(!ZsHugePage(zspage)))  in obj_malloc()
1274 		zspage->first_zpdesc->handle = handle | OBJ_ALLOCATED_TAG;  in obj_malloc()
1277 	mod_zspage_inuse(zspage, 1);  in obj_malloc()
1303 	struct zspage *zspage;  in zs_malloc()  local
1321 	zspage = find_get_zspage(class);  in zs_malloc()
1322 	if (likely(zspage)) {  in zs_malloc()
1323 		obj_malloc(pool, zspage, handle);  in zs_malloc()
1325 		fix_fullness_group(class, zspage);  in zs_malloc()
1333 	zspage = alloc_zspage(pool, class, gfp, nid);  in zs_malloc()
1334 	if (!zspage) {  in zs_malloc()
1340 	obj_malloc(pool, zspage, handle);  in zs_malloc()
1341 	newfg = get_fullness_group(class, zspage);  in zs_malloc()
1342 	insert_zspage(class, zspage, newfg);  in zs_malloc()
1348 	SetZsPageMovable(pool, zspage);  in zs_malloc()
1359 	struct zspage *zspage;  in obj_free()  local
1368 	zspage = get_zspage(f_zpdesc);  in obj_free()
1374 	if (likely(!ZsHugePage(zspage)))  in obj_free()
1375 		link->next = get_freeobj(zspage) << OBJ_TAG_BITS;  in obj_free()
1378 	set_freeobj(zspage, f_objidx);  in obj_free()
1381 	mod_zspage_inuse(zspage, -1);  in obj_free()
1386 	struct zspage *zspage;  in zs_free()  local
1402 	zspage = get_zspage(f_zpdesc);  in zs_free()
1403 	class = zspage_class(pool, zspage);  in zs_free()
1410 	fullness = fix_fullness_group(class, zspage);  in zs_free()
1412 		free_zspage(pool, class, zspage);  in zs_free()
1519 static void migrate_zspage(struct zs_pool *pool, struct zspage *src_zspage,  in migrate_zspage()
1520 			   struct zspage *dst_zspage)  in migrate_zspage()
1554 static struct zspage *isolate_src_zspage(struct size_class *class)  in isolate_src_zspage()
1556 	struct zspage *zspage;  in isolate_src_zspage()  local
1560 		zspage = list_first_entry_or_null(&class->fullness_list[fg],  in isolate_src_zspage()
1561 						  struct zspage, list);  in isolate_src_zspage()
1562 		if (zspage) {  in isolate_src_zspage()
1563 			remove_zspage(class, zspage);  in isolate_src_zspage()
1564 			return zspage;  in isolate_src_zspage()
1568 	return zspage;  in isolate_src_zspage()
1571 static struct zspage *isolate_dst_zspage(struct size_class *class)  in isolate_dst_zspage()
1573 	struct zspage *zspage;  in isolate_dst_zspage()  local
1577 		zspage = list_first_entry_or_null(&class->fullness_list[fg],  in isolate_dst_zspage()
1578 						  struct zspage, list);  in isolate_dst_zspage()
1579 		if (zspage) {  in isolate_dst_zspage()
1580 			remove_zspage(class, zspage);  in isolate_dst_zspage()
1581 			return zspage;  in isolate_dst_zspage()
1585 	return zspage;  in isolate_dst_zspage()
1595 static int putback_zspage(struct size_class *class, struct zspage *zspage)  in putback_zspage()  argument
1599 	fullness = get_fullness_group(class, zspage);  in putback_zspage()
1600 	insert_zspage(class, zspage, fullness);  in putback_zspage()
1610 static void lock_zspage(struct zspage *zspage)  in lock_zspage()  argument
1623 		zspage_read_lock(zspage);  in lock_zspage()
1624 		zpdesc = get_first_zpdesc(zspage);  in lock_zspage()
1628 		zspage_read_unlock(zspage);  in lock_zspage()
1639 			zspage_read_unlock(zspage);  in lock_zspage()
1642 			zspage_read_lock(zspage);  in lock_zspage()
1645 	zspage_read_unlock(zspage);  in lock_zspage()
1651 static void replace_sub_page(struct size_class *class, struct zspage *zspage,  in replace_sub_page()  argument
1659 	zpdesc = get_first_zpdesc(zspage);  in replace_sub_page()
1668 	create_page_chain(class, zspage, zpdescs);  in replace_sub_page()
1671 	if (unlikely(ZsHugePage(zspage)))  in replace_sub_page()
1683 	return page_zpdesc(page)->zspage;  in zs_page_isolate()
1691 	struct zspage *zspage;  in zs_page_migrate()  local
1708 	if (!zpdesc->zspage)  in zs_page_migrate()
1712 	zspage = get_zspage(zpdesc);  in zs_page_migrate()
1713 	pool = zspage->pool;  in zs_page_migrate()
1720 	class = zspage_class(pool, zspage);  in zs_page_migrate()
1727 	if (!zspage_write_trylock(zspage)) {  in zs_page_migrate()
1758 	replace_sub_page(class, zspage, newzpdesc, zpdesc);  in zs_page_migrate()
1765 	zspage_write_unlock(zspage);  in zs_page_migrate()
1797 	struct zspage *zspage, *tmp;  in async_free_zspage()  local
1813 	list_for_each_entry_safe(zspage, tmp, &free_pages, list) {  in async_free_zspage()
1814 		list_del(&zspage->list);  in async_free_zspage()
1815 		lock_zspage(zspage);  in async_free_zspage()
1817 		class = zspage_class(pool, zspage);  in async_free_zspage()
1820 		__free_zspage(pool, class, zspage);  in async_free_zspage()
1840 static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage)  in SetZsPageMovable()  argument
1842 	struct zpdesc *zpdesc = get_first_zpdesc(zspage);  in SetZsPageMovable()
1877 	struct zspage *src_zspage = NULL;  in __zs_compact()
1878 	struct zspage *dst_zspage = NULL;  in __zs_compact()
2215 	zspage_cachep = kmem_cache_create("zspage", sizeof(struct zspage),  in zs_init_caches()