135 static void memcg_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages);
141 	unsigned int nr_pages;  in obj_cgroup_release()  local
166 	nr_pages = nr_bytes >> PAGE_SHIFT;  in obj_cgroup_release()
168 	if (nr_pages) {  in obj_cgroup_release()
172 		mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages);  in obj_cgroup_release()
173 		memcg1_account_kmem(memcg, -nr_pages);  in obj_cgroup_release()
175 			memcg_uncharge(memcg, nr_pages);  in obj_cgroup_release()
1272 				int zid, int nr_pages)  in mem_cgroup_update_lru_size()  argument
1284 	if (nr_pages < 0)  in mem_cgroup_update_lru_size()
1285 		*lru_size += nr_pages;  in mem_cgroup_update_lru_size()
1290 		__func__, lruvec, lru, nr_pages, size)) {  in mem_cgroup_update_lru_size()
1295 	if (nr_pages > 0)  in mem_cgroup_update_lru_size()
1296 		*lru_size += nr_pages;  in mem_cgroup_update_lru_size()
1752 	uint8_t nr_pages[NR_MEMCG_STOCK];  member
1796 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)  in consume_stock()  argument
1803 	if (nr_pages > MEMCG_CHARGE_BATCH ||  in consume_stock()
1813 		stock_pages = READ_ONCE(stock->nr_pages[i]);  in consume_stock()
1814 		if (stock_pages >= nr_pages) {  in consume_stock()
1815 			WRITE_ONCE(stock->nr_pages[i], stock_pages - nr_pages);  in consume_stock()
1826 static void memcg_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages)  in memcg_uncharge()  argument
1828 	page_counter_uncharge(&memcg->memory, nr_pages);  in memcg_uncharge()
1830 		page_counter_uncharge(&memcg->memsw, nr_pages);  in memcg_uncharge()
1844 	stock_pages = READ_ONCE(stock->nr_pages[i]);  in drain_stock()
1847 		WRITE_ONCE(stock->nr_pages[i], 0);  in drain_stock()
1894 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)  in refill_stock()  argument
1912 	if (nr_pages > MEMCG_CHARGE_BATCH ||  in refill_stock()
1918 		memcg_uncharge(memcg, nr_pages);  in refill_stock()
1928 			stock_pages = READ_ONCE(stock->nr_pages[i]) + nr_pages;  in refill_stock()
1929 			WRITE_ONCE(stock->nr_pages[i], stock_pages);  in refill_stock()
1945 		WRITE_ONCE(stock->nr_pages[i], nr_pages);  in refill_stock()
1964 		if (READ_ONCE(stock->nr_pages[i]) &&  in is_memcg_drain_needed()
2031 				  unsigned int nr_pages,  in reclaim_high()  argument
2046 		nr_reclaimed += try_to_free_mem_cgroup_pages(memcg, nr_pages,  in reclaim_high()
2171 					  unsigned int nr_pages,  in calculate_high_delay()  argument
2199 	return penalty_jiffies * nr_pages / MEMCG_CHARGE_BATCH;  in calculate_high_delay()
2212 	unsigned int nr_pages = current->memcg_nr_pages_over_high;  in __mem_cgroup_handle_over_high()  local
2242 				    in_retry ? SWAP_CLUSTER_MAX : nr_pages,  in __mem_cgroup_handle_over_high()
2249 	penalty_jiffies = calculate_high_delay(memcg, nr_pages,  in __mem_cgroup_handle_over_high()
2252 	penalty_jiffies += calculate_high_delay(memcg, nr_pages,  in __mem_cgroup_handle_over_high()
2298 			    unsigned int nr_pages)  in try_charge_memcg()  argument
2300 	unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages);  in try_charge_memcg()
2313 	if (consume_stock(memcg, nr_pages))  in try_charge_memcg()
2318 		batch = nr_pages;  in try_charge_memcg()
2332 	if (batch > nr_pages) {  in try_charge_memcg()
2333 		batch = nr_pages;  in try_charge_memcg()
2356 	nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,  in try_charge_memcg()
2360 	if (mem_cgroup_margin(mem_over_limit) >= nr_pages)  in try_charge_memcg()
2380 	if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))  in try_charge_memcg()
2399 			   get_order(nr_pages * PAGE_SIZE))) {  in try_charge_memcg()
2426 	page_counter_charge(&memcg->memory, nr_pages);  in try_charge_memcg()
2428 		page_counter_charge(&memcg->memsw, nr_pages);  in try_charge_memcg()
2433 	if (batch > nr_pages)  in try_charge_memcg()
2434 		refill_stock(memcg, batch - nr_pages);  in try_charge_memcg()
2493 			     unsigned int nr_pages)  in try_charge()  argument
2498 	return try_charge_memcg(memcg, gfp_mask, nr_pages);  in try_charge()
2769 				      unsigned int nr_pages)  in obj_cgroup_uncharge_pages()  argument
2775 	account_kmem_nmi_safe(memcg, -nr_pages);  in obj_cgroup_uncharge_pages()
2776 	memcg1_account_kmem(memcg, -nr_pages);  in obj_cgroup_uncharge_pages()
2778 		refill_stock(memcg, nr_pages);  in obj_cgroup_uncharge_pages()
2792 				   unsigned int nr_pages)  in obj_cgroup_charge_pages()  argument
2799 	ret = try_charge_memcg(memcg, gfp, nr_pages);  in obj_cgroup_charge_pages()
2803 	account_kmem_nmi_safe(memcg, nr_pages);  in obj_cgroup_charge_pages()
2804 	memcg1_account_kmem(memcg, nr_pages);  in obj_cgroup_charge_pages()
2861 	unsigned int nr_pages = 1 << order;  in __memcg_kmem_uncharge_page()  local
2866 	obj_cgroup_uncharge_pages(objcg, nr_pages);  in __memcg_kmem_uncharge_page()
2951 		unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT;  in drain_obj_stock()  local
2954 		if (nr_pages) {  in drain_obj_stock()
2959 			mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages);  in drain_obj_stock()
2960 			memcg1_account_kmem(memcg, -nr_pages);  in drain_obj_stock()
2962 				memcg_uncharge(memcg, nr_pages);  in drain_obj_stock()
3027 	unsigned int nr_pages = 0;  in refill_obj_stock()  local
3032 		nr_pages = nr_bytes >> PAGE_SHIFT;  in refill_obj_stock()
3054 		nr_pages = stock->nr_bytes >> PAGE_SHIFT;  in refill_obj_stock()
3060 	if (nr_pages)  in refill_obj_stock()
3061 		obj_cgroup_uncharge_pages(objcg, nr_pages);  in refill_obj_stock()
3067 	unsigned int nr_pages, nr_bytes;  in obj_cgroup_charge_account()  local
3096 	nr_pages = size >> PAGE_SHIFT;  in obj_cgroup_charge_account()
3100 		nr_pages += 1;  in obj_cgroup_charge_account()
3102 	ret = obj_cgroup_charge_pages(objcg, gfp, nr_pages);  in obj_cgroup_charge_account()
4369 		unsigned long nr_pages = page_counter_read(&memcg->memory);  in memory_high_write()  local
4372 		if (nr_pages <= high)  in memory_high_write()
4384 		reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high,  in memory_high_write()
4421 		unsigned long nr_pages = page_counter_read(&memcg->memory);  in memory_max_write()  local
4423 		if (nr_pages <= max)  in memory_max_write()
4436 			if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,  in memory_max_write()
4825 	long nr_pages;  in uncharge_folio()  local
4862 	nr_pages = folio_nr_pages(folio);  in uncharge_folio()
4865 		ug->nr_memory += nr_pages;  in uncharge_folio()
4866 		ug->nr_kmem += nr_pages;  in uncharge_folio()
4873 			ug->nr_memory += nr_pages;  in uncharge_folio()
4921 	long nr_pages = folio_nr_pages(new);  in mem_cgroup_replace_folio()  local
4926 	VM_BUG_ON_FOLIO(folio_nr_pages(old) != nr_pages, new);  in mem_cgroup_replace_folio()
4942 		page_counter_charge(&memcg->memory, nr_pages);  in mem_cgroup_replace_folio()
4944 			page_counter_charge(&memcg->memsw, nr_pages);  in mem_cgroup_replace_folio()
5053 bool mem_cgroup_sk_charge(const struct sock *sk, unsigned int nr_pages,  in mem_cgroup_sk_charge()  argument
5059 		return memcg1_charge_skmem(memcg, nr_pages, gfp_mask);  in mem_cgroup_sk_charge()
5061 	if (try_charge_memcg(memcg, gfp_mask, nr_pages) == 0) {  in mem_cgroup_sk_charge()
5062 		mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);  in mem_cgroup_sk_charge()
5074 void mem_cgroup_sk_uncharge(const struct sock *sk, unsigned int nr_pages)  in mem_cgroup_sk_uncharge()  argument
5079 		memcg1_uncharge_skmem(memcg, nr_pages);  in mem_cgroup_sk_uncharge()
5083 	mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);  in mem_cgroup_sk_uncharge()
5085 	refill_stock(memcg, nr_pages);  in mem_cgroup_sk_uncharge()
5159 	unsigned int nr_pages = folio_nr_pages(folio);  in __mem_cgroup_try_charge_swap()  local
5180 	    !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {  in __mem_cgroup_try_charge_swap()
5188 	if (nr_pages > 1)  in __mem_cgroup_try_charge_swap()
5189 		mem_cgroup_id_get_many(memcg, nr_pages - 1);  in __mem_cgroup_try_charge_swap()
5190 	mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);  in __mem_cgroup_try_charge_swap()
5202 void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)  in __mem_cgroup_uncharge_swap()  argument
5207 	id = swap_cgroup_clear(entry, nr_pages);  in __mem_cgroup_uncharge_swap()
5213 				page_counter_uncharge(&memcg->memsw, nr_pages);  in __mem_cgroup_uncharge_swap()
5215 				page_counter_uncharge(&memcg->swap, nr_pages);  in __mem_cgroup_uncharge_swap()
5217 		mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);  in __mem_cgroup_uncharge_swap()
5218 		mem_cgroup_id_put_many(memcg, nr_pages);  in __mem_cgroup_uncharge_swap()