linux/io_uring/memmap.c

18 static void *io_mem_alloc_compound(struct page **pages, int nr_pages,
34 	for (i = 0; i < nr_pages; i++)
42 	unsigned long start, end, nr_pages;
53 	nr_pages = end - start;
54 	if (WARN_ON_ONCE(!nr_pages))
56 	if (WARN_ON_ONCE(nr_pages > INT_MAX))
59 	pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
63 	ret = pin_user_pages_fast(uaddr, nr_pages, FOLL_WRITE | FOLL_LONGTERM,
66 	if (ret == nr_pages) {
67 		*npages = nr_pages;
94 		long nr_refs = mr->nr_pages;
108 	if (mr->nr_pages && ctx->user)
109 		__io_unaccount_mem(ctx->user, mr->nr_pages);
119 	if (io_check_coalesce_buffer(mr->pages, mr->nr_pages, &ifd)) {
125 	ptr = vmap(mr->pages, mr->nr_pages, VM_MAP, PAGE_KERNEL);
138 	unsigned long size = mr->nr_pages << PAGE_SHIFT;
140 	int nr_pages;
142 	pages = io_pin_pages(reg->user_addr, size, &nr_pages);
145 	if (WARN_ON_ONCE(nr_pages != mr->nr_pages))
159 	size_t size = (size_t) mr->nr_pages << PAGE_SHIFT;
164 	pages = kvmalloc_array(mr->nr_pages, sizeof(*pages), gfp);
168 	p = io_mem_alloc_compound(pages, mr->nr_pages, size, gfp);
175 					     mr->nr_pages, pages);
176 	if (nr_allocated != mr->nr_pages) {
192 	int nr_pages, ret;
195 	if (WARN_ON_ONCE(mr->pages || mr->ptr || mr->nr_pages))
213 	nr_pages = reg->size >> PAGE_SHIFT;
215 		ret = __io_account_mem(ctx->user, nr_pages);
219 	mr->nr_pages = nr_pages;
315 	unsigned long nr_pages = min(mr->nr_pages, max_pages);
318 	return vm_insert_pages(vma, vma->vm_start, mr->pages, &nr_pages);