1 /*
2  * Copyright (c) 2005 Topspin Communications.  All rights reserved.
3  * Copyright (c) 2005 Cisco Systems.  All rights reserved.
4  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5  * Copyright (c) 2020 Intel Corporation. All rights reserved.
6  *
7  * This software is available to you under a choice of one of two
8  * licenses.  You may choose to be licensed under the terms of the GNU
9  * General Public License (GPL) Version 2, available from the file
10  * COPYING in the main directory of this source tree, or the
11  * OpenIB.org BSD license below:
12  *
13  *     Redistribution and use in source and binary forms, with or
14  *     without modification, are permitted provided that the following
15  *     conditions are met:
16  *
17  *      - Redistributions of source code must retain the above
18  *        copyright notice, this list of conditions and the following
19  *        disclaimer.
20  *
21  *      - Redistributions in binary form must reproduce the above
22  *        copyright notice, this list of conditions and the following
23  *        disclaimer in the documentation and/or other materials
24  *        provided with the distribution.
25  *
26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33  * SOFTWARE.
34  */
35 
36 #include <linux/mm.h>
37 #include <linux/dma-mapping.h>
38 #include <linux/sched/signal.h>
39 #include <linux/sched/mm.h>
40 #include <linux/export.h>
41 #include <linux/slab.h>
42 #include <linux/pagemap.h>
43 #include <linux/count_zeros.h>
44 #include <rdma/ib_umem_odp.h>
45 
46 #include "uverbs.h"
47 
48 static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int dirty)
49 {
50 	bool make_dirty = umem->writable && dirty;
51 	struct scatterlist *sg;
52 	unsigned int i;
53 
54 	if (dirty)
55 		ib_dma_unmap_sgtable_attrs(dev, &umem->sgt_append.sgt,
56 					   DMA_BIDIRECTIONAL, 0);
57 
58 	for_each_sgtable_sg(&umem->sgt_append.sgt, sg, i)
59 		unpin_user_page_range_dirty_lock(sg_page(sg),
60 			DIV_ROUND_UP(sg->length, PAGE_SIZE), make_dirty);
61 
62 	sg_free_append_table(&umem->sgt_append);
63 }
64 
65 /**
66  * ib_umem_find_best_pgsz - Find best HW page size to use for this MR
67  *
68  * @umem: umem struct
69  * @pgsz_bitmap: bitmap of HW supported page sizes
70  * @virt: IOVA
71  *
72  * This helper is intended for HW that support multiple page
73  * sizes but can do only a single page size in an MR.
74  *
75  * Returns 0 if the umem requires page sizes not supported by
76  * the driver to be mapped. Drivers always supporting PAGE_SIZE
77  * or smaller will never see a 0 result.
78  */
79 unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,
80 				     unsigned long pgsz_bitmap,
81 				     unsigned long virt)
82 {
83 	unsigned long curr_len = 0;
84 	dma_addr_t curr_base = ~0;
85 	unsigned long va, pgoff;
86 	struct scatterlist *sg;
87 	dma_addr_t mask;
88 	dma_addr_t end;
89 	int i;
90 
91 	umem->iova = va = virt;
92 
93 	if (umem->is_odp) {
94 		unsigned int page_size = BIT(to_ib_umem_odp(umem)->page_shift);
95 
96 		/* ODP must always be self consistent. */
97 		if (!(pgsz_bitmap & page_size))
98 			return 0;
99 		return page_size;
100 	}
101 
102 	/* The best result is the smallest page size that results in the minimum
103 	 * number of required pages. Compute the largest page size that could
104 	 * work based on VA address bits that don't change.
105 	 */
106 	mask = pgsz_bitmap &
107 	       GENMASK(BITS_PER_LONG - 1,
108 		       bits_per((umem->length - 1 + virt) ^ virt));
109 	/* offset into first SGL */
110 	pgoff = umem->address & ~PAGE_MASK;
111 
112 	for_each_sgtable_dma_sg(&umem->sgt_append.sgt, sg, i) {
113 		/* If the current entry is physically contiguous with the previous
114 		 * one, no need to take its start addresses into consideration.
115 		 */
116 		if (check_add_overflow(curr_base, curr_len, &end) ||
117 		    end != sg_dma_address(sg)) {
118 
119 			curr_base = sg_dma_address(sg);
120 			curr_len = 0;
121 
122 			/* Reduce max page size if VA/PA bits differ */
123 			mask |= (curr_base + pgoff) ^ va;
124 
125 			/* The alignment of any VA matching a discontinuity point
126 			* in the physical memory sets the maximum possible page
127 			* size as this must be a starting point of a new page that
128 			* needs to be aligned.
129 			*/
130 			if (i != 0)
131 				mask |= va;
132 		}
133 
134 		curr_len += sg_dma_len(sg);
135 		va += sg_dma_len(sg) - pgoff;
136 
137 		pgoff = 0;
138 	}
139 
140 	/* The mask accumulates 1's in each position where the VA and physical
141 	 * address differ, thus the length of trailing 0 is the largest page
142 	 * size that can pass the VA through to the physical.
143 	 */
144 	if (mask)
145 		pgsz_bitmap &= GENMASK(count_trailing_zeros(mask), 0);
146 	return pgsz_bitmap ? rounddown_pow_of_two(pgsz_bitmap) : 0;
147 }
148 EXPORT_SYMBOL(ib_umem_find_best_pgsz);
149 
150 /**
151  * ib_umem_get - Pin and DMA map userspace memory.
152  *
153  * @device: IB device to connect UMEM
154  * @addr: userspace virtual address to start at
155  * @size: length of region to pin
156  * @access: IB_ACCESS_xxx flags for memory being pinned
157  */
158 struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr,
159 			    size_t size, int access)
160 {
161 	struct ib_umem *umem;
162 	struct page **page_list;
163 	unsigned long lock_limit;
164 	unsigned long new_pinned;
165 	unsigned long cur_base;
166 	unsigned long dma_attr = 0;
167 	struct mm_struct *mm;
168 	unsigned long npages;
169 	int pinned, ret;
170 	unsigned int gup_flags = FOLL_LONGTERM;
171 
172 	/*
173 	 * If the combination of the addr and size requested for this memory
174 	 * region causes an integer overflow, return error.
175 	 */
176 	if (((addr + size) < addr) ||
177 	    PAGE_ALIGN(addr + size) < (addr + size))
178 		return ERR_PTR(-EINVAL);
179 
180 	if (!can_do_mlock())
181 		return ERR_PTR(-EPERM);
182 
183 	if (access & IB_ACCESS_ON_DEMAND)
184 		return ERR_PTR(-EOPNOTSUPP);
185 
186 	umem = kzalloc(sizeof(*umem), GFP_KERNEL);
187 	if (!umem)
188 		return ERR_PTR(-ENOMEM);
189 	umem->ibdev      = device;
190 	umem->length     = size;
191 	umem->address    = addr;
192 	/*
193 	 * Drivers should call ib_umem_find_best_pgsz() to set the iova
194 	 * correctly.
195 	 */
196 	umem->iova = addr;
197 	umem->writable   = ib_access_writable(access);
198 	umem->owning_mm = mm = current->mm;
199 	mmgrab(mm);
200 
201 	page_list = (struct page **) __get_free_page(GFP_KERNEL);
202 	if (!page_list) {
203 		ret = -ENOMEM;
204 		goto umem_kfree;
205 	}
206 
207 	npages = ib_umem_num_pages(umem);
208 	if (npages == 0 || npages > UINT_MAX) {
209 		ret = -EINVAL;
210 		goto out;
211 	}
212 
213 	lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
214 
215 	new_pinned = atomic64_add_return(npages, &mm->pinned_vm);
216 	if (new_pinned > lock_limit && !capable(CAP_IPC_LOCK)) {
217 		atomic64_sub(npages, &mm->pinned_vm);
218 		ret = -ENOMEM;
219 		goto out;
220 	}
221 
222 	cur_base = addr & PAGE_MASK;
223 
224 	if (umem->writable)
225 		gup_flags |= FOLL_WRITE;
226 
227 	while (npages) {
228 		cond_resched();
229 		pinned = pin_user_pages_fast(cur_base,
230 					  min_t(unsigned long, npages,
231 						PAGE_SIZE /
232 						sizeof(struct page *)),
233 					  gup_flags, page_list);
234 		if (pinned < 0) {
235 			ret = pinned;
236 			goto umem_release;
237 		}
238 
239 		cur_base += pinned * PAGE_SIZE;
240 		npages -= pinned;
241 		ret = sg_alloc_append_table_from_pages(
242 			&umem->sgt_append, page_list, pinned, 0,
243 			pinned << PAGE_SHIFT, ib_dma_max_seg_size(device),
244 			npages, GFP_KERNEL);
245 		if (ret) {
246 			unpin_user_pages_dirty_lock(page_list, pinned, 0);
247 			goto umem_release;
248 		}
249 	}
250 
251 	if (access & IB_ACCESS_RELAXED_ORDERING)
252 		dma_attr |= DMA_ATTR_WEAK_ORDERING;
253 
254 	ret = ib_dma_map_sgtable_attrs(device, &umem->sgt_append.sgt,
255 				       DMA_BIDIRECTIONAL, dma_attr);
256 	if (ret)
257 		goto umem_release;
258 	goto out;
259 
260 umem_release:
261 	__ib_umem_release(device, umem, 0);
262 	atomic64_sub(ib_umem_num_pages(umem), &mm->pinned_vm);
263 out:
264 	free_page((unsigned long) page_list);
265 umem_kfree:
266 	if (ret) {
267 		mmdrop(umem->owning_mm);
268 		kfree(umem);
269 	}
270 	return ret ? ERR_PTR(ret) : umem;
271 }
272 EXPORT_SYMBOL(ib_umem_get);
273 
274 /**
275  * ib_umem_release - release memory pinned with ib_umem_get
276  * @umem: umem struct to release
277  */
278 void ib_umem_release(struct ib_umem *umem)
279 {
280 	if (!umem)
281 		return;
282 	if (umem->is_dmabuf)
283 		return ib_umem_dmabuf_release(to_ib_umem_dmabuf(umem));
284 	if (umem->is_odp)
285 		return ib_umem_odp_release(to_ib_umem_odp(umem));
286 
287 	__ib_umem_release(umem->ibdev, umem, 1);
288 
289 	atomic64_sub(ib_umem_num_pages(umem), &umem->owning_mm->pinned_vm);
290 	mmdrop(umem->owning_mm);
291 	kfree(umem);
292 }
293 EXPORT_SYMBOL(ib_umem_release);
294 
295 /*
296  * Copy from the given ib_umem's pages to the given buffer.
297  *
298  * umem - the umem to copy from
299  * offset - offset to start copying from
300  * dst - destination buffer
301  * length - buffer length
302  *
303  * Returns 0 on success, or an error code.
304  */
305 int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset,
306 		      size_t length)
307 {
308 	size_t end = offset + length;
309 	int ret;
310 
311 	if (offset > umem->length || length > umem->length - offset) {
312 		pr_err("%s not in range. offset: %zd umem length: %zd end: %zd\n",
313 		       __func__, offset, umem->length, end);
314 		return -EINVAL;
315 	}
316 
317 	ret = sg_pcopy_to_buffer(umem->sgt_append.sgt.sgl,
318 				 umem->sgt_append.sgt.orig_nents, dst, length,
319 				 offset + ib_umem_offset(umem));
320 
321 	if (ret < 0)
322 		return ret;
323 	else if (ret != length)
324 		return -EINVAL;
325 	else
326 		return 0;
327 }
328 EXPORT_SYMBOL(ib_umem_copy_from);
329