1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2022, Microsoft Corporation. All rights reserved.
4  */
5 
6 #include "mana_ib.h"
7 
8 #define VALID_MR_FLAGS                                                         \
9 	(IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ)
10 
11 #define VALID_DMA_MR_FLAGS (IB_ACCESS_LOCAL_WRITE)
12 
13 static enum gdma_mr_access_flags
mana_ib_verbs_to_gdma_access_flags(int access_flags)14 mana_ib_verbs_to_gdma_access_flags(int access_flags)
15 {
16 	enum gdma_mr_access_flags flags = GDMA_ACCESS_FLAG_LOCAL_READ;
17 
18 	if (access_flags & IB_ACCESS_LOCAL_WRITE)
19 		flags |= GDMA_ACCESS_FLAG_LOCAL_WRITE;
20 
21 	if (access_flags & IB_ACCESS_REMOTE_WRITE)
22 		flags |= GDMA_ACCESS_FLAG_REMOTE_WRITE;
23 
24 	if (access_flags & IB_ACCESS_REMOTE_READ)
25 		flags |= GDMA_ACCESS_FLAG_REMOTE_READ;
26 
27 	return flags;
28 }
29 
mana_ib_gd_create_mr(struct mana_ib_dev * dev,struct mana_ib_mr * mr,struct gdma_create_mr_params * mr_params)30 static int mana_ib_gd_create_mr(struct mana_ib_dev *dev, struct mana_ib_mr *mr,
31 				struct gdma_create_mr_params *mr_params)
32 {
33 	struct gdma_create_mr_response resp = {};
34 	struct gdma_create_mr_request req = {};
35 	struct gdma_context *gc = mdev_to_gc(dev);
36 	int err;
37 
38 	mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req),
39 			     sizeof(resp));
40 	req.pd_handle = mr_params->pd_handle;
41 	req.mr_type = mr_params->mr_type;
42 
43 	switch (mr_params->mr_type) {
44 	case GDMA_MR_TYPE_GPA:
45 		break;
46 	case GDMA_MR_TYPE_GVA:
47 		req.gva.dma_region_handle = mr_params->gva.dma_region_handle;
48 		req.gva.virtual_address = mr_params->gva.virtual_address;
49 		req.gva.access_flags = mr_params->gva.access_flags;
50 		break;
51 
52 	default:
53 		ibdev_dbg(&dev->ib_dev,
54 			  "invalid param (GDMA_MR_TYPE) passed, type %d\n",
55 			  req.mr_type);
56 		return -EINVAL;
57 	}
58 
59 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
60 
61 	if (err || resp.hdr.status) {
62 		ibdev_dbg(&dev->ib_dev, "Failed to create mr %d, %u", err,
63 			  resp.hdr.status);
64 		if (!err)
65 			err = -EPROTO;
66 
67 		return err;
68 	}
69 
70 	mr->ibmr.lkey = resp.lkey;
71 	mr->ibmr.rkey = resp.rkey;
72 	mr->mr_handle = resp.mr_handle;
73 
74 	return 0;
75 }
76 
mana_ib_gd_destroy_mr(struct mana_ib_dev * dev,u64 mr_handle)77 static int mana_ib_gd_destroy_mr(struct mana_ib_dev *dev, u64 mr_handle)
78 {
79 	struct gdma_destroy_mr_response resp = {};
80 	struct gdma_destroy_mr_request req = {};
81 	struct gdma_context *gc = mdev_to_gc(dev);
82 	int err;
83 
84 	mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_MR, sizeof(req),
85 			     sizeof(resp));
86 
87 	req.mr_handle = mr_handle;
88 
89 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
90 	if (err || resp.hdr.status) {
91 		dev_err(gc->dev, "Failed to destroy MR: %d, 0x%x\n", err,
92 			resp.hdr.status);
93 		if (!err)
94 			err = -EPROTO;
95 		return err;
96 	}
97 
98 	return 0;
99 }
100 
mana_ib_reg_user_mr(struct ib_pd * ibpd,u64 start,u64 length,u64 iova,int access_flags,struct ib_udata * udata)101 struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
102 				  u64 iova, int access_flags,
103 				  struct ib_udata *udata)
104 {
105 	struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
106 	struct gdma_create_mr_params mr_params = {};
107 	struct ib_device *ibdev = ibpd->device;
108 	struct mana_ib_dev *dev;
109 	struct mana_ib_mr *mr;
110 	u64 dma_region_handle;
111 	int err;
112 
113 	dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
114 
115 	ibdev_dbg(ibdev,
116 		  "start 0x%llx, iova 0x%llx length 0x%llx access_flags 0x%x",
117 		  start, iova, length, access_flags);
118 
119 	access_flags &= ~IB_ACCESS_OPTIONAL;
120 	if (access_flags & ~VALID_MR_FLAGS)
121 		return ERR_PTR(-EINVAL);
122 
123 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
124 	if (!mr)
125 		return ERR_PTR(-ENOMEM);
126 
127 	mr->umem = ib_umem_get(ibdev, start, length, access_flags);
128 	if (IS_ERR(mr->umem)) {
129 		err = PTR_ERR(mr->umem);
130 		ibdev_dbg(ibdev,
131 			  "Failed to get umem for register user-mr, %d\n", err);
132 		goto err_free;
133 	}
134 
135 	err = mana_ib_create_dma_region(dev, mr->umem, &dma_region_handle, iova);
136 	if (err) {
137 		ibdev_dbg(ibdev, "Failed create dma region for user-mr, %d\n",
138 			  err);
139 		goto err_umem;
140 	}
141 
142 	ibdev_dbg(ibdev,
143 		  "created dma region for user-mr 0x%llx\n",
144 		  dma_region_handle);
145 
146 	mr_params.pd_handle = pd->pd_handle;
147 	mr_params.mr_type = GDMA_MR_TYPE_GVA;
148 	mr_params.gva.dma_region_handle = dma_region_handle;
149 	mr_params.gva.virtual_address = iova;
150 	mr_params.gva.access_flags =
151 		mana_ib_verbs_to_gdma_access_flags(access_flags);
152 
153 	err = mana_ib_gd_create_mr(dev, mr, &mr_params);
154 	if (err)
155 		goto err_dma_region;
156 
157 	/*
158 	 * There is no need to keep track of dma_region_handle after MR is
159 	 * successfully created. The dma_region_handle is tracked in the PF
160 	 * as part of the lifecycle of this MR.
161 	 */
162 
163 	return &mr->ibmr;
164 
165 err_dma_region:
166 	mana_gd_destroy_dma_region(mdev_to_gc(dev), dma_region_handle);
167 
168 err_umem:
169 	ib_umem_release(mr->umem);
170 
171 err_free:
172 	kfree(mr);
173 	return ERR_PTR(err);
174 }
175 
mana_ib_reg_user_mr_dmabuf(struct ib_pd * ibpd,u64 start,u64 length,u64 iova,int fd,int access_flags,struct uverbs_attr_bundle * attrs)176 struct ib_mr *mana_ib_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, u64 length,
177 					 u64 iova, int fd, int access_flags,
178 					 struct uverbs_attr_bundle *attrs)
179 {
180 	struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
181 	struct gdma_create_mr_params mr_params = {};
182 	struct ib_device *ibdev = ibpd->device;
183 	struct ib_umem_dmabuf *umem_dmabuf;
184 	struct mana_ib_dev *dev;
185 	struct mana_ib_mr *mr;
186 	u64 dma_region_handle;
187 	int err;
188 
189 	dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
190 
191 	access_flags &= ~IB_ACCESS_OPTIONAL;
192 	if (access_flags & ~VALID_MR_FLAGS)
193 		return ERR_PTR(-EOPNOTSUPP);
194 
195 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
196 	if (!mr)
197 		return ERR_PTR(-ENOMEM);
198 
199 	umem_dmabuf = ib_umem_dmabuf_get_pinned(ibdev, start, length, fd, access_flags);
200 	if (IS_ERR(umem_dmabuf)) {
201 		err = PTR_ERR(umem_dmabuf);
202 		ibdev_dbg(ibdev, "Failed to get dmabuf umem, %d\n", err);
203 		goto err_free;
204 	}
205 
206 	mr->umem = &umem_dmabuf->umem;
207 
208 	err = mana_ib_create_dma_region(dev, mr->umem, &dma_region_handle, iova);
209 	if (err) {
210 		ibdev_dbg(ibdev, "Failed create dma region for user-mr, %d\n",
211 			  err);
212 		goto err_umem;
213 	}
214 
215 	mr_params.pd_handle = pd->pd_handle;
216 	mr_params.mr_type = GDMA_MR_TYPE_GVA;
217 	mr_params.gva.dma_region_handle = dma_region_handle;
218 	mr_params.gva.virtual_address = iova;
219 	mr_params.gva.access_flags =
220 		mana_ib_verbs_to_gdma_access_flags(access_flags);
221 
222 	err = mana_ib_gd_create_mr(dev, mr, &mr_params);
223 	if (err)
224 		goto err_dma_region;
225 
226 	/*
227 	 * There is no need to keep track of dma_region_handle after MR is
228 	 * successfully created. The dma_region_handle is tracked in the PF
229 	 * as part of the lifecycle of this MR.
230 	 */
231 
232 	return &mr->ibmr;
233 
234 err_dma_region:
235 	mana_gd_destroy_dma_region(mdev_to_gc(dev), dma_region_handle);
236 
237 err_umem:
238 	ib_umem_release(mr->umem);
239 
240 err_free:
241 	kfree(mr);
242 	return ERR_PTR(err);
243 }
244 
mana_ib_get_dma_mr(struct ib_pd * ibpd,int access_flags)245 struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags)
246 {
247 	struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
248 	struct gdma_create_mr_params mr_params = {};
249 	struct ib_device *ibdev = ibpd->device;
250 	struct mana_ib_dev *dev;
251 	struct mana_ib_mr *mr;
252 	int err;
253 
254 	dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
255 
256 	if (access_flags & ~VALID_DMA_MR_FLAGS)
257 		return ERR_PTR(-EINVAL);
258 
259 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
260 	if (!mr)
261 		return ERR_PTR(-ENOMEM);
262 
263 	mr_params.pd_handle = pd->pd_handle;
264 	mr_params.mr_type = GDMA_MR_TYPE_GPA;
265 
266 	err = mana_ib_gd_create_mr(dev, mr, &mr_params);
267 	if (err)
268 		goto err_free;
269 
270 	return &mr->ibmr;
271 
272 err_free:
273 	kfree(mr);
274 	return ERR_PTR(err);
275 }
276 
mana_ib_dereg_mr(struct ib_mr * ibmr,struct ib_udata * udata)277 int mana_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
278 {
279 	struct mana_ib_mr *mr = container_of(ibmr, struct mana_ib_mr, ibmr);
280 	struct ib_device *ibdev = ibmr->device;
281 	struct mana_ib_dev *dev;
282 	int err;
283 
284 	dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
285 
286 	err = mana_ib_gd_destroy_mr(dev, mr->mr_handle);
287 	if (err)
288 		return err;
289 
290 	if (mr->umem)
291 		ib_umem_release(mr->umem);
292 
293 	kfree(mr);
294 
295 	return 0;
296 }
297