1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2022, Microsoft Corporation. All rights reserved.
4  */
5 
6 #include "mana_ib.h"
7 
mana_ib_create_cq(struct ib_cq * ibcq,const struct ib_cq_init_attr * attr,struct uverbs_attr_bundle * attrs)8 int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
9 		      struct uverbs_attr_bundle *attrs)
10 {
11 	struct ib_udata *udata = &attrs->driver_udata;
12 	struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
13 	struct mana_ib_create_cq_resp resp = {};
14 	struct mana_ib_ucontext *mana_ucontext;
15 	struct ib_device *ibdev = ibcq->device;
16 	struct mana_ib_create_cq ucmd = {};
17 	struct mana_ib_dev *mdev;
18 	struct gdma_context *gc;
19 	bool is_rnic_cq;
20 	u32 doorbell;
21 	u32 buf_size;
22 	int err;
23 
24 	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
25 	gc = mdev_to_gc(mdev);
26 
27 	cq->comp_vector = attr->comp_vector % ibdev->num_comp_vectors;
28 	cq->cq_handle = INVALID_MANA_HANDLE;
29 
30 	if (udata) {
31 		if (udata->inlen < offsetof(struct mana_ib_create_cq, flags))
32 			return -EINVAL;
33 
34 		err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
35 		if (err) {
36 			ibdev_dbg(ibdev, "Failed to copy from udata for create cq, %d\n", err);
37 			return err;
38 		}
39 
40 		is_rnic_cq = !!(ucmd.flags & MANA_IB_CREATE_RNIC_CQ);
41 
42 		if ((!is_rnic_cq && attr->cqe > mdev->adapter_caps.max_qp_wr) ||
43 		    attr->cqe > U32_MAX / COMP_ENTRY_SIZE) {
44 			ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe);
45 			return -EINVAL;
46 		}
47 
48 		cq->cqe = attr->cqe;
49 		err = mana_ib_create_queue(mdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE,
50 					   &cq->queue);
51 		if (err) {
52 			ibdev_dbg(ibdev, "Failed to create queue for create cq, %d\n", err);
53 			return err;
54 		}
55 
56 		mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext,
57 							  ibucontext);
58 		doorbell = mana_ucontext->doorbell;
59 	} else {
60 		is_rnic_cq = true;
61 		buf_size = MANA_PAGE_ALIGN(roundup_pow_of_two(attr->cqe * COMP_ENTRY_SIZE));
62 		cq->cqe = buf_size / COMP_ENTRY_SIZE;
63 		err = mana_ib_create_kernel_queue(mdev, buf_size, GDMA_CQ, &cq->queue);
64 		if (err) {
65 			ibdev_dbg(ibdev, "Failed to create kernel queue for create cq, %d\n", err);
66 			return err;
67 		}
68 		doorbell = gc->mana_ib.doorbell;
69 	}
70 
71 	if (is_rnic_cq) {
72 		err = mana_ib_gd_create_cq(mdev, cq, doorbell);
73 		if (err) {
74 			ibdev_dbg(ibdev, "Failed to create RNIC cq, %d\n", err);
75 			goto err_destroy_queue;
76 		}
77 
78 		err = mana_ib_install_cq_cb(mdev, cq);
79 		if (err) {
80 			ibdev_dbg(ibdev, "Failed to install cq callback, %d\n", err);
81 			goto err_destroy_rnic_cq;
82 		}
83 	}
84 
85 	if (udata) {
86 		resp.cqid = cq->queue.id;
87 		err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen));
88 		if (err) {
89 			ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err);
90 			goto err_remove_cq_cb;
91 		}
92 	}
93 
94 	spin_lock_init(&cq->cq_lock);
95 	INIT_LIST_HEAD(&cq->list_send_qp);
96 	INIT_LIST_HEAD(&cq->list_recv_qp);
97 
98 	return 0;
99 
100 err_remove_cq_cb:
101 	mana_ib_remove_cq_cb(mdev, cq);
102 err_destroy_rnic_cq:
103 	mana_ib_gd_destroy_cq(mdev, cq);
104 err_destroy_queue:
105 	mana_ib_destroy_queue(mdev, &cq->queue);
106 
107 	return err;
108 }
109 
mana_ib_destroy_cq(struct ib_cq * ibcq,struct ib_udata * udata)110 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
111 {
112 	struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
113 	struct ib_device *ibdev = ibcq->device;
114 	struct mana_ib_dev *mdev;
115 
116 	mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
117 
118 	mana_ib_remove_cq_cb(mdev, cq);
119 
120 	/* Ignore return code as there is not much we can do about it.
121 	 * The error message is printed inside.
122 	 */
123 	mana_ib_gd_destroy_cq(mdev, cq);
124 
125 	mana_ib_destroy_queue(mdev, &cq->queue);
126 
127 	return 0;
128 }
129 
mana_ib_cq_handler(void * ctx,struct gdma_queue * gdma_cq)130 static void mana_ib_cq_handler(void *ctx, struct gdma_queue *gdma_cq)
131 {
132 	struct mana_ib_cq *cq = ctx;
133 
134 	if (cq->ibcq.comp_handler)
135 		cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
136 }
137 
mana_ib_install_cq_cb(struct mana_ib_dev * mdev,struct mana_ib_cq * cq)138 int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
139 {
140 	struct gdma_context *gc = mdev_to_gc(mdev);
141 	struct gdma_queue *gdma_cq;
142 
143 	if (cq->queue.id >= gc->max_num_cqs)
144 		return -EINVAL;
145 	/* Create CQ table entry */
146 	WARN_ON(gc->cq_table[cq->queue.id]);
147 	if (cq->queue.kmem)
148 		gdma_cq = cq->queue.kmem;
149 	else
150 		gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
151 	if (!gdma_cq)
152 		return -ENOMEM;
153 
154 	gdma_cq->cq.context = cq;
155 	gdma_cq->type = GDMA_CQ;
156 	gdma_cq->cq.callback = mana_ib_cq_handler;
157 	gdma_cq->id = cq->queue.id;
158 	gc->cq_table[cq->queue.id] = gdma_cq;
159 	return 0;
160 }
161 
mana_ib_remove_cq_cb(struct mana_ib_dev * mdev,struct mana_ib_cq * cq)162 void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
163 {
164 	struct gdma_context *gc = mdev_to_gc(mdev);
165 
166 	if (cq->queue.id >= gc->max_num_cqs || cq->queue.id == INVALID_QUEUE_ID)
167 		return;
168 
169 	if (cq->queue.kmem)
170 	/* Then it will be cleaned and removed by the mana */
171 		return;
172 
173 	kfree(gc->cq_table[cq->queue.id]);
174 	gc->cq_table[cq->queue.id] = NULL;
175 }
176 
mana_ib_arm_cq(struct ib_cq * ibcq,enum ib_cq_notify_flags flags)177 int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
178 {
179 	struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
180 	struct gdma_queue *gdma_cq = cq->queue.kmem;
181 
182 	if (!gdma_cq)
183 		return -EINVAL;
184 
185 	mana_gd_ring_cq(gdma_cq, SET_ARM_BIT);
186 	return 0;
187 }
188 
handle_ud_sq_cqe(struct mana_ib_qp * qp,struct gdma_comp * cqe)189 static inline void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe)
190 {
191 	struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
192 	struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem;
193 	struct ud_sq_shadow_wqe *shadow_wqe;
194 
195 	shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq);
196 	if (!shadow_wqe)
197 		return;
198 
199 	shadow_wqe->header.error_code = rdma_cqe->ud_send.vendor_error;
200 
201 	wq->tail += shadow_wqe->header.posted_wqe_size;
202 	shadow_queue_advance_next_to_complete(&qp->shadow_sq);
203 }
204 
handle_ud_rq_cqe(struct mana_ib_qp * qp,struct gdma_comp * cqe)205 static inline void handle_ud_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe)
206 {
207 	struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
208 	struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].kmem;
209 	struct ud_rq_shadow_wqe *shadow_wqe;
210 
211 	shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_rq);
212 	if (!shadow_wqe)
213 		return;
214 
215 	shadow_wqe->byte_len = rdma_cqe->ud_recv.msg_len;
216 	shadow_wqe->src_qpn = rdma_cqe->ud_recv.src_qpn;
217 	shadow_wqe->header.error_code = IB_WC_SUCCESS;
218 
219 	wq->tail += shadow_wqe->header.posted_wqe_size;
220 	shadow_queue_advance_next_to_complete(&qp->shadow_rq);
221 }
222 
mana_handle_cqe(struct mana_ib_dev * mdev,struct gdma_comp * cqe)223 static void mana_handle_cqe(struct mana_ib_dev *mdev, struct gdma_comp *cqe)
224 {
225 	struct mana_ib_qp *qp = mana_get_qp_ref(mdev, cqe->wq_num, cqe->is_sq);
226 
227 	if (!qp)
228 		return;
229 
230 	if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) {
231 		if (cqe->is_sq)
232 			handle_ud_sq_cqe(qp, cqe);
233 		else
234 			handle_ud_rq_cqe(qp, cqe);
235 	}
236 
237 	mana_put_qp_ref(qp);
238 }
239 
fill_verbs_from_shadow_wqe(struct mana_ib_qp * qp,struct ib_wc * wc,const struct shadow_wqe_header * shadow_wqe)240 static void fill_verbs_from_shadow_wqe(struct mana_ib_qp *qp, struct ib_wc *wc,
241 				       const struct shadow_wqe_header *shadow_wqe)
242 {
243 	const struct ud_rq_shadow_wqe *ud_wqe = (const struct ud_rq_shadow_wqe *)shadow_wqe;
244 
245 	wc->wr_id = shadow_wqe->wr_id;
246 	wc->status = shadow_wqe->error_code;
247 	wc->opcode = shadow_wqe->opcode;
248 	wc->vendor_err = shadow_wqe->error_code;
249 	wc->wc_flags = 0;
250 	wc->qp = &qp->ibqp;
251 	wc->pkey_index = 0;
252 
253 	if (shadow_wqe->opcode == IB_WC_RECV) {
254 		wc->byte_len = ud_wqe->byte_len;
255 		wc->src_qp = ud_wqe->src_qpn;
256 		wc->wc_flags |= IB_WC_GRH;
257 	}
258 }
259 
mana_process_completions(struct mana_ib_cq * cq,int nwc,struct ib_wc * wc)260 static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc *wc)
261 {
262 	struct shadow_wqe_header *shadow_wqe;
263 	struct mana_ib_qp *qp;
264 	int wc_index = 0;
265 
266 	/* process send shadow queue completions  */
267 	list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) {
268 		while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq))
269 				!= NULL) {
270 			if (wc_index >= nwc)
271 				goto out;
272 
273 			fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe);
274 			shadow_queue_advance_consumer(&qp->shadow_sq);
275 			wc_index++;
276 		}
277 	}
278 
279 	/* process recv shadow queue completions */
280 	list_for_each_entry(qp, &cq->list_recv_qp, cq_recv_list) {
281 		while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq))
282 				!= NULL) {
283 			if (wc_index >= nwc)
284 				goto out;
285 
286 			fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe);
287 			shadow_queue_advance_consumer(&qp->shadow_rq);
288 			wc_index++;
289 		}
290 	}
291 
292 out:
293 	return wc_index;
294 }
295 
mana_ib_poll_cq(struct ib_cq * ibcq,int num_entries,struct ib_wc * wc)296 int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
297 {
298 	struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
299 	struct mana_ib_dev *mdev = container_of(ibcq->device, struct mana_ib_dev, ib_dev);
300 	struct gdma_queue *queue = cq->queue.kmem;
301 	struct gdma_comp gdma_cqe;
302 	unsigned long flags;
303 	int num_polled = 0;
304 	int comp_read, i;
305 
306 	spin_lock_irqsave(&cq->cq_lock, flags);
307 	for (i = 0; i < num_entries; i++) {
308 		comp_read = mana_gd_poll_cq(queue, &gdma_cqe, 1);
309 		if (comp_read < 1)
310 			break;
311 		mana_handle_cqe(mdev, &gdma_cqe);
312 	}
313 
314 	num_polled = mana_process_completions(cq, num_entries, wc);
315 	spin_unlock_irqrestore(&cq->cq_lock, flags);
316 
317 	return num_polled;
318 }
319