1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved.
4 */
5
6 #include "mana_ib.h"
7
mana_ib_create_cq(struct ib_cq * ibcq,const struct ib_cq_init_attr * attr,struct uverbs_attr_bundle * attrs)8 int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
9 struct uverbs_attr_bundle *attrs)
10 {
11 struct ib_udata *udata = &attrs->driver_udata;
12 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
13 struct mana_ib_create_cq_resp resp = {};
14 struct mana_ib_ucontext *mana_ucontext;
15 struct ib_device *ibdev = ibcq->device;
16 struct mana_ib_create_cq ucmd = {};
17 struct mana_ib_dev *mdev;
18 struct gdma_context *gc;
19 bool is_rnic_cq;
20 u32 doorbell;
21 u32 buf_size;
22 int err;
23
24 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
25 gc = mdev_to_gc(mdev);
26
27 cq->comp_vector = attr->comp_vector % ibdev->num_comp_vectors;
28 cq->cq_handle = INVALID_MANA_HANDLE;
29
30 if (udata) {
31 if (udata->inlen < offsetof(struct mana_ib_create_cq, flags))
32 return -EINVAL;
33
34 err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
35 if (err) {
36 ibdev_dbg(ibdev, "Failed to copy from udata for create cq, %d\n", err);
37 return err;
38 }
39
40 is_rnic_cq = !!(ucmd.flags & MANA_IB_CREATE_RNIC_CQ);
41
42 if ((!is_rnic_cq && attr->cqe > mdev->adapter_caps.max_qp_wr) ||
43 attr->cqe > U32_MAX / COMP_ENTRY_SIZE) {
44 ibdev_dbg(ibdev, "CQE %d exceeding limit\n", attr->cqe);
45 return -EINVAL;
46 }
47
48 cq->cqe = attr->cqe;
49 err = mana_ib_create_queue(mdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE,
50 &cq->queue);
51 if (err) {
52 ibdev_dbg(ibdev, "Failed to create queue for create cq, %d\n", err);
53 return err;
54 }
55
56 mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext,
57 ibucontext);
58 doorbell = mana_ucontext->doorbell;
59 } else {
60 is_rnic_cq = true;
61 buf_size = MANA_PAGE_ALIGN(roundup_pow_of_two(attr->cqe * COMP_ENTRY_SIZE));
62 cq->cqe = buf_size / COMP_ENTRY_SIZE;
63 err = mana_ib_create_kernel_queue(mdev, buf_size, GDMA_CQ, &cq->queue);
64 if (err) {
65 ibdev_dbg(ibdev, "Failed to create kernel queue for create cq, %d\n", err);
66 return err;
67 }
68 doorbell = gc->mana_ib.doorbell;
69 }
70
71 if (is_rnic_cq) {
72 err = mana_ib_gd_create_cq(mdev, cq, doorbell);
73 if (err) {
74 ibdev_dbg(ibdev, "Failed to create RNIC cq, %d\n", err);
75 goto err_destroy_queue;
76 }
77
78 err = mana_ib_install_cq_cb(mdev, cq);
79 if (err) {
80 ibdev_dbg(ibdev, "Failed to install cq callback, %d\n", err);
81 goto err_destroy_rnic_cq;
82 }
83 }
84
85 if (udata) {
86 resp.cqid = cq->queue.id;
87 err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen));
88 if (err) {
89 ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err);
90 goto err_remove_cq_cb;
91 }
92 }
93
94 spin_lock_init(&cq->cq_lock);
95 INIT_LIST_HEAD(&cq->list_send_qp);
96 INIT_LIST_HEAD(&cq->list_recv_qp);
97
98 return 0;
99
100 err_remove_cq_cb:
101 mana_ib_remove_cq_cb(mdev, cq);
102 err_destroy_rnic_cq:
103 mana_ib_gd_destroy_cq(mdev, cq);
104 err_destroy_queue:
105 mana_ib_destroy_queue(mdev, &cq->queue);
106
107 return err;
108 }
109
mana_ib_destroy_cq(struct ib_cq * ibcq,struct ib_udata * udata)110 int mana_ib_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
111 {
112 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
113 struct ib_device *ibdev = ibcq->device;
114 struct mana_ib_dev *mdev;
115
116 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev);
117
118 mana_ib_remove_cq_cb(mdev, cq);
119
120 /* Ignore return code as there is not much we can do about it.
121 * The error message is printed inside.
122 */
123 mana_ib_gd_destroy_cq(mdev, cq);
124
125 mana_ib_destroy_queue(mdev, &cq->queue);
126
127 return 0;
128 }
129
mana_ib_cq_handler(void * ctx,struct gdma_queue * gdma_cq)130 static void mana_ib_cq_handler(void *ctx, struct gdma_queue *gdma_cq)
131 {
132 struct mana_ib_cq *cq = ctx;
133
134 if (cq->ibcq.comp_handler)
135 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
136 }
137
mana_ib_install_cq_cb(struct mana_ib_dev * mdev,struct mana_ib_cq * cq)138 int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
139 {
140 struct gdma_context *gc = mdev_to_gc(mdev);
141 struct gdma_queue *gdma_cq;
142
143 if (cq->queue.id >= gc->max_num_cqs)
144 return -EINVAL;
145 /* Create CQ table entry */
146 WARN_ON(gc->cq_table[cq->queue.id]);
147 if (cq->queue.kmem)
148 gdma_cq = cq->queue.kmem;
149 else
150 gdma_cq = kzalloc(sizeof(*gdma_cq), GFP_KERNEL);
151 if (!gdma_cq)
152 return -ENOMEM;
153
154 gdma_cq->cq.context = cq;
155 gdma_cq->type = GDMA_CQ;
156 gdma_cq->cq.callback = mana_ib_cq_handler;
157 gdma_cq->id = cq->queue.id;
158 gc->cq_table[cq->queue.id] = gdma_cq;
159 return 0;
160 }
161
mana_ib_remove_cq_cb(struct mana_ib_dev * mdev,struct mana_ib_cq * cq)162 void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
163 {
164 struct gdma_context *gc = mdev_to_gc(mdev);
165
166 if (cq->queue.id >= gc->max_num_cqs || cq->queue.id == INVALID_QUEUE_ID)
167 return;
168
169 if (cq->queue.kmem)
170 /* Then it will be cleaned and removed by the mana */
171 return;
172
173 kfree(gc->cq_table[cq->queue.id]);
174 gc->cq_table[cq->queue.id] = NULL;
175 }
176
mana_ib_arm_cq(struct ib_cq * ibcq,enum ib_cq_notify_flags flags)177 int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
178 {
179 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
180 struct gdma_queue *gdma_cq = cq->queue.kmem;
181
182 if (!gdma_cq)
183 return -EINVAL;
184
185 mana_gd_ring_cq(gdma_cq, SET_ARM_BIT);
186 return 0;
187 }
188
handle_ud_sq_cqe(struct mana_ib_qp * qp,struct gdma_comp * cqe)189 static inline void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe)
190 {
191 struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
192 struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem;
193 struct ud_sq_shadow_wqe *shadow_wqe;
194
195 shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq);
196 if (!shadow_wqe)
197 return;
198
199 shadow_wqe->header.error_code = rdma_cqe->ud_send.vendor_error;
200
201 wq->tail += shadow_wqe->header.posted_wqe_size;
202 shadow_queue_advance_next_to_complete(&qp->shadow_sq);
203 }
204
handle_ud_rq_cqe(struct mana_ib_qp * qp,struct gdma_comp * cqe)205 static inline void handle_ud_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe)
206 {
207 struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
208 struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].kmem;
209 struct ud_rq_shadow_wqe *shadow_wqe;
210
211 shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_rq);
212 if (!shadow_wqe)
213 return;
214
215 shadow_wqe->byte_len = rdma_cqe->ud_recv.msg_len;
216 shadow_wqe->src_qpn = rdma_cqe->ud_recv.src_qpn;
217 shadow_wqe->header.error_code = IB_WC_SUCCESS;
218
219 wq->tail += shadow_wqe->header.posted_wqe_size;
220 shadow_queue_advance_next_to_complete(&qp->shadow_rq);
221 }
222
mana_handle_cqe(struct mana_ib_dev * mdev,struct gdma_comp * cqe)223 static void mana_handle_cqe(struct mana_ib_dev *mdev, struct gdma_comp *cqe)
224 {
225 struct mana_ib_qp *qp = mana_get_qp_ref(mdev, cqe->wq_num, cqe->is_sq);
226
227 if (!qp)
228 return;
229
230 if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) {
231 if (cqe->is_sq)
232 handle_ud_sq_cqe(qp, cqe);
233 else
234 handle_ud_rq_cqe(qp, cqe);
235 }
236
237 mana_put_qp_ref(qp);
238 }
239
fill_verbs_from_shadow_wqe(struct mana_ib_qp * qp,struct ib_wc * wc,const struct shadow_wqe_header * shadow_wqe)240 static void fill_verbs_from_shadow_wqe(struct mana_ib_qp *qp, struct ib_wc *wc,
241 const struct shadow_wqe_header *shadow_wqe)
242 {
243 const struct ud_rq_shadow_wqe *ud_wqe = (const struct ud_rq_shadow_wqe *)shadow_wqe;
244
245 wc->wr_id = shadow_wqe->wr_id;
246 wc->status = shadow_wqe->error_code;
247 wc->opcode = shadow_wqe->opcode;
248 wc->vendor_err = shadow_wqe->error_code;
249 wc->wc_flags = 0;
250 wc->qp = &qp->ibqp;
251 wc->pkey_index = 0;
252
253 if (shadow_wqe->opcode == IB_WC_RECV) {
254 wc->byte_len = ud_wqe->byte_len;
255 wc->src_qp = ud_wqe->src_qpn;
256 wc->wc_flags |= IB_WC_GRH;
257 }
258 }
259
mana_process_completions(struct mana_ib_cq * cq,int nwc,struct ib_wc * wc)260 static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc *wc)
261 {
262 struct shadow_wqe_header *shadow_wqe;
263 struct mana_ib_qp *qp;
264 int wc_index = 0;
265
266 /* process send shadow queue completions */
267 list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) {
268 while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq))
269 != NULL) {
270 if (wc_index >= nwc)
271 goto out;
272
273 fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe);
274 shadow_queue_advance_consumer(&qp->shadow_sq);
275 wc_index++;
276 }
277 }
278
279 /* process recv shadow queue completions */
280 list_for_each_entry(qp, &cq->list_recv_qp, cq_recv_list) {
281 while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq))
282 != NULL) {
283 if (wc_index >= nwc)
284 goto out;
285
286 fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe);
287 shadow_queue_advance_consumer(&qp->shadow_rq);
288 wc_index++;
289 }
290 }
291
292 out:
293 return wc_index;
294 }
295
mana_ib_poll_cq(struct ib_cq * ibcq,int num_entries,struct ib_wc * wc)296 int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
297 {
298 struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
299 struct mana_ib_dev *mdev = container_of(ibcq->device, struct mana_ib_dev, ib_dev);
300 struct gdma_queue *queue = cq->queue.kmem;
301 struct gdma_comp gdma_cqe;
302 unsigned long flags;
303 int num_polled = 0;
304 int comp_read, i;
305
306 spin_lock_irqsave(&cq->cq_lock, flags);
307 for (i = 0; i < num_entries; i++) {
308 comp_read = mana_gd_poll_cq(queue, &gdma_cqe, 1);
309 if (comp_read < 1)
310 break;
311 mana_handle_cqe(mdev, &gdma_cqe);
312 }
313
314 num_polled = mana_process_completions(cq, num_entries, wc);
315 spin_unlock_irqrestore(&cq->cq_lock, flags);
316
317 return num_polled;
318 }
319