1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2023 Advanced Micro Devices, Inc */
3
4 #include <linux/pci.h>
5 #include <linux/vdpa.h>
6 #include <uapi/linux/vdpa.h>
7 #include <linux/virtio_pci_modern.h>
8
9 #include <linux/pds/pds_common.h>
10 #include <linux/pds/pds_core_if.h>
11 #include <linux/pds/pds_adminq.h>
12 #include <linux/pds/pds_auxbus.h>
13
14 #include "vdpa_dev.h"
15 #include "aux_drv.h"
16 #include "cmds.h"
17 #include "debugfs.h"
18
19 static u64 pds_vdpa_get_driver_features(struct vdpa_device *vdpa_dev);
20
vdpa_to_pdsv(struct vdpa_device * vdpa_dev)21 static struct pds_vdpa_device *vdpa_to_pdsv(struct vdpa_device *vdpa_dev)
22 {
23 return container_of(vdpa_dev, struct pds_vdpa_device, vdpa_dev);
24 }
25
pds_vdpa_notify_handler(struct notifier_block * nb,unsigned long ecode,void * data)26 static int pds_vdpa_notify_handler(struct notifier_block *nb,
27 unsigned long ecode,
28 void *data)
29 {
30 struct pds_vdpa_device *pdsv = container_of(nb, struct pds_vdpa_device, nb);
31 struct device *dev = &pdsv->vdpa_aux->padev->aux_dev.dev;
32
33 dev_dbg(dev, "%s: event code %lu\n", __func__, ecode);
34
35 if (ecode == PDS_EVENT_RESET || ecode == PDS_EVENT_LINK_CHANGE) {
36 if (pdsv->config_cb.callback)
37 pdsv->config_cb.callback(pdsv->config_cb.private);
38 }
39
40 return 0;
41 }
42
pds_vdpa_register_event_handler(struct pds_vdpa_device * pdsv)43 static int pds_vdpa_register_event_handler(struct pds_vdpa_device *pdsv)
44 {
45 struct device *dev = &pdsv->vdpa_aux->padev->aux_dev.dev;
46 struct notifier_block *nb = &pdsv->nb;
47 int err;
48
49 if (!nb->notifier_call) {
50 nb->notifier_call = pds_vdpa_notify_handler;
51 err = pdsc_register_notify(nb);
52 if (err) {
53 nb->notifier_call = NULL;
54 dev_err(dev, "failed to register pds event handler: %ps\n",
55 ERR_PTR(err));
56 return -EINVAL;
57 }
58 dev_dbg(dev, "pds event handler registered\n");
59 }
60
61 return 0;
62 }
63
pds_vdpa_unregister_event_handler(struct pds_vdpa_device * pdsv)64 static void pds_vdpa_unregister_event_handler(struct pds_vdpa_device *pdsv)
65 {
66 if (pdsv->nb.notifier_call) {
67 pdsc_unregister_notify(&pdsv->nb);
68 pdsv->nb.notifier_call = NULL;
69 }
70 }
71
pds_vdpa_set_vq_address(struct vdpa_device * vdpa_dev,u16 qid,u64 desc_addr,u64 driver_addr,u64 device_addr)72 static int pds_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid,
73 u64 desc_addr, u64 driver_addr, u64 device_addr)
74 {
75 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
76
77 pdsv->vqs[qid].desc_addr = desc_addr;
78 pdsv->vqs[qid].avail_addr = driver_addr;
79 pdsv->vqs[qid].used_addr = device_addr;
80
81 return 0;
82 }
83
pds_vdpa_set_vq_num(struct vdpa_device * vdpa_dev,u16 qid,u32 num)84 static void pds_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid, u32 num)
85 {
86 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
87
88 pdsv->vqs[qid].q_len = num;
89 }
90
pds_vdpa_kick_vq(struct vdpa_device * vdpa_dev,u16 qid)91 static void pds_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid)
92 {
93 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
94
95 iowrite16(qid, pdsv->vqs[qid].notify);
96 }
97
pds_vdpa_set_vq_cb(struct vdpa_device * vdpa_dev,u16 qid,struct vdpa_callback * cb)98 static void pds_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid,
99 struct vdpa_callback *cb)
100 {
101 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
102
103 pdsv->vqs[qid].event_cb = *cb;
104 }
105
pds_vdpa_isr(int irq,void * data)106 static irqreturn_t pds_vdpa_isr(int irq, void *data)
107 {
108 struct pds_vdpa_vq_info *vq;
109
110 vq = data;
111 if (vq->event_cb.callback)
112 vq->event_cb.callback(vq->event_cb.private);
113
114 return IRQ_HANDLED;
115 }
116
pds_vdpa_release_irq(struct pds_vdpa_device * pdsv,int qid)117 static void pds_vdpa_release_irq(struct pds_vdpa_device *pdsv, int qid)
118 {
119 if (pdsv->vqs[qid].irq == VIRTIO_MSI_NO_VECTOR)
120 return;
121
122 free_irq(pdsv->vqs[qid].irq, &pdsv->vqs[qid]);
123 pdsv->vqs[qid].irq = VIRTIO_MSI_NO_VECTOR;
124 }
125
pds_vdpa_set_vq_ready(struct vdpa_device * vdpa_dev,u16 qid,bool ready)126 static void pds_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev, u16 qid, bool ready)
127 {
128 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
129 struct device *dev = &pdsv->vdpa_dev.dev;
130 u64 driver_features;
131 u16 invert_idx = 0;
132 int err;
133
134 dev_dbg(dev, "%s: qid %d ready %d => %d\n",
135 __func__, qid, pdsv->vqs[qid].ready, ready);
136 if (ready == pdsv->vqs[qid].ready)
137 return;
138
139 driver_features = pds_vdpa_get_driver_features(vdpa_dev);
140 if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED))
141 invert_idx = PDS_VDPA_PACKED_INVERT_IDX;
142
143 if (ready) {
144 /* Pass vq setup info to DSC using adminq to gather up and
145 * send all info at once so FW can do its full set up in
146 * one easy operation
147 */
148 err = pds_vdpa_cmd_init_vq(pdsv, qid, invert_idx, &pdsv->vqs[qid]);
149 if (err) {
150 dev_err(dev, "Failed to init vq %d: %pe\n",
151 qid, ERR_PTR(err));
152 ready = false;
153 }
154 } else {
155 err = pds_vdpa_cmd_reset_vq(pdsv, qid, invert_idx, &pdsv->vqs[qid]);
156 if (err)
157 dev_err(dev, "%s: reset_vq failed qid %d: %pe\n",
158 __func__, qid, ERR_PTR(err));
159 }
160
161 pdsv->vqs[qid].ready = ready;
162 }
163
pds_vdpa_get_vq_ready(struct vdpa_device * vdpa_dev,u16 qid)164 static bool pds_vdpa_get_vq_ready(struct vdpa_device *vdpa_dev, u16 qid)
165 {
166 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
167
168 return pdsv->vqs[qid].ready;
169 }
170
pds_vdpa_set_vq_state(struct vdpa_device * vdpa_dev,u16 qid,const struct vdpa_vq_state * state)171 static int pds_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
172 const struct vdpa_vq_state *state)
173 {
174 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
175 struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
176 struct device *dev = &padev->aux_dev.dev;
177 u64 driver_features;
178 u16 avail;
179 u16 used;
180
181 if (pdsv->vqs[qid].ready) {
182 dev_err(dev, "Setting device position is denied while vq is enabled\n");
183 return -EINVAL;
184 }
185
186 driver_features = pds_vdpa_get_driver_features(vdpa_dev);
187 if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
188 avail = state->packed.last_avail_idx |
189 (state->packed.last_avail_counter << 15);
190 used = state->packed.last_used_idx |
191 (state->packed.last_used_counter << 15);
192
193 /* The avail and used index are stored with the packed wrap
194 * counter bit inverted. This way, in case set_vq_state is
195 * not called, the initial value can be set to zero prior to
196 * feature negotiation, and it is good for both packed and
197 * split vq.
198 */
199 avail ^= PDS_VDPA_PACKED_INVERT_IDX;
200 used ^= PDS_VDPA_PACKED_INVERT_IDX;
201 } else {
202 avail = state->split.avail_index;
203 /* state->split does not provide a used_index:
204 * the vq will be set to "empty" here, and the vq will read
205 * the current used index the next time the vq is kicked.
206 */
207 used = avail;
208 }
209
210 if (used != avail) {
211 dev_dbg(dev, "Setting used equal to avail, for interoperability\n");
212 used = avail;
213 }
214
215 pdsv->vqs[qid].avail_idx = avail;
216 pdsv->vqs[qid].used_idx = used;
217
218 return 0;
219 }
220
pds_vdpa_get_vq_state(struct vdpa_device * vdpa_dev,u16 qid,struct vdpa_vq_state * state)221 static int pds_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
222 struct vdpa_vq_state *state)
223 {
224 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
225 struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;
226 struct device *dev = &padev->aux_dev.dev;
227 u64 driver_features;
228 u16 avail;
229 u16 used;
230
231 if (pdsv->vqs[qid].ready) {
232 dev_err(dev, "Getting device position is denied while vq is enabled\n");
233 return -EINVAL;
234 }
235
236 avail = pdsv->vqs[qid].avail_idx;
237 used = pdsv->vqs[qid].used_idx;
238
239 driver_features = pds_vdpa_get_driver_features(vdpa_dev);
240 if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
241 avail ^= PDS_VDPA_PACKED_INVERT_IDX;
242 used ^= PDS_VDPA_PACKED_INVERT_IDX;
243
244 state->packed.last_avail_idx = avail & 0x7fff;
245 state->packed.last_avail_counter = avail >> 15;
246 state->packed.last_used_idx = used & 0x7fff;
247 state->packed.last_used_counter = used >> 15;
248 } else {
249 state->split.avail_index = avail;
250 /* state->split does not provide a used_index. */
251 }
252
253 return 0;
254 }
255
256 static struct vdpa_notification_area
pds_vdpa_get_vq_notification(struct vdpa_device * vdpa_dev,u16 qid)257 pds_vdpa_get_vq_notification(struct vdpa_device *vdpa_dev, u16 qid)
258 {
259 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
260 struct virtio_pci_modern_device *vd_mdev;
261 struct vdpa_notification_area area;
262
263 area.addr = pdsv->vqs[qid].notify_pa;
264
265 vd_mdev = &pdsv->vdpa_aux->vd_mdev;
266 if (!vd_mdev->notify_offset_multiplier)
267 area.size = PDS_PAGE_SIZE;
268 else
269 area.size = vd_mdev->notify_offset_multiplier;
270
271 return area;
272 }
273
pds_vdpa_get_vq_irq(struct vdpa_device * vdpa_dev,u16 qid)274 static int pds_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev, u16 qid)
275 {
276 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
277
278 return pdsv->vqs[qid].irq;
279 }
280
pds_vdpa_get_vq_align(struct vdpa_device * vdpa_dev)281 static u32 pds_vdpa_get_vq_align(struct vdpa_device *vdpa_dev)
282 {
283 return PDS_PAGE_SIZE;
284 }
285
pds_vdpa_get_vq_group(struct vdpa_device * vdpa_dev,u16 idx)286 static u32 pds_vdpa_get_vq_group(struct vdpa_device *vdpa_dev, u16 idx)
287 {
288 return 0;
289 }
290
pds_vdpa_get_device_features(struct vdpa_device * vdpa_dev)291 static u64 pds_vdpa_get_device_features(struct vdpa_device *vdpa_dev)
292 {
293 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
294
295 return pdsv->supported_features;
296 }
297
pds_vdpa_set_driver_features(struct vdpa_device * vdpa_dev,u64 features)298 static int pds_vdpa_set_driver_features(struct vdpa_device *vdpa_dev, u64 features)
299 {
300 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
301 struct device *dev = &pdsv->vdpa_dev.dev;
302 u64 driver_features;
303 u64 nego_features;
304 u64 hw_features;
305 u64 missing;
306
307 if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM)) && features) {
308 dev_err(dev, "VIRTIO_F_ACCESS_PLATFORM is not negotiated\n");
309 return -EOPNOTSUPP;
310 }
311
312 /* Check for valid feature bits */
313 nego_features = features & pdsv->supported_features;
314 missing = features & ~nego_features;
315 if (missing) {
316 dev_err(dev, "Can't support all requested features in %#llx, missing %#llx features\n",
317 features, missing);
318 return -EOPNOTSUPP;
319 }
320
321 pdsv->negotiated_features = nego_features;
322
323 driver_features = pds_vdpa_get_driver_features(vdpa_dev);
324 dev_dbg(dev, "%s: %#llx => %#llx\n",
325 __func__, driver_features, nego_features);
326
327 /* if we're faking the F_MAC, strip it before writing to device */
328 hw_features = le64_to_cpu(pdsv->vdpa_aux->ident.hw_features);
329 if (!(hw_features & BIT_ULL(VIRTIO_NET_F_MAC)))
330 nego_features &= ~BIT_ULL(VIRTIO_NET_F_MAC);
331
332 if (driver_features == nego_features)
333 return 0;
334
335 vp_modern_set_features(&pdsv->vdpa_aux->vd_mdev, nego_features);
336
337 return 0;
338 }
339
pds_vdpa_get_driver_features(struct vdpa_device * vdpa_dev)340 static u64 pds_vdpa_get_driver_features(struct vdpa_device *vdpa_dev)
341 {
342 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
343
344 return pdsv->negotiated_features;
345 }
346
pds_vdpa_set_config_cb(struct vdpa_device * vdpa_dev,struct vdpa_callback * cb)347 static void pds_vdpa_set_config_cb(struct vdpa_device *vdpa_dev,
348 struct vdpa_callback *cb)
349 {
350 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
351
352 pdsv->config_cb.callback = cb->callback;
353 pdsv->config_cb.private = cb->private;
354 }
355
pds_vdpa_get_vq_num_max(struct vdpa_device * vdpa_dev)356 static u16 pds_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev)
357 {
358 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
359
360 /* qemu has assert() that vq_num_max <= VIRTQUEUE_MAX_SIZE (1024) */
361 return min_t(u16, 1024, BIT(le16_to_cpu(pdsv->vdpa_aux->ident.max_qlen)));
362 }
363
pds_vdpa_get_device_id(struct vdpa_device * vdpa_dev)364 static u32 pds_vdpa_get_device_id(struct vdpa_device *vdpa_dev)
365 {
366 return VIRTIO_ID_NET;
367 }
368
pds_vdpa_get_vendor_id(struct vdpa_device * vdpa_dev)369 static u32 pds_vdpa_get_vendor_id(struct vdpa_device *vdpa_dev)
370 {
371 return PCI_VENDOR_ID_PENSANDO;
372 }
373
pds_vdpa_get_status(struct vdpa_device * vdpa_dev)374 static u8 pds_vdpa_get_status(struct vdpa_device *vdpa_dev)
375 {
376 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
377
378 return vp_modern_get_status(&pdsv->vdpa_aux->vd_mdev);
379 }
380
pds_vdpa_request_irqs(struct pds_vdpa_device * pdsv)381 static int pds_vdpa_request_irqs(struct pds_vdpa_device *pdsv)
382 {
383 struct pci_dev *pdev = pdsv->vdpa_aux->padev->vf_pdev;
384 struct pds_vdpa_aux *vdpa_aux = pdsv->vdpa_aux;
385 struct device *dev = &pdsv->vdpa_dev.dev;
386 int max_vq, nintrs, qid, err;
387
388 max_vq = vdpa_aux->vdpa_mdev.max_supported_vqs;
389
390 nintrs = pci_alloc_irq_vectors(pdev, max_vq, max_vq, PCI_IRQ_MSIX);
391 if (nintrs < 0) {
392 dev_err(dev, "Couldn't get %d msix vectors: %pe\n",
393 max_vq, ERR_PTR(nintrs));
394 return nintrs;
395 }
396
397 for (qid = 0; qid < pdsv->num_vqs; ++qid) {
398 int irq = pci_irq_vector(pdev, qid);
399
400 snprintf(pdsv->vqs[qid].irq_name, sizeof(pdsv->vqs[qid].irq_name),
401 "vdpa-%s-%d", dev_name(dev), qid);
402
403 err = request_irq(irq, pds_vdpa_isr, 0,
404 pdsv->vqs[qid].irq_name,
405 &pdsv->vqs[qid]);
406 if (err) {
407 dev_err(dev, "%s: no irq for qid %d: %pe\n",
408 __func__, qid, ERR_PTR(err));
409 goto err_release;
410 }
411
412 pdsv->vqs[qid].irq = irq;
413 }
414
415 vdpa_aux->nintrs = nintrs;
416
417 return 0;
418
419 err_release:
420 while (qid--)
421 pds_vdpa_release_irq(pdsv, qid);
422
423 pci_free_irq_vectors(pdev);
424
425 vdpa_aux->nintrs = 0;
426
427 return err;
428 }
429
pds_vdpa_release_irqs(struct pds_vdpa_device * pdsv)430 static void pds_vdpa_release_irqs(struct pds_vdpa_device *pdsv)
431 {
432 struct pci_dev *pdev = pdsv->vdpa_aux->padev->vf_pdev;
433 struct pds_vdpa_aux *vdpa_aux = pdsv->vdpa_aux;
434 int qid;
435
436 if (!vdpa_aux->nintrs)
437 return;
438
439 for (qid = 0; qid < pdsv->num_vqs; qid++)
440 pds_vdpa_release_irq(pdsv, qid);
441
442 pci_free_irq_vectors(pdev);
443
444 vdpa_aux->nintrs = 0;
445 }
446
pds_vdpa_set_status(struct vdpa_device * vdpa_dev,u8 status)447 static void pds_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status)
448 {
449 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
450 struct device *dev = &pdsv->vdpa_dev.dev;
451 u8 old_status;
452 int i;
453
454 old_status = pds_vdpa_get_status(vdpa_dev);
455 dev_dbg(dev, "%s: old %#x new %#x\n", __func__, old_status, status);
456
457 if (status & ~old_status & VIRTIO_CONFIG_S_DRIVER_OK) {
458 if (pds_vdpa_request_irqs(pdsv))
459 status = old_status | VIRTIO_CONFIG_S_FAILED;
460 }
461
462 pds_vdpa_cmd_set_status(pdsv, status);
463
464 /* Note: still working with FW on the need for this reset cmd */
465 if (status == 0) {
466 pds_vdpa_cmd_reset(pdsv);
467
468 for (i = 0; i < pdsv->num_vqs; i++) {
469 pdsv->vqs[i].avail_idx = 0;
470 pdsv->vqs[i].used_idx = 0;
471 }
472
473 pds_vdpa_cmd_set_mac(pdsv, pdsv->mac);
474 }
475
476 if (status & ~old_status & VIRTIO_CONFIG_S_FEATURES_OK) {
477 for (i = 0; i < pdsv->num_vqs; i++) {
478 pdsv->vqs[i].notify =
479 vp_modern_map_vq_notify(&pdsv->vdpa_aux->vd_mdev,
480 i, &pdsv->vqs[i].notify_pa);
481 }
482 }
483
484 if (old_status & ~status & VIRTIO_CONFIG_S_DRIVER_OK)
485 pds_vdpa_release_irqs(pdsv);
486 }
487
pds_vdpa_init_vqs_entry(struct pds_vdpa_device * pdsv,int qid,void __iomem * notify)488 static void pds_vdpa_init_vqs_entry(struct pds_vdpa_device *pdsv, int qid,
489 void __iomem *notify)
490 {
491 memset(&pdsv->vqs[qid], 0, sizeof(pdsv->vqs[0]));
492 pdsv->vqs[qid].qid = qid;
493 pdsv->vqs[qid].pdsv = pdsv;
494 pdsv->vqs[qid].ready = false;
495 pdsv->vqs[qid].irq = VIRTIO_MSI_NO_VECTOR;
496 pdsv->vqs[qid].notify = notify;
497 }
498
pds_vdpa_reset(struct vdpa_device * vdpa_dev)499 static int pds_vdpa_reset(struct vdpa_device *vdpa_dev)
500 {
501 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
502 struct device *dev;
503 int err = 0;
504 u8 status;
505 int i;
506
507 dev = &pdsv->vdpa_aux->padev->aux_dev.dev;
508 status = pds_vdpa_get_status(vdpa_dev);
509
510 if (status == 0)
511 return 0;
512
513 if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
514 /* Reset the vqs */
515 for (i = 0; i < pdsv->num_vqs && !err; i++) {
516 err = pds_vdpa_cmd_reset_vq(pdsv, i, 0, &pdsv->vqs[i]);
517 if (err)
518 dev_err(dev, "%s: reset_vq failed qid %d: %pe\n",
519 __func__, i, ERR_PTR(err));
520 }
521 }
522
523 pds_vdpa_set_status(vdpa_dev, 0);
524
525 if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
526 /* Reset the vq info */
527 for (i = 0; i < pdsv->num_vqs && !err; i++)
528 pds_vdpa_init_vqs_entry(pdsv, i, pdsv->vqs[i].notify);
529 }
530
531 return 0;
532 }
533
pds_vdpa_get_config_size(struct vdpa_device * vdpa_dev)534 static size_t pds_vdpa_get_config_size(struct vdpa_device *vdpa_dev)
535 {
536 return sizeof(struct virtio_net_config);
537 }
538
pds_vdpa_get_config(struct vdpa_device * vdpa_dev,unsigned int offset,void * buf,unsigned int len)539 static void pds_vdpa_get_config(struct vdpa_device *vdpa_dev,
540 unsigned int offset,
541 void *buf, unsigned int len)
542 {
543 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
544 void __iomem *device;
545
546 if (offset + len > sizeof(struct virtio_net_config)) {
547 WARN(true, "%s: bad read, offset %d len %d\n", __func__, offset, len);
548 return;
549 }
550
551 device = pdsv->vdpa_aux->vd_mdev.device;
552 memcpy_fromio(buf, device + offset, len);
553 }
554
pds_vdpa_set_config(struct vdpa_device * vdpa_dev,unsigned int offset,const void * buf,unsigned int len)555 static void pds_vdpa_set_config(struct vdpa_device *vdpa_dev,
556 unsigned int offset, const void *buf,
557 unsigned int len)
558 {
559 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
560 void __iomem *device;
561
562 if (offset + len > sizeof(struct virtio_net_config)) {
563 WARN(true, "%s: bad read, offset %d len %d\n", __func__, offset, len);
564 return;
565 }
566
567 device = pdsv->vdpa_aux->vd_mdev.device;
568 memcpy_toio(device + offset, buf, len);
569 }
570
571 static const struct vdpa_config_ops pds_vdpa_ops = {
572 .set_vq_address = pds_vdpa_set_vq_address,
573 .set_vq_num = pds_vdpa_set_vq_num,
574 .kick_vq = pds_vdpa_kick_vq,
575 .set_vq_cb = pds_vdpa_set_vq_cb,
576 .set_vq_ready = pds_vdpa_set_vq_ready,
577 .get_vq_ready = pds_vdpa_get_vq_ready,
578 .set_vq_state = pds_vdpa_set_vq_state,
579 .get_vq_state = pds_vdpa_get_vq_state,
580 .get_vq_notification = pds_vdpa_get_vq_notification,
581 .get_vq_irq = pds_vdpa_get_vq_irq,
582 .get_vq_align = pds_vdpa_get_vq_align,
583 .get_vq_group = pds_vdpa_get_vq_group,
584
585 .get_device_features = pds_vdpa_get_device_features,
586 .set_driver_features = pds_vdpa_set_driver_features,
587 .get_driver_features = pds_vdpa_get_driver_features,
588 .set_config_cb = pds_vdpa_set_config_cb,
589 .get_vq_num_max = pds_vdpa_get_vq_num_max,
590 .get_device_id = pds_vdpa_get_device_id,
591 .get_vendor_id = pds_vdpa_get_vendor_id,
592 .get_status = pds_vdpa_get_status,
593 .set_status = pds_vdpa_set_status,
594 .reset = pds_vdpa_reset,
595 .get_config_size = pds_vdpa_get_config_size,
596 .get_config = pds_vdpa_get_config,
597 .set_config = pds_vdpa_set_config,
598 };
599 static struct virtio_device_id pds_vdpa_id_table[] = {
600 {VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID},
601 {0},
602 };
603
pds_vdpa_dev_add(struct vdpa_mgmt_dev * mdev,const char * name,const struct vdpa_dev_set_config * add_config)604 static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
605 const struct vdpa_dev_set_config *add_config)
606 {
607 struct pds_vdpa_aux *vdpa_aux;
608 struct pds_vdpa_device *pdsv;
609 struct vdpa_mgmt_dev *mgmt;
610 u16 fw_max_vqs, vq_pairs;
611 struct device *dma_dev;
612 struct pci_dev *pdev;
613 struct device *dev;
614 int err;
615 int i;
616
617 vdpa_aux = container_of(mdev, struct pds_vdpa_aux, vdpa_mdev);
618 dev = &vdpa_aux->padev->aux_dev.dev;
619 mgmt = &vdpa_aux->vdpa_mdev;
620
621 if (vdpa_aux->pdsv) {
622 dev_warn(dev, "Multiple vDPA devices on a VF is not supported.\n");
623 return -EOPNOTSUPP;
624 }
625
626 pdsv = vdpa_alloc_device(struct pds_vdpa_device, vdpa_dev,
627 dev, &pds_vdpa_ops, 1, 1, name, false);
628 if (IS_ERR(pdsv)) {
629 dev_err(dev, "Failed to allocate vDPA structure: %pe\n", pdsv);
630 return PTR_ERR(pdsv);
631 }
632
633 vdpa_aux->pdsv = pdsv;
634 pdsv->vdpa_aux = vdpa_aux;
635
636 pdev = vdpa_aux->padev->vf_pdev;
637 dma_dev = &pdev->dev;
638 pdsv->vdpa_dev.dma_dev = dma_dev;
639
640 pdsv->supported_features = mgmt->supported_features;
641
642 if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {
643 u64 unsupp_features =
644 add_config->device_features & ~pdsv->supported_features;
645
646 if (unsupp_features) {
647 dev_err(dev, "Unsupported features: %#llx\n", unsupp_features);
648 err = -EOPNOTSUPP;
649 goto err_unmap;
650 }
651
652 pdsv->supported_features = add_config->device_features;
653 }
654
655 err = pds_vdpa_cmd_reset(pdsv);
656 if (err) {
657 dev_err(dev, "Failed to reset hw: %pe\n", ERR_PTR(err));
658 goto err_unmap;
659 }
660
661 err = pds_vdpa_init_hw(pdsv);
662 if (err) {
663 dev_err(dev, "Failed to init hw: %pe\n", ERR_PTR(err));
664 goto err_unmap;
665 }
666
667 fw_max_vqs = le16_to_cpu(pdsv->vdpa_aux->ident.max_vqs);
668 vq_pairs = fw_max_vqs / 2;
669
670 /* Make sure we have the queues being requested */
671 if (add_config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MAX_VQP))
672 vq_pairs = add_config->net.max_vq_pairs;
673
674 pdsv->num_vqs = 2 * vq_pairs;
675 if (pdsv->supported_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))
676 pdsv->num_vqs++;
677
678 if (pdsv->num_vqs > fw_max_vqs) {
679 dev_err(dev, "%s: queue count requested %u greater than max %u\n",
680 __func__, pdsv->num_vqs, fw_max_vqs);
681 err = -ENOSPC;
682 goto err_unmap;
683 }
684
685 if (pdsv->num_vqs != fw_max_vqs) {
686 err = pds_vdpa_cmd_set_max_vq_pairs(pdsv, vq_pairs);
687 if (err) {
688 dev_err(dev, "Failed to set max_vq_pairs: %pe\n",
689 ERR_PTR(err));
690 goto err_unmap;
691 }
692 }
693
694 /* Set a mac, either from the user config if provided
695 * or use the device's mac if not 00:..:00
696 * or set a random mac
697 */
698 if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR)) {
699 ether_addr_copy(pdsv->mac, add_config->net.mac);
700 } else {
701 struct virtio_net_config __iomem *vc;
702
703 vc = pdsv->vdpa_aux->vd_mdev.device;
704 memcpy_fromio(pdsv->mac, vc->mac, sizeof(pdsv->mac));
705 if (is_zero_ether_addr(pdsv->mac) &&
706 (pdsv->supported_features & BIT_ULL(VIRTIO_NET_F_MAC))) {
707 eth_random_addr(pdsv->mac);
708 dev_info(dev, "setting random mac %pM\n", pdsv->mac);
709 }
710 }
711 pds_vdpa_cmd_set_mac(pdsv, pdsv->mac);
712
713 for (i = 0; i < pdsv->num_vqs; i++) {
714 void __iomem *notify;
715
716 notify = vp_modern_map_vq_notify(&pdsv->vdpa_aux->vd_mdev,
717 i, &pdsv->vqs[i].notify_pa);
718 pds_vdpa_init_vqs_entry(pdsv, i, notify);
719 }
720
721 pdsv->vdpa_dev.mdev = &vdpa_aux->vdpa_mdev;
722
723 err = pds_vdpa_register_event_handler(pdsv);
724 if (err) {
725 dev_err(dev, "Failed to register for PDS events: %pe\n", ERR_PTR(err));
726 goto err_unmap;
727 }
728
729 /* We use the _vdpa_register_device() call rather than the
730 * vdpa_register_device() to avoid a deadlock because our
731 * dev_add() is called with the vdpa_dev_lock already set
732 * by vdpa_nl_cmd_dev_add_set_doit()
733 */
734 err = _vdpa_register_device(&pdsv->vdpa_dev, pdsv->num_vqs);
735 if (err) {
736 dev_err(dev, "Failed to register to vDPA bus: %pe\n", ERR_PTR(err));
737 goto err_unevent;
738 }
739
740 pds_vdpa_debugfs_add_vdpadev(vdpa_aux);
741
742 return 0;
743
744 err_unevent:
745 pds_vdpa_unregister_event_handler(pdsv);
746 err_unmap:
747 put_device(&pdsv->vdpa_dev.dev);
748 vdpa_aux->pdsv = NULL;
749 return err;
750 }
751
pds_vdpa_dev_del(struct vdpa_mgmt_dev * mdev,struct vdpa_device * vdpa_dev)752 static void pds_vdpa_dev_del(struct vdpa_mgmt_dev *mdev,
753 struct vdpa_device *vdpa_dev)
754 {
755 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);
756 struct pds_vdpa_aux *vdpa_aux;
757
758 pds_vdpa_unregister_event_handler(pdsv);
759
760 vdpa_aux = container_of(mdev, struct pds_vdpa_aux, vdpa_mdev);
761 _vdpa_unregister_device(vdpa_dev);
762
763 pds_vdpa_cmd_reset(vdpa_aux->pdsv);
764 pds_vdpa_debugfs_reset_vdpadev(vdpa_aux);
765
766 vdpa_aux->pdsv = NULL;
767
768 dev_info(&vdpa_aux->padev->aux_dev.dev, "Removed vdpa device\n");
769 }
770
771 static const struct vdpa_mgmtdev_ops pds_vdpa_mgmt_dev_ops = {
772 .dev_add = pds_vdpa_dev_add,
773 .dev_del = pds_vdpa_dev_del
774 };
775
pds_vdpa_get_mgmt_info(struct pds_vdpa_aux * vdpa_aux)776 int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux)
777 {
778 union pds_core_adminq_cmd cmd = {
779 .vdpa_ident.opcode = PDS_VDPA_CMD_IDENT,
780 .vdpa_ident.vf_id = cpu_to_le16(vdpa_aux->vf_id),
781 };
782 union pds_core_adminq_comp comp = {};
783 struct vdpa_mgmt_dev *mgmt;
784 struct pci_dev *pf_pdev;
785 struct device *pf_dev;
786 struct pci_dev *pdev;
787 dma_addr_t ident_pa;
788 struct device *dev;
789 u16 dev_intrs;
790 u16 max_vqs;
791 int err;
792
793 dev = &vdpa_aux->padev->aux_dev.dev;
794 pdev = vdpa_aux->padev->vf_pdev;
795 mgmt = &vdpa_aux->vdpa_mdev;
796
797 /* Get resource info through the PF's adminq. It is a block of info,
798 * so we need to map some memory for PF to make available to the
799 * firmware for writing the data.
800 */
801 pf_pdev = pci_physfn(vdpa_aux->padev->vf_pdev);
802 pf_dev = &pf_pdev->dev;
803 ident_pa = dma_map_single(pf_dev, &vdpa_aux->ident,
804 sizeof(vdpa_aux->ident), DMA_FROM_DEVICE);
805 if (dma_mapping_error(pf_dev, ident_pa)) {
806 dev_err(dev, "Failed to map ident space\n");
807 return -ENOMEM;
808 }
809
810 cmd.vdpa_ident.ident_pa = cpu_to_le64(ident_pa);
811 cmd.vdpa_ident.len = cpu_to_le32(sizeof(vdpa_aux->ident));
812 err = pds_client_adminq_cmd(vdpa_aux->padev, &cmd,
813 sizeof(cmd.vdpa_ident), &comp, 0);
814 dma_unmap_single(pf_dev, ident_pa,
815 sizeof(vdpa_aux->ident), DMA_FROM_DEVICE);
816 if (err) {
817 dev_err(dev, "Failed to ident hw, status %d: %pe\n",
818 comp.status, ERR_PTR(err));
819 return err;
820 }
821
822 max_vqs = le16_to_cpu(vdpa_aux->ident.max_vqs);
823 dev_intrs = pci_msix_vec_count(pdev);
824 dev_dbg(dev, "ident.max_vqs %d dev_intrs %d\n", max_vqs, dev_intrs);
825
826 max_vqs = min_t(u16, dev_intrs, max_vqs);
827 mgmt->max_supported_vqs = min_t(u16, PDS_VDPA_MAX_QUEUES, max_vqs);
828 vdpa_aux->nintrs = 0;
829
830 mgmt->ops = &pds_vdpa_mgmt_dev_ops;
831 mgmt->id_table = pds_vdpa_id_table;
832 mgmt->device = dev;
833 mgmt->supported_features = le64_to_cpu(vdpa_aux->ident.hw_features);
834
835 /* advertise F_MAC even if the device doesn't */
836 mgmt->supported_features |= BIT_ULL(VIRTIO_NET_F_MAC);
837
838 mgmt->config_attr_mask = BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR);
839 mgmt->config_attr_mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP);
840 mgmt->config_attr_mask |= BIT_ULL(VDPA_ATTR_DEV_FEATURES);
841
842 return 0;
843 }
844