1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2020 Intel Corporation
4 * Author: Johannes Berg <johannes@sipsolutions.net>
5 */
6 #include <linux/module.h>
7 #include <linux/pci.h>
8 #include <linux/virtio.h>
9 #include <linux/virtio_config.h>
10 #include <linux/logic_iomem.h>
11 #include <linux/of_platform.h>
12 #include <linux/irqdomain.h>
13 #include <linux/virtio_pcidev.h>
14 #include <linux/virtio-uml.h>
15 #include <linux/delay.h>
16 #include <linux/msi.h>
17 #include <linux/unaligned.h>
18 #include <irq_kern.h>
19
20 #include "virt-pci.h"
21
22 #define to_virtio_pcidev(_pdev) \
23 container_of(_pdev, struct virtio_pcidev_device, pdev)
24
25 /* for MSI-X we have a 32-bit payload */
26 #define MAX_IRQ_MSG_SIZE (sizeof(struct virtio_pcidev_msg) + sizeof(u32))
27 #define NUM_IRQ_MSGS 10
28
29 struct virtio_pcidev_message_buffer {
30 struct virtio_pcidev_msg hdr;
31 u8 data[8];
32 };
33
34 struct virtio_pcidev_device {
35 struct um_pci_device pdev;
36 struct virtio_device *vdev;
37
38 struct virtqueue *cmd_vq, *irq_vq;
39
40 #define VIRTIO_PCIDEV_WRITE_BUFS 20
41 struct virtio_pcidev_message_buffer bufs[VIRTIO_PCIDEV_WRITE_BUFS + 1];
42 void *extra_ptrs[VIRTIO_PCIDEV_WRITE_BUFS + 1];
43 DECLARE_BITMAP(used_bufs, VIRTIO_PCIDEV_WRITE_BUFS);
44
45 #define UM_PCI_STAT_WAITING 0
46 unsigned long status;
47
48 bool platform;
49 };
50
51 static unsigned int virtio_pcidev_max_delay_us = 40000;
52 module_param_named(max_delay_us, virtio_pcidev_max_delay_us, uint, 0644);
53
virtio_pcidev_get_buf(struct virtio_pcidev_device * dev,bool * posted)54 static int virtio_pcidev_get_buf(struct virtio_pcidev_device *dev, bool *posted)
55 {
56 int i;
57
58 for (i = 0; i < VIRTIO_PCIDEV_WRITE_BUFS; i++) {
59 if (!test_and_set_bit(i, dev->used_bufs))
60 return i;
61 }
62
63 *posted = false;
64 return VIRTIO_PCIDEV_WRITE_BUFS;
65 }
66
virtio_pcidev_free_buf(struct virtio_pcidev_device * dev,void * buf)67 static void virtio_pcidev_free_buf(struct virtio_pcidev_device *dev, void *buf)
68 {
69 int i;
70
71 if (buf == &dev->bufs[VIRTIO_PCIDEV_WRITE_BUFS]) {
72 kfree(dev->extra_ptrs[VIRTIO_PCIDEV_WRITE_BUFS]);
73 dev->extra_ptrs[VIRTIO_PCIDEV_WRITE_BUFS] = NULL;
74 return;
75 }
76
77 for (i = 0; i < VIRTIO_PCIDEV_WRITE_BUFS; i++) {
78 if (buf == &dev->bufs[i]) {
79 kfree(dev->extra_ptrs[i]);
80 dev->extra_ptrs[i] = NULL;
81 WARN_ON(!test_and_clear_bit(i, dev->used_bufs));
82 return;
83 }
84 }
85
86 WARN_ON(1);
87 }
88
virtio_pcidev_send_cmd(struct virtio_pcidev_device * dev,struct virtio_pcidev_msg * cmd,unsigned int cmd_size,const void * extra,unsigned int extra_size,void * out,unsigned int out_size)89 static int virtio_pcidev_send_cmd(struct virtio_pcidev_device *dev,
90 struct virtio_pcidev_msg *cmd,
91 unsigned int cmd_size,
92 const void *extra, unsigned int extra_size,
93 void *out, unsigned int out_size)
94 {
95 struct scatterlist out_sg, extra_sg, in_sg;
96 struct scatterlist *sgs_list[] = {
97 [0] = &out_sg,
98 [1] = extra ? &extra_sg : &in_sg,
99 [2] = extra ? &in_sg : NULL,
100 };
101 struct virtio_pcidev_message_buffer *buf;
102 int delay_count = 0;
103 bool bounce_out;
104 int ret, len;
105 int buf_idx;
106 bool posted;
107
108 if (WARN_ON(cmd_size < sizeof(*cmd) || cmd_size > sizeof(*buf)))
109 return -EINVAL;
110
111 switch (cmd->op) {
112 case VIRTIO_PCIDEV_OP_CFG_WRITE:
113 case VIRTIO_PCIDEV_OP_MMIO_WRITE:
114 case VIRTIO_PCIDEV_OP_MMIO_MEMSET:
115 /* in PCI, writes are posted, so don't wait */
116 posted = !out;
117 WARN_ON(!posted);
118 break;
119 default:
120 posted = false;
121 break;
122 }
123
124 bounce_out = !posted && cmd_size <= sizeof(*cmd) &&
125 out && out_size <= sizeof(buf->data);
126
127 buf_idx = virtio_pcidev_get_buf(dev, &posted);
128 buf = &dev->bufs[buf_idx];
129 memcpy(buf, cmd, cmd_size);
130
131 if (posted && extra && extra_size > sizeof(buf) - cmd_size) {
132 dev->extra_ptrs[buf_idx] = kmemdup(extra, extra_size,
133 GFP_ATOMIC);
134
135 if (!dev->extra_ptrs[buf_idx]) {
136 virtio_pcidev_free_buf(dev, buf);
137 return -ENOMEM;
138 }
139 extra = dev->extra_ptrs[buf_idx];
140 } else if (extra && extra_size <= sizeof(buf) - cmd_size) {
141 memcpy((u8 *)buf + cmd_size, extra, extra_size);
142 cmd_size += extra_size;
143 extra_size = 0;
144 extra = NULL;
145 cmd = (void *)buf;
146 } else {
147 cmd = (void *)buf;
148 }
149
150 sg_init_one(&out_sg, cmd, cmd_size);
151 if (extra)
152 sg_init_one(&extra_sg, extra, extra_size);
153 /* allow stack for small buffers */
154 if (bounce_out)
155 sg_init_one(&in_sg, buf->data, out_size);
156 else if (out)
157 sg_init_one(&in_sg, out, out_size);
158
159 /* add to internal virtio queue */
160 ret = virtqueue_add_sgs(dev->cmd_vq, sgs_list,
161 extra ? 2 : 1,
162 out ? 1 : 0,
163 cmd, GFP_ATOMIC);
164 if (ret) {
165 virtio_pcidev_free_buf(dev, buf);
166 return ret;
167 }
168
169 if (posted) {
170 virtqueue_kick(dev->cmd_vq);
171 return 0;
172 }
173
174 /* kick and poll for getting a response on the queue */
175 set_bit(UM_PCI_STAT_WAITING, &dev->status);
176 virtqueue_kick(dev->cmd_vq);
177 ret = 0;
178
179 while (1) {
180 void *completed = virtqueue_get_buf(dev->cmd_vq, &len);
181
182 if (completed == buf)
183 break;
184
185 if (completed)
186 virtio_pcidev_free_buf(dev, completed);
187
188 if (WARN_ONCE(virtqueue_is_broken(dev->cmd_vq) ||
189 ++delay_count > virtio_pcidev_max_delay_us,
190 "um virt-pci delay: %d", delay_count)) {
191 ret = -EIO;
192 break;
193 }
194 udelay(1);
195 }
196 clear_bit(UM_PCI_STAT_WAITING, &dev->status);
197
198 if (bounce_out)
199 memcpy(out, buf->data, out_size);
200
201 virtio_pcidev_free_buf(dev, buf);
202
203 return ret;
204 }
205
virtio_pcidev_cfgspace_read(struct um_pci_device * pdev,unsigned int offset,int size)206 static unsigned long virtio_pcidev_cfgspace_read(struct um_pci_device *pdev,
207 unsigned int offset, int size)
208 {
209 struct virtio_pcidev_device *dev = to_virtio_pcidev(pdev);
210 struct virtio_pcidev_msg hdr = {
211 .op = VIRTIO_PCIDEV_OP_CFG_READ,
212 .size = size,
213 .addr = offset,
214 };
215 /* max 8, we might not use it all */
216 u8 data[8];
217
218 memset(data, 0xff, sizeof(data));
219
220 /* size has been checked in um_pci_cfgspace_read() */
221 if (virtio_pcidev_send_cmd(dev, &hdr, sizeof(hdr), NULL, 0, data, size))
222 return ULONG_MAX;
223
224 switch (size) {
225 case 1:
226 return data[0];
227 case 2:
228 return le16_to_cpup((void *)data);
229 case 4:
230 return le32_to_cpup((void *)data);
231 #ifdef CONFIG_64BIT
232 case 8:
233 return le64_to_cpup((void *)data);
234 #endif
235 default:
236 return ULONG_MAX;
237 }
238 }
239
virtio_pcidev_cfgspace_write(struct um_pci_device * pdev,unsigned int offset,int size,unsigned long val)240 static void virtio_pcidev_cfgspace_write(struct um_pci_device *pdev,
241 unsigned int offset, int size,
242 unsigned long val)
243 {
244 struct virtio_pcidev_device *dev = to_virtio_pcidev(pdev);
245 struct {
246 struct virtio_pcidev_msg hdr;
247 /* maximum size - we may only use parts of it */
248 u8 data[8];
249 } msg = {
250 .hdr = {
251 .op = VIRTIO_PCIDEV_OP_CFG_WRITE,
252 .size = size,
253 .addr = offset,
254 },
255 };
256
257 /* size has been checked in um_pci_cfgspace_write() */
258 switch (size) {
259 case 1:
260 msg.data[0] = (u8)val;
261 break;
262 case 2:
263 put_unaligned_le16(val, (void *)msg.data);
264 break;
265 case 4:
266 put_unaligned_le32(val, (void *)msg.data);
267 break;
268 #ifdef CONFIG_64BIT
269 case 8:
270 put_unaligned_le64(val, (void *)msg.data);
271 break;
272 #endif
273 }
274
275 WARN_ON(virtio_pcidev_send_cmd(dev, &msg.hdr, sizeof(msg), NULL, 0, NULL, 0));
276 }
277
virtio_pcidev_bar_copy_from(struct um_pci_device * pdev,int bar,void * buffer,unsigned int offset,int size)278 static void virtio_pcidev_bar_copy_from(struct um_pci_device *pdev,
279 int bar, void *buffer,
280 unsigned int offset, int size)
281 {
282 struct virtio_pcidev_device *dev = to_virtio_pcidev(pdev);
283 struct virtio_pcidev_msg hdr = {
284 .op = VIRTIO_PCIDEV_OP_MMIO_READ,
285 .bar = bar,
286 .size = size,
287 .addr = offset,
288 };
289
290 memset(buffer, 0xff, size);
291
292 virtio_pcidev_send_cmd(dev, &hdr, sizeof(hdr), NULL, 0, buffer, size);
293 }
294
virtio_pcidev_bar_read(struct um_pci_device * pdev,int bar,unsigned int offset,int size)295 static unsigned long virtio_pcidev_bar_read(struct um_pci_device *pdev, int bar,
296 unsigned int offset, int size)
297 {
298 /* 8 is maximum size - we may only use parts of it */
299 u8 data[8];
300
301 /* size has been checked in um_pci_bar_read() */
302 virtio_pcidev_bar_copy_from(pdev, bar, data, offset, size);
303
304 switch (size) {
305 case 1:
306 return data[0];
307 case 2:
308 return le16_to_cpup((void *)data);
309 case 4:
310 return le32_to_cpup((void *)data);
311 #ifdef CONFIG_64BIT
312 case 8:
313 return le64_to_cpup((void *)data);
314 #endif
315 default:
316 return ULONG_MAX;
317 }
318 }
319
virtio_pcidev_bar_copy_to(struct um_pci_device * pdev,int bar,unsigned int offset,const void * buffer,int size)320 static void virtio_pcidev_bar_copy_to(struct um_pci_device *pdev,
321 int bar, unsigned int offset,
322 const void *buffer, int size)
323 {
324 struct virtio_pcidev_device *dev = to_virtio_pcidev(pdev);
325 struct virtio_pcidev_msg hdr = {
326 .op = VIRTIO_PCIDEV_OP_MMIO_WRITE,
327 .bar = bar,
328 .size = size,
329 .addr = offset,
330 };
331
332 virtio_pcidev_send_cmd(dev, &hdr, sizeof(hdr), buffer, size, NULL, 0);
333 }
334
virtio_pcidev_bar_write(struct um_pci_device * pdev,int bar,unsigned int offset,int size,unsigned long val)335 static void virtio_pcidev_bar_write(struct um_pci_device *pdev, int bar,
336 unsigned int offset, int size,
337 unsigned long val)
338 {
339 /* maximum size - we may only use parts of it */
340 u8 data[8];
341
342 /* size has been checked in um_pci_bar_write() */
343 switch (size) {
344 case 1:
345 data[0] = (u8)val;
346 break;
347 case 2:
348 put_unaligned_le16(val, (void *)data);
349 break;
350 case 4:
351 put_unaligned_le32(val, (void *)data);
352 break;
353 #ifdef CONFIG_64BIT
354 case 8:
355 put_unaligned_le64(val, (void *)data);
356 break;
357 #endif
358 }
359
360 virtio_pcidev_bar_copy_to(pdev, bar, offset, data, size);
361 }
362
virtio_pcidev_bar_set(struct um_pci_device * pdev,int bar,unsigned int offset,u8 value,int size)363 static void virtio_pcidev_bar_set(struct um_pci_device *pdev, int bar,
364 unsigned int offset, u8 value, int size)
365 {
366 struct virtio_pcidev_device *dev = to_virtio_pcidev(pdev);
367 struct {
368 struct virtio_pcidev_msg hdr;
369 u8 data;
370 } msg = {
371 .hdr = {
372 .op = VIRTIO_PCIDEV_OP_CFG_WRITE,
373 .bar = bar,
374 .size = size,
375 .addr = offset,
376 },
377 .data = value,
378 };
379
380 virtio_pcidev_send_cmd(dev, &msg.hdr, sizeof(msg), NULL, 0, NULL, 0);
381 }
382
383 static const struct um_pci_ops virtio_pcidev_um_pci_ops = {
384 .cfgspace_read = virtio_pcidev_cfgspace_read,
385 .cfgspace_write = virtio_pcidev_cfgspace_write,
386 .bar_read = virtio_pcidev_bar_read,
387 .bar_write = virtio_pcidev_bar_write,
388 .bar_copy_from = virtio_pcidev_bar_copy_from,
389 .bar_copy_to = virtio_pcidev_bar_copy_to,
390 .bar_set = virtio_pcidev_bar_set,
391 };
392
virtio_pcidev_irq_vq_addbuf(struct virtqueue * vq,void * buf,bool kick)393 static void virtio_pcidev_irq_vq_addbuf(struct virtqueue *vq, void *buf, bool kick)
394 {
395 struct scatterlist sg[1];
396
397 sg_init_one(sg, buf, MAX_IRQ_MSG_SIZE);
398 if (virtqueue_add_inbuf(vq, sg, 1, buf, GFP_ATOMIC))
399 kfree(buf);
400 else if (kick)
401 virtqueue_kick(vq);
402 }
403
virtio_pcidev_handle_irq_message(struct virtqueue * vq,struct virtio_pcidev_msg * msg)404 static void virtio_pcidev_handle_irq_message(struct virtqueue *vq,
405 struct virtio_pcidev_msg *msg)
406 {
407 struct virtio_device *vdev = vq->vdev;
408 struct virtio_pcidev_device *dev = vdev->priv;
409
410 if (!dev->pdev.irq)
411 return;
412
413 /* we should properly chain interrupts, but on ARCH=um we don't care */
414
415 switch (msg->op) {
416 case VIRTIO_PCIDEV_OP_INT:
417 generic_handle_irq(dev->pdev.irq);
418 break;
419 case VIRTIO_PCIDEV_OP_MSI:
420 /* our MSI message is just the interrupt number */
421 if (msg->size == sizeof(u32))
422 generic_handle_irq(le32_to_cpup((void *)msg->data));
423 else
424 generic_handle_irq(le16_to_cpup((void *)msg->data));
425 break;
426 case VIRTIO_PCIDEV_OP_PME:
427 /* nothing to do - we already woke up due to the message */
428 break;
429 default:
430 dev_err(&vdev->dev, "unexpected virt-pci message %d\n", msg->op);
431 break;
432 }
433 }
434
virtio_pcidev_cmd_vq_cb(struct virtqueue * vq)435 static void virtio_pcidev_cmd_vq_cb(struct virtqueue *vq)
436 {
437 struct virtio_device *vdev = vq->vdev;
438 struct virtio_pcidev_device *dev = vdev->priv;
439 void *cmd;
440 int len;
441
442 if (test_bit(UM_PCI_STAT_WAITING, &dev->status))
443 return;
444
445 while ((cmd = virtqueue_get_buf(vq, &len)))
446 virtio_pcidev_free_buf(dev, cmd);
447 }
448
virtio_pcidev_irq_vq_cb(struct virtqueue * vq)449 static void virtio_pcidev_irq_vq_cb(struct virtqueue *vq)
450 {
451 struct virtio_pcidev_msg *msg;
452 int len;
453
454 while ((msg = virtqueue_get_buf(vq, &len))) {
455 if (len >= sizeof(*msg))
456 virtio_pcidev_handle_irq_message(vq, msg);
457
458 /* recycle the message buffer */
459 virtio_pcidev_irq_vq_addbuf(vq, msg, true);
460 }
461 }
462
virtio_pcidev_init_vqs(struct virtio_pcidev_device * dev)463 static int virtio_pcidev_init_vqs(struct virtio_pcidev_device *dev)
464 {
465 struct virtqueue_info vqs_info[] = {
466 { "cmd", virtio_pcidev_cmd_vq_cb },
467 { "irq", virtio_pcidev_irq_vq_cb },
468 };
469 struct virtqueue *vqs[2];
470 int err, i;
471
472 err = virtio_find_vqs(dev->vdev, 2, vqs, vqs_info, NULL);
473 if (err)
474 return err;
475
476 dev->cmd_vq = vqs[0];
477 dev->irq_vq = vqs[1];
478
479 virtio_device_ready(dev->vdev);
480
481 for (i = 0; i < NUM_IRQ_MSGS; i++) {
482 void *msg = kzalloc(MAX_IRQ_MSG_SIZE, GFP_KERNEL);
483
484 if (msg)
485 virtio_pcidev_irq_vq_addbuf(dev->irq_vq, msg, false);
486 }
487
488 virtqueue_kick(dev->irq_vq);
489
490 return 0;
491 }
492
__virtio_pcidev_virtio_platform_remove(struct virtio_device * vdev,struct virtio_pcidev_device * dev)493 static void __virtio_pcidev_virtio_platform_remove(struct virtio_device *vdev,
494 struct virtio_pcidev_device *dev)
495 {
496 um_pci_platform_device_unregister(&dev->pdev);
497
498 virtio_reset_device(vdev);
499 vdev->config->del_vqs(vdev);
500
501 kfree(dev);
502 }
503
virtio_pcidev_virtio_platform_probe(struct virtio_device * vdev,struct virtio_pcidev_device * dev)504 static int virtio_pcidev_virtio_platform_probe(struct virtio_device *vdev,
505 struct virtio_pcidev_device *dev)
506 {
507 int err;
508
509 dev->platform = true;
510
511 err = virtio_pcidev_init_vqs(dev);
512 if (err)
513 goto err_free;
514
515 err = um_pci_platform_device_register(&dev->pdev);
516 if (err)
517 goto err_reset;
518
519 err = of_platform_default_populate(vdev->dev.of_node, NULL, &vdev->dev);
520 if (err)
521 goto err_unregister;
522
523 return 0;
524
525 err_unregister:
526 um_pci_platform_device_unregister(&dev->pdev);
527 err_reset:
528 virtio_reset_device(vdev);
529 vdev->config->del_vqs(vdev);
530 err_free:
531 kfree(dev);
532 return err;
533 }
534
virtio_pcidev_virtio_probe(struct virtio_device * vdev)535 static int virtio_pcidev_virtio_probe(struct virtio_device *vdev)
536 {
537 struct virtio_pcidev_device *dev;
538 int err;
539
540 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
541 if (!dev)
542 return -ENOMEM;
543
544 dev->vdev = vdev;
545 vdev->priv = dev;
546
547 dev->pdev.ops = &virtio_pcidev_um_pci_ops;
548
549 if (of_device_is_compatible(vdev->dev.of_node, "simple-bus"))
550 return virtio_pcidev_virtio_platform_probe(vdev, dev);
551
552 err = virtio_pcidev_init_vqs(dev);
553 if (err)
554 goto err_free;
555
556 err = um_pci_device_register(&dev->pdev);
557 if (err)
558 goto err_reset;
559
560 device_set_wakeup_enable(&vdev->dev, true);
561
562 /*
563 * In order to do suspend-resume properly, don't allow VQs
564 * to be suspended.
565 */
566 virtio_uml_set_no_vq_suspend(vdev, true);
567
568 return 0;
569
570 err_reset:
571 virtio_reset_device(vdev);
572 vdev->config->del_vqs(vdev);
573 err_free:
574 kfree(dev);
575 return err;
576 }
577
virtio_pcidev_virtio_remove(struct virtio_device * vdev)578 static void virtio_pcidev_virtio_remove(struct virtio_device *vdev)
579 {
580 struct virtio_pcidev_device *dev = vdev->priv;
581
582 if (dev->platform) {
583 of_platform_depopulate(&vdev->dev);
584 __virtio_pcidev_virtio_platform_remove(vdev, dev);
585 return;
586 }
587
588 device_set_wakeup_enable(&vdev->dev, false);
589
590 um_pci_device_unregister(&dev->pdev);
591
592 /* Stop all virtqueues */
593 virtio_reset_device(vdev);
594 dev->cmd_vq = NULL;
595 dev->irq_vq = NULL;
596 vdev->config->del_vqs(vdev);
597
598 kfree(dev);
599 }
600
601 static struct virtio_device_id id_table[] = {
602 { CONFIG_UML_PCI_OVER_VIRTIO_DEVICE_ID, VIRTIO_DEV_ANY_ID },
603 { 0 },
604 };
605 MODULE_DEVICE_TABLE(virtio, id_table);
606
607 static struct virtio_driver virtio_pcidev_virtio_driver = {
608 .driver.name = "virtio-pci",
609 .id_table = id_table,
610 .probe = virtio_pcidev_virtio_probe,
611 .remove = virtio_pcidev_virtio_remove,
612 };
613
virtio_pcidev_init(void)614 static int __init virtio_pcidev_init(void)
615 {
616 if (WARN(CONFIG_UML_PCI_OVER_VIRTIO_DEVICE_ID < 0,
617 "No virtio device ID configured for PCI - no PCI support\n"))
618 return 0;
619
620 return register_virtio_driver(&virtio_pcidev_virtio_driver);
621 }
622 late_initcall(virtio_pcidev_init);
623
virtio_pcidev_exit(void)624 static void __exit virtio_pcidev_exit(void)
625 {
626 unregister_virtio_driver(&virtio_pcidev_virtio_driver);
627 }
628 module_exit(virtio_pcidev_exit);
629