1 /*
2 * libqos VIRTIO 1.0 PCI driver
3 *
4 * Copyright (c) 2019 Red Hat, Inc
5 *
6 * This work is licensed under the terms of the GNU GPL, version 2 or later.
7 * See the COPYING file in the top-level directory.
8 */
9
10 #include "qemu/osdep.h"
11 #include "standard-headers/linux/pci_regs.h"
12 #include "standard-headers/linux/virtio_pci.h"
13 #include "standard-headers/linux/virtio_config.h"
14 #include "virtio-pci-modern.h"
15
config_readb(QVirtioDevice * d,uint64_t addr)16 static uint8_t config_readb(QVirtioDevice *d, uint64_t addr)
17 {
18 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
19 return qpci_io_readb(dev->pdev, dev->bar, dev->device_cfg_offset + addr);
20 }
21
config_readw(QVirtioDevice * d,uint64_t addr)22 static uint16_t config_readw(QVirtioDevice *d, uint64_t addr)
23 {
24 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
25 return qpci_io_readw(dev->pdev, dev->bar, dev->device_cfg_offset + addr);
26 }
27
config_readl(QVirtioDevice * d,uint64_t addr)28 static uint32_t config_readl(QVirtioDevice *d, uint64_t addr)
29 {
30 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
31 return qpci_io_readl(dev->pdev, dev->bar, dev->device_cfg_offset + addr);
32 }
33
config_readq(QVirtioDevice * d,uint64_t addr)34 static uint64_t config_readq(QVirtioDevice *d, uint64_t addr)
35 {
36 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
37 return qpci_io_readq(dev->pdev, dev->bar, dev->device_cfg_offset + addr);
38 }
39
get_features(QVirtioDevice * d)40 static uint64_t get_features(QVirtioDevice *d)
41 {
42 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
43 uint64_t lo, hi;
44
45 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
46 offsetof(struct virtio_pci_common_cfg,
47 device_feature_select),
48 0);
49 lo = qpci_io_readl(dev->pdev, dev->bar, dev->common_cfg_offset +
50 offsetof(struct virtio_pci_common_cfg, device_feature));
51
52 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
53 offsetof(struct virtio_pci_common_cfg,
54 device_feature_select),
55 1);
56 hi = qpci_io_readl(dev->pdev, dev->bar, dev->common_cfg_offset +
57 offsetof(struct virtio_pci_common_cfg, device_feature));
58
59 return (hi << 32) | lo;
60 }
61
set_features(QVirtioDevice * d,uint64_t features)62 static void set_features(QVirtioDevice *d, uint64_t features)
63 {
64 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
65
66 /* Drivers must enable VIRTIO 1.0 or else use the Legacy interface */
67 g_assert_cmphex(features & (1ull << VIRTIO_F_VERSION_1), !=, 0);
68
69 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
70 offsetof(struct virtio_pci_common_cfg,
71 guest_feature_select),
72 0);
73 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
74 offsetof(struct virtio_pci_common_cfg,
75 guest_feature),
76 features);
77 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
78 offsetof(struct virtio_pci_common_cfg,
79 guest_feature_select),
80 1);
81 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
82 offsetof(struct virtio_pci_common_cfg,
83 guest_feature),
84 features >> 32);
85 }
86
get_guest_features(QVirtioDevice * d)87 static uint64_t get_guest_features(QVirtioDevice *d)
88 {
89 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
90 uint64_t lo, hi;
91
92 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
93 offsetof(struct virtio_pci_common_cfg,
94 guest_feature_select),
95 0);
96 lo = qpci_io_readl(dev->pdev, dev->bar, dev->common_cfg_offset +
97 offsetof(struct virtio_pci_common_cfg, guest_feature));
98
99 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
100 offsetof(struct virtio_pci_common_cfg,
101 guest_feature_select),
102 1);
103 hi = qpci_io_readl(dev->pdev, dev->bar, dev->common_cfg_offset +
104 offsetof(struct virtio_pci_common_cfg, guest_feature));
105
106 return (hi << 32) | lo;
107 }
108
get_status(QVirtioDevice * d)109 static uint8_t get_status(QVirtioDevice *d)
110 {
111 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
112
113 return qpci_io_readb(dev->pdev, dev->bar, dev->common_cfg_offset +
114 offsetof(struct virtio_pci_common_cfg,
115 device_status));
116 }
117
set_status(QVirtioDevice * d,uint8_t status)118 static void set_status(QVirtioDevice *d, uint8_t status)
119 {
120 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
121
122 return qpci_io_writeb(dev->pdev, dev->bar, dev->common_cfg_offset +
123 offsetof(struct virtio_pci_common_cfg,
124 device_status),
125 status);
126 }
127
get_msix_status(QVirtioPCIDevice * dev,uint32_t msix_entry,uint32_t msix_addr,uint32_t msix_data)128 static bool get_msix_status(QVirtioPCIDevice *dev, uint32_t msix_entry,
129 uint32_t msix_addr, uint32_t msix_data)
130 {
131 uint32_t data;
132
133 g_assert_cmpint(msix_entry, !=, -1);
134 if (qpci_msix_masked(dev->pdev, msix_entry)) {
135 /* No ISR checking should be done if masked, but read anyway */
136 return qpci_msix_pending(dev->pdev, msix_entry);
137 }
138
139 data = qtest_readl(dev->pdev->bus->qts, msix_addr);
140 if (data == msix_data) {
141 qtest_writel(dev->pdev->bus->qts, msix_addr, 0);
142 return true;
143 } else {
144 return false;
145 }
146 }
147
get_queue_isr_status(QVirtioDevice * d,QVirtQueue * vq)148 static bool get_queue_isr_status(QVirtioDevice *d, QVirtQueue *vq)
149 {
150 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
151
152 if (dev->pdev->msix_enabled) {
153 QVirtQueuePCI *vqpci = container_of(vq, QVirtQueuePCI, vq);
154
155 return get_msix_status(dev, vqpci->msix_entry, vqpci->msix_addr,
156 vqpci->msix_data);
157 }
158
159 return qpci_io_readb(dev->pdev, dev->bar, dev->isr_cfg_offset) & 1;
160 }
161
get_config_isr_status(QVirtioDevice * d)162 static bool get_config_isr_status(QVirtioDevice *d)
163 {
164 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
165
166 if (dev->pdev->msix_enabled) {
167 return get_msix_status(dev, dev->config_msix_entry,
168 dev->config_msix_addr, dev->config_msix_data);
169 }
170
171 return qpci_io_readb(dev->pdev, dev->bar, dev->isr_cfg_offset) & 2;
172 }
173
wait_config_isr_status(QVirtioDevice * d,gint64 timeout_us)174 static void wait_config_isr_status(QVirtioDevice *d, gint64 timeout_us)
175 {
176 gint64 start_time = g_get_monotonic_time();
177
178 while (!get_config_isr_status(d)) {
179 g_assert(g_get_monotonic_time() - start_time <= timeout_us);
180 }
181 }
182
queue_select(QVirtioDevice * d,uint16_t index)183 static void queue_select(QVirtioDevice *d, uint16_t index)
184 {
185 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
186
187 qpci_io_writew(dev->pdev, dev->bar, dev->common_cfg_offset +
188 offsetof(struct virtio_pci_common_cfg, queue_select),
189 index);
190 }
191
get_queue_size(QVirtioDevice * d)192 static uint16_t get_queue_size(QVirtioDevice *d)
193 {
194 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
195
196 return qpci_io_readw(dev->pdev, dev->bar, dev->common_cfg_offset +
197 offsetof(struct virtio_pci_common_cfg, queue_size));
198 }
199
set_queue_address(QVirtioDevice * d,QVirtQueue * vq)200 static void set_queue_address(QVirtioDevice *d, QVirtQueue *vq)
201 {
202 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
203
204 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
205 offsetof(struct virtio_pci_common_cfg, queue_desc_lo),
206 vq->desc);
207 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
208 offsetof(struct virtio_pci_common_cfg, queue_desc_hi),
209 vq->desc >> 32);
210
211 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
212 offsetof(struct virtio_pci_common_cfg, queue_avail_lo),
213 vq->avail);
214 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
215 offsetof(struct virtio_pci_common_cfg, queue_avail_hi),
216 vq->avail >> 32);
217
218 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
219 offsetof(struct virtio_pci_common_cfg, queue_used_lo),
220 vq->used);
221 qpci_io_writel(dev->pdev, dev->bar, dev->common_cfg_offset +
222 offsetof(struct virtio_pci_common_cfg, queue_used_hi),
223 vq->used >> 32);
224 }
225
virtqueue_setup(QVirtioDevice * d,QGuestAllocator * alloc,uint16_t index)226 static QVirtQueue *virtqueue_setup(QVirtioDevice *d, QGuestAllocator *alloc,
227 uint16_t index)
228 {
229 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
230 QVirtQueue *vq;
231 QVirtQueuePCI *vqpci;
232 uint16_t notify_off;
233
234 vq = qvirtio_pci_virtqueue_setup_common(d, alloc, index);
235 vqpci = container_of(vq, QVirtQueuePCI, vq);
236
237 notify_off = qpci_io_readw(dev->pdev, dev->bar, dev->common_cfg_offset +
238 offsetof(struct virtio_pci_common_cfg,
239 queue_notify_off));
240
241 vqpci->notify_offset = dev->notify_cfg_offset +
242 notify_off * dev->notify_off_multiplier;
243
244 qpci_io_writew(dev->pdev, dev->bar, dev->common_cfg_offset +
245 offsetof(struct virtio_pci_common_cfg, queue_enable), 1);
246
247 return vq;
248 }
249
virtqueue_kick(QVirtioDevice * d,QVirtQueue * vq)250 static void virtqueue_kick(QVirtioDevice *d, QVirtQueue *vq)
251 {
252 QVirtioPCIDevice *dev = container_of(d, QVirtioPCIDevice, vdev);
253 QVirtQueuePCI *vqpci = container_of(vq, QVirtQueuePCI, vq);
254
255 qpci_io_writew(dev->pdev, dev->bar, vqpci->notify_offset, vq->index);
256 }
257
258 static const QVirtioBus qvirtio_pci_virtio_1 = {
259 .config_readb = config_readb,
260 .config_readw = config_readw,
261 .config_readl = config_readl,
262 .config_readq = config_readq,
263 .get_features = get_features,
264 .set_features = set_features,
265 .get_guest_features = get_guest_features,
266 .get_status = get_status,
267 .set_status = set_status,
268 .get_queue_isr_status = get_queue_isr_status,
269 .wait_config_isr_status = wait_config_isr_status,
270 .queue_select = queue_select,
271 .get_queue_size = get_queue_size,
272 .set_queue_address = set_queue_address,
273 .virtqueue_setup = virtqueue_setup,
274 .virtqueue_cleanup = qvirtio_pci_virtqueue_cleanup_common,
275 .virtqueue_kick = virtqueue_kick,
276 };
277
set_config_vector(QVirtioPCIDevice * d,uint16_t entry)278 static void set_config_vector(QVirtioPCIDevice *d, uint16_t entry)
279 {
280 uint16_t vector;
281
282 qpci_io_writew(d->pdev, d->bar, d->common_cfg_offset +
283 offsetof(struct virtio_pci_common_cfg, msix_config), entry);
284 vector = qpci_io_readw(d->pdev, d->bar, d->common_cfg_offset +
285 offsetof(struct virtio_pci_common_cfg,
286 msix_config));
287 g_assert_cmphex(vector, !=, VIRTIO_MSI_NO_VECTOR);
288 }
289
set_queue_vector(QVirtioPCIDevice * d,uint16_t vq_idx,uint16_t entry)290 static void set_queue_vector(QVirtioPCIDevice *d, uint16_t vq_idx,
291 uint16_t entry)
292 {
293 uint16_t vector;
294
295 queue_select(&d->vdev, vq_idx);
296 qpci_io_writew(d->pdev, d->bar, d->common_cfg_offset +
297 offsetof(struct virtio_pci_common_cfg, queue_msix_vector),
298 entry);
299 vector = qpci_io_readw(d->pdev, d->bar, d->common_cfg_offset +
300 offsetof(struct virtio_pci_common_cfg,
301 queue_msix_vector));
302 g_assert_cmphex(vector, !=, VIRTIO_MSI_NO_VECTOR);
303 }
304
305 static const QVirtioPCIMSIXOps qvirtio_pci_msix_ops_virtio_1 = {
306 .set_config_vector = set_config_vector,
307 .set_queue_vector = set_queue_vector,
308 };
309
probe_device_type(QVirtioPCIDevice * dev)310 static bool probe_device_type(QVirtioPCIDevice *dev)
311 {
312 uint16_t vendor_id;
313 uint16_t device_id;
314
315 /* "Drivers MUST match devices with the PCI Vendor ID 0x1AF4" */
316 vendor_id = qpci_config_readw(dev->pdev, PCI_VENDOR_ID);
317 if (vendor_id != 0x1af4) {
318 return false;
319 }
320
321 /*
322 * "Any PCI device with ... PCI Device ID 0x1000 through 0x107F inclusive
323 * is a virtio device"
324 */
325 device_id = qpci_config_readw(dev->pdev, PCI_DEVICE_ID);
326 if (device_id < 0x1000 || device_id > 0x107f) {
327 return false;
328 }
329
330 /*
331 * "Devices MAY utilize a Transitional PCI Device ID range, 0x1000 to
332 * 0x103F depending on the device type"
333 */
334 if (device_id < 0x1040) {
335 /*
336 * "Transitional devices MUST have the PCI Subsystem Device ID matching
337 * the Virtio Device ID"
338 */
339 dev->vdev.device_type = qpci_config_readw(dev->pdev, PCI_SUBSYSTEM_ID);
340 } else {
341 /*
342 * "The PCI Device ID is calculated by adding 0x1040 to the Virtio
343 * Device ID"
344 */
345 dev->vdev.device_type = device_id - 0x1040;
346 }
347
348 return true;
349 }
350
351 /* Find the first VIRTIO 1.0 PCI structure for a given type */
find_structure(QVirtioPCIDevice * dev,uint8_t cfg_type,uint8_t * bar,uint32_t * offset,uint32_t * length,uint8_t * cfg_addr)352 static bool find_structure(QVirtioPCIDevice *dev, uint8_t cfg_type,
353 uint8_t *bar, uint32_t *offset, uint32_t *length,
354 uint8_t *cfg_addr)
355 {
356 uint8_t addr = 0;
357
358 while ((addr = qpci_find_capability(dev->pdev, PCI_CAP_ID_VNDR,
359 addr)) != 0) {
360 uint8_t type;
361
362 type = qpci_config_readb(dev->pdev,
363 addr + offsetof(struct virtio_pci_cap, cfg_type));
364 if (type != cfg_type) {
365 continue;
366 }
367
368 *bar = qpci_config_readb(dev->pdev,
369 addr + offsetof(struct virtio_pci_cap, bar));
370 *offset = qpci_config_readl(dev->pdev,
371 addr + offsetof(struct virtio_pci_cap, offset));
372 *length = qpci_config_readl(dev->pdev,
373 addr + offsetof(struct virtio_pci_cap, length));
374 if (cfg_addr) {
375 *cfg_addr = addr;
376 }
377
378 return true;
379 }
380
381 return false;
382 }
383
probe_device_layout(QVirtioPCIDevice * dev)384 static bool probe_device_layout(QVirtioPCIDevice *dev)
385 {
386 uint8_t bar;
387 uint8_t cfg_addr;
388 uint32_t length;
389
390 /*
391 * Due to the qpci_iomap() API we only support devices that put all
392 * structures in the same PCI BAR. Luckily this is true with QEMU.
393 */
394
395 if (!find_structure(dev, VIRTIO_PCI_CAP_COMMON_CFG, &bar,
396 &dev->common_cfg_offset, &length, NULL)) {
397 return false;
398 }
399 dev->bar_idx = bar;
400
401 if (!find_structure(dev, VIRTIO_PCI_CAP_NOTIFY_CFG, &bar,
402 &dev->notify_cfg_offset, &length, &cfg_addr)) {
403 return false;
404 }
405 g_assert_cmphex(bar, ==, dev->bar_idx);
406
407 dev->notify_off_multiplier = qpci_config_readl(dev->pdev,
408 cfg_addr + offsetof(struct virtio_pci_notify_cap,
409 notify_off_multiplier));
410
411 if (!find_structure(dev, VIRTIO_PCI_CAP_ISR_CFG, &bar,
412 &dev->isr_cfg_offset, &length, NULL)) {
413 return false;
414 }
415 g_assert_cmphex(bar, ==, dev->bar_idx);
416
417 if (!find_structure(dev, VIRTIO_PCI_CAP_DEVICE_CFG, &bar,
418 &dev->device_cfg_offset, &length, NULL)) {
419 return false;
420 }
421 g_assert_cmphex(bar, ==, dev->bar_idx);
422
423 return true;
424 }
425
426 /* Probe a VIRTIO 1.0 device */
qvirtio_pci_init_virtio_1(QVirtioPCIDevice * dev)427 bool qvirtio_pci_init_virtio_1(QVirtioPCIDevice *dev)
428 {
429 if (!probe_device_type(dev)) {
430 return false;
431 }
432
433 if (!probe_device_layout(dev)) {
434 return false;
435 }
436
437 dev->vdev.bus = &qvirtio_pci_virtio_1;
438 dev->msix_ops = &qvirtio_pci_msix_ops_virtio_1;
439 dev->vdev.big_endian = false;
440 return true;
441 }
442