37 static MSIMessage msix_prepare_message(PCIDevice *dev, unsigned vector)  in msix_prepare_message()  argument
39     uint8_t *table_entry = dev->msix_table + vector * PCI_MSIX_ENTRY_SIZE;  in msix_prepare_message()
47 MSIMessage msix_get_message(PCIDevice *dev, unsigned vector)  in msix_get_message()  argument
49     return dev->msix_prepare_message(dev, vector);  in msix_get_message()
56 void msix_set_message(PCIDevice *dev, int vector, struct MSIMessage msg)  in msix_set_message()  argument
58     uint8_t *table_entry = dev->msix_table + vector * PCI_MSIX_ENTRY_SIZE;  in msix_set_message()
70 static uint8_t *msix_pending_byte(PCIDevice *dev, int vector)  in msix_pending_byte()  argument
72     return dev->msix_pba + vector / 8;  in msix_pending_byte()
75 static int msix_is_pending(PCIDevice *dev, int vector)  in msix_is_pending()  argument
77     return *msix_pending_byte(dev, vector) & msix_pending_mask(vector);  in msix_is_pending()
80 void msix_set_pending(PCIDevice *dev, unsigned int vector)  in msix_set_pending()  argument
82     *msix_pending_byte(dev, vector) |= msix_pending_mask(vector);  in msix_set_pending()
85 void msix_clr_pending(PCIDevice *dev, int vector)  in msix_clr_pending()  argument
87     *msix_pending_byte(dev, vector) &= ~msix_pending_mask(vector);  in msix_clr_pending()
90 static bool msix_vector_masked(PCIDevice *dev, unsigned int vector, bool fmask)  in msix_vector_masked()  argument
93     uint8_t *data = &dev->msix_table[offset + PCI_MSIX_ENTRY_DATA];  in msix_vector_masked()
99     return fmask || dev->msix_table[offset + PCI_MSIX_ENTRY_VECTOR_CTRL] &  in msix_vector_masked()
103 bool msix_is_masked(PCIDevice *dev, unsigned int vector)  in msix_is_masked()  argument
105     return msix_vector_masked(dev, vector, dev->msix_function_masked);  in msix_is_masked()
108 static void msix_fire_vector_notifier(PCIDevice *dev,  in msix_fire_vector_notifier()  argument
114     if (!dev->msix_vector_use_notifier) {  in msix_fire_vector_notifier()
118         dev->msix_vector_release_notifier(dev, vector);  in msix_fire_vector_notifier()
120         msg = msix_get_message(dev, vector);  in msix_fire_vector_notifier()
121         ret = dev->msix_vector_use_notifier(dev, vector, msg);  in msix_fire_vector_notifier()
126 static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked)  in msix_handle_mask_update()  argument
128     bool is_masked = msix_is_masked(dev, vector);  in msix_handle_mask_update()
131         MSIMessage msg = msix_prepare_message(dev, vector);  in msix_handle_mask_update()
133         xen_evtchn_snoop_msi(dev, true, vector, msg.address, msg.data,  in msix_handle_mask_update()
141     msix_fire_vector_notifier(dev, vector, is_masked);  in msix_handle_mask_update()
143     if (!is_masked && msix_is_pending(dev, vector)) {  in msix_handle_mask_update()
144         msix_clr_pending(dev, vector);  in msix_handle_mask_update()
145         msix_notify(dev, vector);  in msix_handle_mask_update()
149 void msix_set_mask(PCIDevice *dev, int vector, bool mask)  in msix_set_mask()  argument
154     assert(vector < dev->msix_entries_nr);  in msix_set_mask()
158     was_masked = msix_is_masked(dev, vector);  in msix_set_mask()
161         dev->msix_table[offset] |= PCI_MSIX_ENTRY_CTRL_MASKBIT;  in msix_set_mask()
163         dev->msix_table[offset] &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT;  in msix_set_mask()
166     msix_handle_mask_update(dev, vector, was_masked);  in msix_set_mask()
169 static bool msix_masked(PCIDevice *dev)  in msix_masked()  argument
171     return dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK;  in msix_masked()
174 static void msix_update_function_masked(PCIDevice *dev)  in msix_update_function_masked()  argument
176     dev->msix_function_masked = !msix_enabled(dev) || msix_masked(dev);  in msix_update_function_masked()
180 void msix_write_config(PCIDevice *dev, uint32_t addr,  in msix_write_config()  argument
183     unsigned enable_pos = dev->msix_cap + MSIX_CONTROL_OFFSET;  in msix_write_config()
187     if (!msix_present(dev) || !range_covers_byte(addr, len, enable_pos)) {  in msix_write_config()
191     trace_msix_write_config(dev->name, msix_enabled(dev), msix_masked(dev));  in msix_write_config()
193     was_masked = dev->msix_function_masked;  in msix_write_config()
194     msix_update_function_masked(dev);  in msix_write_config()
196     if (!msix_enabled(dev)) {  in msix_write_config()
200     pci_device_deassert_intx(dev);  in msix_write_config()
202     if (dev->msix_function_masked == was_masked) {  in msix_write_config()
206     for (vector = 0; vector < dev->msix_entries_nr; ++vector) {  in msix_write_config()
207         msix_handle_mask_update(dev, vector,  in msix_write_config()
208                                 msix_vector_masked(dev, vector, was_masked));  in msix_write_config()
215     PCIDevice *dev = opaque;  in msix_table_mmio_read()  local
217     assert(addr + size <= dev->msix_entries_nr * PCI_MSIX_ENTRY_SIZE);  in msix_table_mmio_read()
218     return pci_get_long(dev->msix_table + addr);  in msix_table_mmio_read()
224     PCIDevice *dev = opaque;  in msix_table_mmio_write()  local
228     assert(addr + size <= dev->msix_entries_nr * PCI_MSIX_ENTRY_SIZE);  in msix_table_mmio_write()
230     was_masked = msix_is_masked(dev, vector);  in msix_table_mmio_write()
231     pci_set_long(dev->msix_table + addr, val);  in msix_table_mmio_write()
232     msix_handle_mask_update(dev, vector, was_masked);  in msix_table_mmio_write()
251     PCIDevice *dev = opaque;  in msix_pba_mmio_read()  local
252     if (dev->msix_vector_poll_notifier) {  in msix_pba_mmio_read()
254         unsigned vector_end = MIN((addr + size) * 8, dev->msix_entries_nr);  in msix_pba_mmio_read()
255         dev->msix_vector_poll_notifier(dev, vector_start, vector_end);  in msix_pba_mmio_read()
258     return pci_get_long(dev->msix_pba + addr);  in msix_pba_mmio_read()
264     PCIDevice *dev = opaque;  in msix_pba_mmio_write()  local
269                   pci_root_bus_path(dev), pci_dev_bus_num(dev),  in msix_pba_mmio_write()
270                   PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),  in msix_pba_mmio_write()
287 static void msix_mask_all(struct PCIDevice *dev, unsigned nentries)  in msix_mask_all()  argument
294         bool was_masked = msix_is_masked(dev, vector);  in msix_mask_all()
296         dev->msix_table[offset] |= PCI_MSIX_ENTRY_CTRL_MASKBIT;  in msix_mask_all()
297         msix_handle_mask_update(dev, vector, was_masked);  in msix_mask_all()
321 int msix_init(struct PCIDevice *dev, unsigned short nentries,  in msix_init()  argument
356     cap = pci_add_capability(dev, PCI_CAP_ID_MSIX,  in msix_init()
362     dev->msix_cap = cap;  in msix_init()
363     dev->cap_present |= QEMU_PCI_CAP_MSIX;  in msix_init()
364     config = dev->config + cap;  in msix_init()
367     dev->msix_entries_nr = nentries;  in msix_init()
368     dev->msix_function_masked = true;  in msix_init()
374     dev->wmask[cap + MSIX_CONTROL_OFFSET] |= MSIX_ENABLE_MASK |  in msix_init()
377     dev->msix_table = g_malloc0(table_size);  in msix_init()
378     dev->msix_pba = g_malloc0(pba_size);  in msix_init()
379     dev->msix_entry_used = g_malloc0(nentries * sizeof *dev->msix_entry_used);  in msix_init()
381     msix_mask_all(dev, nentries);  in msix_init()
383     memory_region_init_io(&dev->msix_table_mmio, OBJECT(dev), &msix_table_mmio_ops, dev,  in msix_init()
385     memory_region_add_subregion(table_bar, table_offset, &dev->msix_table_mmio);  in msix_init()
386     memory_region_init_io(&dev->msix_pba_mmio, OBJECT(dev), &msix_pba_mmio_ops, dev,  in msix_init()
388     memory_region_add_subregion(pba_bar, pba_offset, &dev->msix_pba_mmio);  in msix_init()
390     dev->msix_prepare_message = msix_prepare_message;  in msix_init()
395 int msix_init_exclusive_bar(PCIDevice *dev, unsigned short nentries,  in msix_init_exclusive_bar()  argument
421     name = g_strdup_printf("%s-msix", dev->name);  in msix_init_exclusive_bar()
422     memory_region_init(&dev->msix_exclusive_bar, OBJECT(dev), name, bar_size);  in msix_init_exclusive_bar()
425     ret = msix_init(dev, nentries, &dev->msix_exclusive_bar, bar_nr,  in msix_init_exclusive_bar()
426                     0, &dev->msix_exclusive_bar,  in msix_init_exclusive_bar()
433     pci_register_bar(dev, bar_nr, PCI_BASE_ADDRESS_SPACE_MEMORY,  in msix_init_exclusive_bar()
434                      &dev->msix_exclusive_bar);  in msix_init_exclusive_bar()
439 static void msix_free_irq_entries(PCIDevice *dev)  in msix_free_irq_entries()  argument
443     for (vector = 0; vector < dev->msix_entries_nr; ++vector) {  in msix_free_irq_entries()
444         dev->msix_entry_used[vector] = 0;  in msix_free_irq_entries()
445         msix_clr_pending(dev, vector);  in msix_free_irq_entries()
449 static void msix_clear_all_vectors(PCIDevice *dev)  in msix_clear_all_vectors()  argument
453     for (vector = 0; vector < dev->msix_entries_nr; ++vector) {  in msix_clear_all_vectors()
454         msix_clr_pending(dev, vector);  in msix_clear_all_vectors()
459 void msix_uninit(PCIDevice *dev, MemoryRegion *table_bar, MemoryRegion *pba_bar)  in msix_uninit()  argument
461     if (!msix_present(dev)) {  in msix_uninit()
464     pci_del_capability(dev, PCI_CAP_ID_MSIX, MSIX_CAP_LENGTH);  in msix_uninit()
465     dev->msix_cap = 0;  in msix_uninit()
466     msix_free_irq_entries(dev);  in msix_uninit()
467     dev->msix_entries_nr = 0;  in msix_uninit()
468     memory_region_del_subregion(pba_bar, &dev->msix_pba_mmio);  in msix_uninit()
469     g_free(dev->msix_pba);  in msix_uninit()
470     dev->msix_pba = NULL;  in msix_uninit()
471     memory_region_del_subregion(table_bar, &dev->msix_table_mmio);  in msix_uninit()
472     g_free(dev->msix_table);  in msix_uninit()
473     dev->msix_table = NULL;  in msix_uninit()
474     g_free(dev->msix_entry_used);  in msix_uninit()
475     dev->msix_entry_used = NULL;  in msix_uninit()
476     dev->cap_present &= ~QEMU_PCI_CAP_MSIX;  in msix_uninit()
477     dev->msix_prepare_message = NULL;  in msix_uninit()
480 void msix_uninit_exclusive_bar(PCIDevice *dev)  in msix_uninit_exclusive_bar()  argument
482     if (msix_present(dev)) {  in msix_uninit_exclusive_bar()
483         msix_uninit(dev, &dev->msix_exclusive_bar, &dev->msix_exclusive_bar);  in msix_uninit_exclusive_bar()
487 void msix_save(PCIDevice *dev, QEMUFile *f)  in msix_save()  argument
489     unsigned n = dev->msix_entries_nr;  in msix_save()
491     if (!msix_present(dev)) {  in msix_save()
495     qemu_put_buffer(f, dev->msix_table, n * PCI_MSIX_ENTRY_SIZE);  in msix_save()
496     qemu_put_buffer(f, dev->msix_pba, DIV_ROUND_UP(n, 8));  in msix_save()
500 void msix_load(PCIDevice *dev, QEMUFile *f)  in msix_load()  argument
502     unsigned n = dev->msix_entries_nr;  in msix_load()
505     if (!msix_present(dev)) {  in msix_load()
509     msix_clear_all_vectors(dev);  in msix_load()
510     qemu_get_buffer(f, dev->msix_table, n * PCI_MSIX_ENTRY_SIZE);  in msix_load()
511     qemu_get_buffer(f, dev->msix_pba, DIV_ROUND_UP(n, 8));  in msix_load()
512     msix_update_function_masked(dev);  in msix_load()
515         msix_handle_mask_update(dev, vector, true);  in msix_load()
520 int msix_present(PCIDevice *dev)  in msix_present()  argument
522     return dev->cap_present & QEMU_PCI_CAP_MSIX;  in msix_present()
526 int msix_enabled(PCIDevice *dev)  in msix_enabled()  argument
528     return (dev->cap_present & QEMU_PCI_CAP_MSIX) &&  in msix_enabled()
529         (dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &  in msix_enabled()
534 void msix_notify(PCIDevice *dev, unsigned vector)  in msix_notify()  argument
538     assert(vector < dev->msix_entries_nr);  in msix_notify()
540     if (!dev->msix_entry_used[vector]) {  in msix_notify()
544     if (msix_is_masked(dev, vector)) {  in msix_notify()
545         msix_set_pending(dev, vector);  in msix_notify()
549     msg = msix_get_message(dev, vector);  in msix_notify()
551     msi_send_message(dev, msg);  in msix_notify()
554 void msix_reset(PCIDevice *dev)  in msix_reset()  argument
556     if (!msix_present(dev)) {  in msix_reset()
559     msix_clear_all_vectors(dev);  in msix_reset()
560     dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &=  in msix_reset()
561             ~dev->wmask[dev->msix_cap + MSIX_CONTROL_OFFSET];  in msix_reset()
562     memset(dev->msix_table, 0, dev->msix_entries_nr * PCI_MSIX_ENTRY_SIZE);  in msix_reset()
563     memset(dev->msix_pba, 0, QEMU_ALIGN_UP(dev->msix_entries_nr, 64) / 8);  in msix_reset()
564     msix_mask_all(dev, dev->msix_entries_nr);  in msix_reset()
576 void msix_vector_use(PCIDevice *dev, unsigned vector)  in msix_vector_use()  argument
578     assert(vector < dev->msix_entries_nr);  in msix_vector_use()
579     dev->msix_entry_used[vector]++;  in msix_vector_use()
583 void msix_vector_unuse(PCIDevice *dev, unsigned vector)  in msix_vector_unuse()  argument
585     assert(vector < dev->msix_entries_nr);  in msix_vector_unuse()
586     if (!dev->msix_entry_used[vector]) {  in msix_vector_unuse()
589     if (--dev->msix_entry_used[vector]) {  in msix_vector_unuse()
592     msix_clr_pending(dev, vector);  in msix_vector_unuse()
595 void msix_unuse_all_vectors(PCIDevice *dev)  in msix_unuse_all_vectors()  argument
597     if (!msix_present(dev)) {  in msix_unuse_all_vectors()
600     msix_free_irq_entries(dev);  in msix_unuse_all_vectors()
603 unsigned int msix_nr_vectors_allocated(const PCIDevice *dev)  in msix_nr_vectors_allocated()  argument
605     return dev->msix_entries_nr;  in msix_nr_vectors_allocated()
608 static int msix_set_notifier_for_vector(PCIDevice *dev, unsigned int vector)  in msix_set_notifier_for_vector()  argument
612     if (msix_is_masked(dev, vector)) {  in msix_set_notifier_for_vector()
615     msg = msix_get_message(dev, vector);  in msix_set_notifier_for_vector()
616     return dev->msix_vector_use_notifier(dev, vector, msg);  in msix_set_notifier_for_vector()
619 static void msix_unset_notifier_for_vector(PCIDevice *dev, unsigned int vector)  in msix_unset_notifier_for_vector()  argument
621     if (msix_is_masked(dev, vector)) {  in msix_unset_notifier_for_vector()
624     dev->msix_vector_release_notifier(dev, vector);  in msix_unset_notifier_for_vector()
627 int msix_set_vector_notifiers(PCIDevice *dev,  in msix_set_vector_notifiers()  argument
636     dev->msix_vector_use_notifier = use_notifier;  in msix_set_vector_notifiers()
637     dev->msix_vector_release_notifier = release_notifier;  in msix_set_vector_notifiers()
638     dev->msix_vector_poll_notifier = poll_notifier;  in msix_set_vector_notifiers()
640     if ((dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &  in msix_set_vector_notifiers()
642         for (vector = 0; vector < dev->msix_entries_nr; vector++) {  in msix_set_vector_notifiers()
643             ret = msix_set_notifier_for_vector(dev, vector);  in msix_set_vector_notifiers()
649     if (dev->msix_vector_poll_notifier) {  in msix_set_vector_notifiers()
650         dev->msix_vector_poll_notifier(dev, 0, dev->msix_entries_nr);  in msix_set_vector_notifiers()
656         msix_unset_notifier_for_vector(dev, vector);  in msix_set_vector_notifiers()
658     dev->msix_vector_use_notifier = NULL;  in msix_set_vector_notifiers()
659     dev->msix_vector_release_notifier = NULL;  in msix_set_vector_notifiers()
660     dev->msix_vector_poll_notifier = NULL;  in msix_set_vector_notifiers()
664 void msix_unset_vector_notifiers(PCIDevice *dev)  in msix_unset_vector_notifiers()  argument
668     assert(dev->msix_vector_use_notifier &&  in msix_unset_vector_notifiers()
669            dev->msix_vector_release_notifier);  in msix_unset_vector_notifiers()
671     if ((dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &  in msix_unset_vector_notifiers()
673         for (vector = 0; vector < dev->msix_entries_nr; vector++) {  in msix_unset_vector_notifiers()
674             msix_unset_notifier_for_vector(dev, vector);  in msix_unset_vector_notifiers()
677     dev->msix_vector_use_notifier = NULL;  in msix_unset_vector_notifiers()
678     dev->msix_vector_release_notifier = NULL;  in msix_unset_vector_notifiers()
679     dev->msix_vector_poll_notifier = NULL;  in msix_unset_vector_notifiers()