Lines Matching +full:post +full:- +full:processing
4 * Copyright 2016 - 2018 Red Hat, Inc.
11 * See the COPYING file in the top-level directory.
19 #include "qemu/defer-call.h"
20 #include "qemu/error-report.h"
21 #include "qemu/host-pci-mmio.h"
22 #include "qemu/main-loop.h"
27 #include "qemu/vfio-helpers.h"
28 #include "block/block-io.h"
30 #include "system/block-backend.h"
45 #define NVME_NUM_REQS (NVME_QUEUE_SIZE - 1)
73 int free_req_next; /* q->reqs[] index of next free req */
95 /* Thread-safe, no lock necessary */
177 q->head = q->tail = 0; in nvme_init_queue()
178 q->queue = qemu_try_memalign(qemu_real_host_page_size(), bytes); in nvme_init_queue()
179 if (!q->queue) { in nvme_init_queue()
183 memset(q->queue, 0, bytes); in nvme_init_queue()
184 r = qemu_vfio_dma_map(s->vfio, q->queue, bytes, false, &q->iova, errp); in nvme_init_queue()
193 qemu_vfree(q->queue); in nvme_free_queue()
198 trace_nvme_free_queue_pair(q->index, q, &q->cq, &q->sq); in nvme_free_queue_pair()
199 if (q->completion_bh) { in nvme_free_queue_pair()
200 qemu_bh_delete(q->completion_bh); in nvme_free_queue_pair()
202 nvme_free_queue(&q->sq); in nvme_free_queue_pair()
203 nvme_free_queue(&q->cq); in nvme_free_queue_pair()
204 qemu_vfree(q->prp_list_pages); in nvme_free_queue_pair()
205 qemu_mutex_destroy(&q->lock); in nvme_free_queue_pair()
213 qemu_mutex_lock(&q->lock); in nvme_free_req_queue_cb()
214 while (q->free_req_head != -1 && in nvme_free_req_queue_cb()
215 qemu_co_enter_next(&q->free_req_queue, &q->lock)) { in nvme_free_req_queue_cb()
218 qemu_mutex_unlock(&q->lock); in nvme_free_req_queue_cb()
238 event_notifier_get_fd(s->irq_notifier)); in nvme_create_queue_pair()
239 bytes = QEMU_ALIGN_UP(s->page_size * NVME_NUM_REQS, in nvme_create_queue_pair()
241 q->prp_list_pages = qemu_try_memalign(qemu_real_host_page_size(), bytes); in nvme_create_queue_pair()
242 if (!q->prp_list_pages) { in nvme_create_queue_pair()
246 memset(q->prp_list_pages, 0, bytes); in nvme_create_queue_pair()
247 qemu_mutex_init(&q->lock); in nvme_create_queue_pair()
248 q->s = s; in nvme_create_queue_pair()
249 q->index = idx; in nvme_create_queue_pair()
250 qemu_co_queue_init(&q->free_req_queue); in nvme_create_queue_pair()
251 q->completion_bh = aio_bh_new(aio_context, nvme_process_completion_bh, q); in nvme_create_queue_pair()
252 r = qemu_vfio_dma_map(s->vfio, q->prp_list_pages, bytes, in nvme_create_queue_pair()
258 q->free_req_head = -1; in nvme_create_queue_pair()
260 NVMeRequest *req = &q->reqs[i]; in nvme_create_queue_pair()
261 req->cid = i + 1; in nvme_create_queue_pair()
262 req->free_req_next = q->free_req_head; in nvme_create_queue_pair()
263 q->free_req_head = i; in nvme_create_queue_pair()
264 req->prp_list_page = q->prp_list_pages + i * s->page_size; in nvme_create_queue_pair()
265 req->prp_list_iova = prp_list_iova + i * s->page_size; in nvme_create_queue_pair()
268 if (!nvme_init_queue(s, &q->sq, size, NVME_SQ_ENTRY_BYTES, errp)) { in nvme_create_queue_pair()
271 q->sq.doorbell = &s->doorbells[idx * s->doorbell_scale].sq_tail; in nvme_create_queue_pair()
273 if (!nvme_init_queue(s, &q->cq, size, NVME_CQ_ENTRY_BYTES, errp)) { in nvme_create_queue_pair()
276 q->cq.doorbell = &s->doorbells[idx * s->doorbell_scale].cq_head; in nvme_create_queue_pair()
284 /* With q->lock */
287 BDRVNVMeState *s = q->s; in nvme_kick()
289 if (!q->need_kick) { in nvme_kick()
292 trace_nvme_kick(s, q->index); in nvme_kick()
293 assert(!(q->sq.tail & 0xFF00)); in nvme_kick()
296 host_pci_stl_le_p(q->sq.doorbell, q->sq.tail); in nvme_kick()
297 q->inflight += q->need_kick; in nvme_kick()
298 q->need_kick = 0; in nvme_kick()
305 req = &q->reqs[q->free_req_head]; in nvme_get_free_req_nofail_locked()
306 q->free_req_head = req->free_req_next; in nvme_get_free_req_nofail_locked()
307 req->free_req_next = -1; in nvme_get_free_req_nofail_locked()
314 QEMU_LOCK_GUARD(&q->lock); in nvme_get_free_req_nowait()
315 if (q->free_req_head == -1) { in nvme_get_free_req_nowait()
327 QEMU_LOCK_GUARD(&q->lock); in nvme_get_free_req()
329 while (q->free_req_head == -1) { in nvme_get_free_req()
330 trace_nvme_free_req_queue_wait(q->s, q->index); in nvme_get_free_req()
331 qemu_co_queue_wait(&q->free_req_queue, &q->lock); in nvme_get_free_req()
337 /* With q->lock */
340 req->free_req_next = q->free_req_head; in nvme_put_free_req_locked()
341 q->free_req_head = req - q->reqs; in nvme_put_free_req_locked()
344 /* With q->lock */
347 if (!qemu_co_queue_empty(&q->free_req_queue)) { in nvme_wake_free_req_locked()
348 replay_bh_schedule_oneshot_event(q->s->aio_context, in nvme_wake_free_req_locked()
356 qemu_mutex_lock(&q->lock); in nvme_put_free_req_and_wake()
359 qemu_mutex_unlock(&q->lock); in nvme_put_free_req_and_wake()
364 uint16_t status = (le16_to_cpu(c->status) >> 1) & 0xFF; in nvme_translate_error()
366 trace_nvme_error(le32_to_cpu(c->result), in nvme_translate_error()
367 le16_to_cpu(c->sq_head), in nvme_translate_error()
368 le16_to_cpu(c->sq_id), in nvme_translate_error()
369 le16_to_cpu(c->cid), in nvme_translate_error()
376 return -ENOSYS; in nvme_translate_error()
378 return -EINVAL; in nvme_translate_error()
380 return -EIO; in nvme_translate_error()
384 /* With q->lock */
387 BDRVNVMeState *s = q->s; in nvme_process_completion()
393 trace_nvme_process_completion(s, q->index, q->inflight); in nvme_process_completion()
396 * Support re-entrancy when a request cb() function invokes aio_poll(). in nvme_process_completion()
401 * processing there. in nvme_process_completion()
403 qemu_bh_schedule(q->completion_bh); in nvme_process_completion()
405 assert(q->inflight >= 0); in nvme_process_completion()
406 while (q->inflight) { in nvme_process_completion()
410 c = (NvmeCqe *)&q->cq.queue[q->cq.head * NVME_CQ_ENTRY_BYTES]; in nvme_process_completion()
411 if ((le16_to_cpu(c->status) & 0x1) == q->cq_phase) { in nvme_process_completion()
416 s->stats.completion_errors++; in nvme_process_completion()
418 q->cq.head = (q->cq.head + 1) % NVME_QUEUE_SIZE; in nvme_process_completion()
419 if (!q->cq.head) { in nvme_process_completion()
420 q->cq_phase = !q->cq_phase; in nvme_process_completion()
422 cid = le16_to_cpu(c->cid); in nvme_process_completion()
429 trace_nvme_complete_command(s, q->index, cid); in nvme_process_completion()
430 preq = &q->reqs[cid - 1]; in nvme_process_completion()
435 preq->cb = preq->opaque = NULL; in nvme_process_completion()
436 q->inflight--; in nvme_process_completion()
437 qemu_mutex_unlock(&q->lock); in nvme_process_completion()
439 qemu_mutex_lock(&q->lock); in nvme_process_completion()
443 /* Notify the device so it can post more completions. */ in nvme_process_completion()
445 host_pci_stl_le_p(q->cq.doorbell, q->cq.head); in nvme_process_completion()
449 qemu_bh_cancel(q->completion_bh); in nvme_process_completion()
464 host_pci_stl_le_p(q->cq.doorbell, q->cq.head); in nvme_process_completion_bh()
488 QEMU_LOCK_GUARD(&q->lock); in nvme_deferred_fn()
497 assert(!req->cb); in nvme_submit_command()
498 req->cb = cb; in nvme_submit_command()
499 req->opaque = opaque; in nvme_submit_command()
500 cmd->cid = cpu_to_le16(req->cid); in nvme_submit_command()
502 trace_nvme_submit_command(q->s, q->index, req->cid); in nvme_submit_command()
504 qemu_mutex_lock(&q->lock); in nvme_submit_command()
505 memcpy((uint8_t *)q->sq.queue + in nvme_submit_command()
506 q->sq.tail * NVME_SQ_ENTRY_BYTES, cmd, sizeof(*cmd)); in nvme_submit_command()
507 q->sq.tail = (q->sq.tail + 1) % NVME_QUEUE_SIZE; in nvme_submit_command()
508 q->need_kick++; in nvme_submit_command()
509 qemu_mutex_unlock(&q->lock); in nvme_submit_command()
523 BDRVNVMeState *s = bs->opaque; in nvme_admin_cmd_sync()
524 NVMeQueuePair *q = s->queues[INDEX_ADMIN]; in nvme_admin_cmd_sync()
527 int ret = -EINPROGRESS; in nvme_admin_cmd_sync()
530 return -EBUSY; in nvme_admin_cmd_sync()
534 AIO_WAIT_WHILE(aio_context, ret == -EINPROGRESS); in nvme_admin_cmd_sync()
542 BDRVNVMeState *s = bs->opaque; in nvme_identify()
563 r = qemu_vfio_dma_map(s->vfio, id, id_size, true, &iova, errp); in nvme_identify()
576 if (le32_to_cpu(id->ctrl.nn) < namespace) { in nvme_identify()
580 s->write_cache_supported = le32_to_cpu(id->ctrl.vwc) & 0x1; in nvme_identify()
581 s->max_transfer = (id->ctrl.mdts ? 1 << id->ctrl.mdts : 0) * s->page_size; in nvme_identify()
583 * s->page_size / sizeof(uint64_t) entries. */ in nvme_identify()
584 s->max_transfer = MIN_NON_ZERO(s->max_transfer, in nvme_identify()
585 s->page_size / sizeof(uint64_t) * s->page_size); in nvme_identify()
587 oncs = le16_to_cpu(id->ctrl.oncs); in nvme_identify()
588 s->supports_write_zeroes = !!(oncs & NVME_ONCS_WRITE_ZEROES); in nvme_identify()
589 s->supports_discard = !!(oncs & NVME_ONCS_DSM); in nvme_identify()
599 s->nsze = le64_to_cpu(id->ns.nsze); in nvme_identify()
600 lbaf = &id->ns.lbaf[NVME_ID_NS_FLBAS_INDEX(id->ns.flbas)]; in nvme_identify()
602 if (NVME_ID_NS_DLFEAT_WRITE_ZEROES(id->ns.dlfeat) && in nvme_identify()
603 NVME_ID_NS_DLFEAT_READ_BEHAVIOR(id->ns.dlfeat) == in nvme_identify()
605 bs->supported_write_flags |= BDRV_REQ_MAY_UNMAP; in nvme_identify()
608 if (lbaf->ms) { in nvme_identify()
613 if (lbaf->ds < BDRV_SECTOR_BITS || lbaf->ds > 12 || in nvme_identify()
614 (1 << lbaf->ds) > s->page_size) in nvme_identify()
617 lbaf->ds); in nvme_identify()
622 s->blkshift = lbaf->ds; in nvme_identify()
624 qemu_vfio_dma_unmap(s->vfio, id); in nvme_identify()
631 const size_t cqe_offset = q->cq.head * NVME_CQ_ENTRY_BYTES; in nvme_poll_queue()
632 NvmeCqe *cqe = (NvmeCqe *)&q->cq.queue[cqe_offset]; in nvme_poll_queue()
634 trace_nvme_poll_queue(q->s, q->index); in nvme_poll_queue()
636 * Do an early check for completions. q->lock isn't needed because in nvme_poll_queue()
640 if ((le16_to_cpu(cqe->status) & 0x1) == q->cq_phase) { in nvme_poll_queue()
644 qemu_mutex_lock(&q->lock); in nvme_poll_queue()
648 qemu_mutex_unlock(&q->lock); in nvme_poll_queue()
655 for (i = 0; i < s->queue_count; i++) { in nvme_poll_queues()
656 nvme_poll_queue(s->queues[i]); in nvme_poll_queues()
672 BDRVNVMeState *s = bs->opaque; in nvme_add_io_queue()
673 unsigned n = s->queue_count; in nvme_add_io_queue()
686 .dptr.prp1 = cpu_to_le64(q->cq.iova), in nvme_add_io_queue()
687 .cdw10 = cpu_to_le32(((queue_size - 1) << 16) | n), in nvme_add_io_queue()
696 .dptr.prp1 = cpu_to_le64(q->sq.iova), in nvme_add_io_queue()
697 .cdw10 = cpu_to_le32(((queue_size - 1) << 16) | n), in nvme_add_io_queue()
704 s->queues = g_renew(NVMeQueuePair *, s->queues, n + 1); in nvme_add_io_queue()
705 s->queues[n] = q; in nvme_add_io_queue()
706 s->queue_count++; in nvme_add_io_queue()
720 for (i = 0; i < s->queue_count; i++) { in nvme_poll_cb()
721 NVMeQueuePair *q = s->queues[i]; in nvme_poll_cb()
722 const size_t cqe_offset = q->cq.head * NVME_CQ_ENTRY_BYTES; in nvme_poll_cb()
723 NvmeCqe *cqe = (NvmeCqe *)&q->cq.queue[cqe_offset]; in nvme_poll_cb()
726 * q->lock isn't needed because nvme_process_completion() only runs in in nvme_poll_cb()
729 if ((le16_to_cpu(cqe->status) & 0x1) != q->cq_phase) { in nvme_poll_cb()
747 BDRVNVMeState *s = bs->opaque; in nvme_init()
758 qemu_co_mutex_init(&s->dma_map_lock); in nvme_init()
759 qemu_co_queue_init(&s->dma_flush_queue); in nvme_init()
760 s->device = g_strdup(device); in nvme_init()
761 s->nsid = namespace; in nvme_init()
762 s->aio_context = bdrv_get_aio_context(bs); in nvme_init()
763 ret = event_notifier_init(&s->irq_notifier[MSIX_SHARED_IRQ_IDX], 0); in nvme_init()
769 s->vfio = qemu_vfio_open_pci(device, errp); in nvme_init()
770 if (!s->vfio) { in nvme_init()
771 ret = -EINVAL; in nvme_init()
775 regs = qemu_vfio_pci_map_bar(s->vfio, 0, 0, sizeof(NvmeBar), in nvme_init()
778 ret = -EINVAL; in nvme_init()
784 cap = host_pci_ldq_le_p(®s->cap); in nvme_init()
800 ret = -EINVAL; in nvme_init()
804 s->page_size = 1u << (12 + NVME_CAP_MPSMIN(cap)); in nvme_init()
805 s->doorbell_scale = (4 << NVME_CAP_DSTRD(cap)) / sizeof(uint32_t); in nvme_init()
806 bs->bl.opt_mem_alignment = s->page_size; in nvme_init()
807 bs->bl.request_alignment = s->page_size; in nvme_init()
810 ver = host_pci_ldl_le_p(®s->vs); in nvme_init()
816 cc = host_pci_ldl_le_p(®s->cc); in nvme_init()
817 host_pci_stl_le_p(®s->cc, cc & 0xFE); in nvme_init()
820 while (NVME_CSTS_RDY(host_pci_ldl_le_p(®s->csts))) { in nvme_init()
825 ret = -ETIMEDOUT; in nvme_init()
830 s->bar0_wo_map = qemu_vfio_pci_map_bar(s->vfio, 0, 0, in nvme_init()
833 s->doorbells = (void *)((uintptr_t)s->bar0_wo_map + sizeof(NvmeBar)); in nvme_init()
834 if (!s->doorbells) { in nvme_init()
835 ret = -EINVAL; in nvme_init()
840 s->queues = g_new(NVMeQueuePair *, 1); in nvme_init()
843 ret = -EINVAL; in nvme_init()
846 s->queues[INDEX_ADMIN] = q; in nvme_init()
847 s->queue_count = 1; in nvme_init()
848 QEMU_BUILD_BUG_ON((NVME_QUEUE_SIZE - 1) & 0xF000); in nvme_init()
849 host_pci_stl_le_p(®s->aqa, in nvme_init()
850 ((NVME_QUEUE_SIZE - 1) << AQA_ACQS_SHIFT) | in nvme_init()
851 ((NVME_QUEUE_SIZE - 1) << AQA_ASQS_SHIFT)); in nvme_init()
852 host_pci_stq_le_p(®s->asq, q->sq.iova); in nvme_init()
853 host_pci_stq_le_p(®s->acq, q->cq.iova); in nvme_init()
856 host_pci_stl_le_p(®s->cc, in nvme_init()
863 while (!NVME_CSTS_RDY(host_pci_ldl_le_p(®s->csts))) { in nvme_init()
868 ret = -ETIMEDOUT; in nvme_init()
873 ret = qemu_vfio_pci_init_irq(s->vfio, s->irq_notifier, in nvme_init()
879 &s->irq_notifier[MSIX_SHARED_IRQ_IDX], in nvme_init()
884 ret = -EIO; in nvme_init()
890 ret = -EIO; in nvme_init()
894 qemu_vfio_pci_unmap_bar(s->vfio, 0, (void *)regs, 0, sizeof(NvmeBar)); in nvme_init()
923 device = g_strndup(tmp, slash - tmp); in nvme_parse_filename()
941 BDRVNVMeState *s = bs->opaque; in nvme_enable_disable_write_cache()
944 .nsid = cpu_to_le32(s->nsid), in nvme_enable_disable_write_cache()
958 BDRVNVMeState *s = bs->opaque; in nvme_close()
960 for (unsigned i = 0; i < s->queue_count; ++i) { in nvme_close()
961 nvme_free_queue_pair(s->queues[i]); in nvme_close()
963 g_free(s->queues); in nvme_close()
965 &s->irq_notifier[MSIX_SHARED_IRQ_IDX], in nvme_close()
967 event_notifier_cleanup(&s->irq_notifier[MSIX_SHARED_IRQ_IDX]); in nvme_close()
968 qemu_vfio_pci_unmap_bar(s->vfio, 0, s->bar0_wo_map, in nvme_close()
970 qemu_vfio_close(s->vfio); in nvme_close()
972 g_free(s->device); in nvme_close()
982 BDRVNVMeState *s = bs->opaque; in nvme_open()
984 bs->supported_write_flags = BDRV_REQ_FUA; in nvme_open()
992 return -EINVAL; in nvme_open()
1002 if (!s->write_cache_supported) { in nvme_open()
1005 ret = -EINVAL; in nvme_open()
1022 BDRVNVMeState *s = bs->opaque; in nvme_co_getlength()
1023 return s->nsze << s->blkshift; in nvme_co_getlength()
1028 BDRVNVMeState *s = bs->opaque; in nvme_get_blocksize()
1029 assert(s->blkshift >= BDRV_SECTOR_BITS && s->blkshift <= 12); in nvme_get_blocksize()
1030 return UINT32_C(1) << s->blkshift; in nvme_get_blocksize()
1036 bsz->phys = blocksize; in nvme_probe_blocksizes()
1037 bsz->log = blocksize; in nvme_probe_blocksizes()
1041 /* Called with s->dma_map_lock */
1046 BDRVNVMeState *s = bs->opaque; in nvme_cmd_unmap_qiov()
1048 s->dma_map_count -= qiov->size; in nvme_cmd_unmap_qiov()
1049 if (!s->dma_map_count && !qemu_co_queue_empty(&s->dma_flush_queue)) { in nvme_cmd_unmap_qiov()
1050 r = qemu_vfio_dma_reset_temporary(s->vfio); in nvme_cmd_unmap_qiov()
1052 qemu_co_queue_restart_all(&s->dma_flush_queue); in nvme_cmd_unmap_qiov()
1058 /* Called with s->dma_map_lock */
1062 BDRVNVMeState *s = bs->opaque; in nvme_cmd_map_qiov()
1063 uint64_t *pagelist = req->prp_list_page; in nvme_cmd_map_qiov()
1068 assert(qiov->size); in nvme_cmd_map_qiov()
1069 assert(QEMU_IS_ALIGNED(qiov->size, s->page_size)); in nvme_cmd_map_qiov()
1070 assert(qiov->size / s->page_size <= s->page_size / sizeof(uint64_t)); in nvme_cmd_map_qiov()
1071 for (i = 0; i < qiov->niov; ++i) { in nvme_cmd_map_qiov()
1074 size_t len = QEMU_ALIGN_UP(qiov->iov[i].iov_len, in nvme_cmd_map_qiov()
1077 r = qemu_vfio_dma_map(s->vfio, in nvme_cmd_map_qiov()
1078 qiov->iov[i].iov_base, in nvme_cmd_map_qiov()
1080 if (r == -ENOSPC) { in nvme_cmd_map_qiov()
1082 * In addition to the -ENOMEM error, the VFIO_IOMMU_MAP_DMA in nvme_cmd_map_qiov()
1083 * ioctl returns -ENOSPC to signal the user exhausted the DMA in nvme_cmd_map_qiov()
1086 * April 2019, see CVE-2019-3882). in nvme_cmd_map_qiov()
1089 * for the -ENOMEM error, so we directly replace -ENOSPC by in nvme_cmd_map_qiov()
1090 * -ENOMEM. Beside, -ENOSPC has a specific meaning for blockdev in nvme_cmd_map_qiov()
1096 r = -ENOMEM; in nvme_cmd_map_qiov()
1098 if (r == -ENOMEM && retry) { in nvme_cmd_map_qiov()
1105 if (s->dma_map_count) { in nvme_cmd_map_qiov()
1107 qemu_co_queue_wait(&s->dma_flush_queue, &s->dma_map_lock); in nvme_cmd_map_qiov()
1109 r = qemu_vfio_dma_reset_temporary(s->vfio); in nvme_cmd_map_qiov()
1122 for (j = 0; j < qiov->iov[i].iov_len / s->page_size; j++) { in nvme_cmd_map_qiov()
1123 pagelist[entries++] = cpu_to_le64(iova + j * s->page_size); in nvme_cmd_map_qiov()
1125 trace_nvme_cmd_map_qiov_iov(s, i, qiov->iov[i].iov_base, in nvme_cmd_map_qiov()
1126 qiov->iov[i].iov_len / s->page_size); in nvme_cmd_map_qiov()
1129 s->dma_map_count += qiov->size; in nvme_cmd_map_qiov()
1131 assert(entries <= s->page_size / sizeof(uint64_t)); in nvme_cmd_map_qiov()
1136 cmd->dptr.prp1 = pagelist[0]; in nvme_cmd_map_qiov()
1137 cmd->dptr.prp2 = 0; in nvme_cmd_map_qiov()
1140 cmd->dptr.prp1 = pagelist[0]; in nvme_cmd_map_qiov()
1141 cmd->dptr.prp2 = pagelist[1]; in nvme_cmd_map_qiov()
1144 cmd->dptr.prp1 = pagelist[0]; in nvme_cmd_map_qiov()
1145 cmd->dptr.prp2 = cpu_to_le64(req->prp_list_iova + sizeof(uint64_t)); in nvme_cmd_map_qiov()
1154 /* No need to unmap [0 - i) iovs even if we've failed, since we don't in nvme_cmd_map_qiov()
1155 * increment s->dma_map_count. This is okay for fixed mapping memory areas in nvme_cmd_map_qiov()
1174 qemu_coroutine_enter(data->co); in nvme_rw_cb_bh()
1180 data->ret = ret; in nvme_rw_cb()
1181 if (!data->co) { in nvme_rw_cb()
1185 replay_bh_schedule_oneshot_event(data->ctx, nvme_rw_cb_bh, data); in nvme_rw_cb()
1195 BDRVNVMeState *s = bs->opaque; in nvme_co_prw_aligned()
1196 NVMeQueuePair *ioq = s->queues[INDEX_IO(0)]; in nvme_co_prw_aligned()
1199 uint32_t cdw12 = (((bytes >> s->blkshift) - 1) & 0xFFFF) | in nvme_co_prw_aligned()
1203 .nsid = cpu_to_le32(s->nsid), in nvme_co_prw_aligned()
1204 .cdw10 = cpu_to_le32((offset >> s->blkshift) & 0xFFFFFFFF), in nvme_co_prw_aligned()
1205 .cdw11 = cpu_to_le32(((offset >> s->blkshift) >> 32) & 0xFFFFFFFF), in nvme_co_prw_aligned()
1210 .ret = -EINPROGRESS, in nvme_co_prw_aligned()
1213 trace_nvme_prw_aligned(s, is_write, offset, bytes, flags, qiov->niov); in nvme_co_prw_aligned()
1214 assert(s->queue_count > 1); in nvme_co_prw_aligned()
1218 qemu_co_mutex_lock(&s->dma_map_lock); in nvme_co_prw_aligned()
1220 qemu_co_mutex_unlock(&s->dma_map_lock); in nvme_co_prw_aligned()
1228 while (data.ret == -EINPROGRESS) { in nvme_co_prw_aligned()
1232 qemu_co_mutex_lock(&s->dma_map_lock); in nvme_co_prw_aligned()
1234 qemu_co_mutex_unlock(&s->dma_map_lock); in nvme_co_prw_aligned()
1247 BDRVNVMeState *s = bs->opaque; in nvme_qiov_aligned()
1249 for (i = 0; i < qiov->niov; ++i) { in nvme_qiov_aligned()
1250 if (!QEMU_PTR_IS_ALIGNED(qiov->iov[i].iov_base, in nvme_qiov_aligned()
1252 !QEMU_IS_ALIGNED(qiov->iov[i].iov_len, qemu_real_host_page_size())) { in nvme_qiov_aligned()
1253 trace_nvme_qiov_unaligned(qiov, i, qiov->iov[i].iov_base, in nvme_qiov_aligned()
1254 qiov->iov[i].iov_len, s->page_size); in nvme_qiov_aligned()
1266 BDRVNVMeState *s = bs->opaque; in nvme_co_prw()
1271 assert(QEMU_IS_ALIGNED(offset, s->page_size)); in nvme_co_prw()
1272 assert(QEMU_IS_ALIGNED(bytes, s->page_size)); in nvme_co_prw()
1273 assert(bytes <= s->max_transfer); in nvme_co_prw()
1275 s->stats.aligned_accesses++; in nvme_co_prw()
1278 s->stats.unaligned_accesses++; in nvme_co_prw()
1279 trace_nvme_prw_buffered(s, offset, bytes, qiov->niov, is_write); in nvme_co_prw()
1283 return -ENOMEM; in nvme_co_prw()
1316 BDRVNVMeState *s = bs->opaque; in nvme_co_flush()
1317 NVMeQueuePair *ioq = s->queues[INDEX_IO(0)]; in nvme_co_flush()
1321 .nsid = cpu_to_le32(s->nsid), in nvme_co_flush()
1325 .ret = -EINPROGRESS, in nvme_co_flush()
1328 assert(s->queue_count > 1); in nvme_co_flush()
1334 if (data.ret == -EINPROGRESS) { in nvme_co_flush()
1347 BDRVNVMeState *s = bs->opaque; in nvme_co_pwrite_zeroes()
1348 NVMeQueuePair *ioq = s->queues[INDEX_IO(0)]; in nvme_co_pwrite_zeroes()
1352 if (!s->supports_write_zeroes) { in nvme_co_pwrite_zeroes()
1353 return -ENOTSUP; in nvme_co_pwrite_zeroes()
1360 cdw12 = ((bytes >> s->blkshift) - 1) & 0xFFFF; in nvme_co_pwrite_zeroes()
1365 assert(((cdw12 + 1) << s->blkshift) == bytes); in nvme_co_pwrite_zeroes()
1369 .nsid = cpu_to_le32(s->nsid), in nvme_co_pwrite_zeroes()
1370 .cdw10 = cpu_to_le32((offset >> s->blkshift) & 0xFFFFFFFF), in nvme_co_pwrite_zeroes()
1371 .cdw11 = cpu_to_le32(((offset >> s->blkshift) >> 32) & 0xFFFFFFFF), in nvme_co_pwrite_zeroes()
1376 .ret = -EINPROGRESS, in nvme_co_pwrite_zeroes()
1390 assert(s->queue_count > 1); in nvme_co_pwrite_zeroes()
1397 while (data.ret == -EINPROGRESS) { in nvme_co_pwrite_zeroes()
1410 BDRVNVMeState *s = bs->opaque; in nvme_co_pdiscard()
1411 NVMeQueuePair *ioq = s->queues[INDEX_IO(0)]; in nvme_co_pdiscard()
1419 .nsid = cpu_to_le32(s->nsid), in nvme_co_pdiscard()
1420 .cdw10 = cpu_to_le32(0), /*number of ranges - 0 based*/ in nvme_co_pdiscard()
1426 .ret = -EINPROGRESS, in nvme_co_pdiscard()
1429 if (!s->supports_discard) { in nvme_co_pdiscard()
1430 return -ENOTSUP; in nvme_co_pdiscard()
1433 assert(s->queue_count > 1); in nvme_co_pdiscard()
1439 assert(QEMU_IS_ALIGNED(bytes, 1UL << s->blkshift)); in nvme_co_pdiscard()
1440 assert(QEMU_IS_ALIGNED(offset, 1UL << s->blkshift)); in nvme_co_pdiscard()
1441 assert((bytes >> s->blkshift) <= UINT32_MAX); in nvme_co_pdiscard()
1443 buf = qemu_try_memalign(s->page_size, s->page_size); in nvme_co_pdiscard()
1445 return -ENOMEM; in nvme_co_pdiscard()
1447 memset(buf, 0, s->page_size); in nvme_co_pdiscard()
1448 buf->nlb = cpu_to_le32(bytes >> s->blkshift); in nvme_co_pdiscard()
1449 buf->slba = cpu_to_le64(offset >> s->blkshift); in nvme_co_pdiscard()
1450 buf->cattr = 0; in nvme_co_pdiscard()
1458 qemu_co_mutex_lock(&s->dma_map_lock); in nvme_co_pdiscard()
1460 qemu_co_mutex_unlock(&s->dma_map_lock); in nvme_co_pdiscard()
1472 while (data.ret == -EINPROGRESS) { in nvme_co_pdiscard()
1476 qemu_co_mutex_lock(&s->dma_map_lock); in nvme_co_pdiscard()
1478 qemu_co_mutex_unlock(&s->dma_map_lock); in nvme_co_pdiscard()
1501 return -ENOTSUP; in nvme_co_truncate()
1507 return -ENOTSUP; in nvme_co_truncate()
1510 return -EINVAL; in nvme_co_truncate()
1524 BDRVNVMeState *s = bs->opaque; in nvme_refresh_filename()
1526 snprintf(bs->exact_filename, sizeof(bs->exact_filename), "nvme://%s/%i", in nvme_refresh_filename()
1527 s->device, s->nsid); in nvme_refresh_filename()
1532 BDRVNVMeState *s = bs->opaque; in nvme_refresh_limits()
1534 bs->bl.opt_mem_alignment = s->page_size; in nvme_refresh_limits()
1535 bs->bl.request_alignment = s->page_size; in nvme_refresh_limits()
1536 bs->bl.max_transfer = s->max_transfer; in nvme_refresh_limits()
1542 bs->bl.max_pwrite_zeroes = 1ULL << (s->blkshift + 16); in nvme_refresh_limits()
1543 bs->bl.pwrite_zeroes_alignment = MAX(bs->bl.request_alignment, in nvme_refresh_limits()
1544 1UL << s->blkshift); in nvme_refresh_limits()
1546 bs->bl.max_pdiscard = (uint64_t)UINT32_MAX << s->blkshift; in nvme_refresh_limits()
1547 bs->bl.pdiscard_alignment = MAX(bs->bl.request_alignment, in nvme_refresh_limits()
1548 1UL << s->blkshift); in nvme_refresh_limits()
1553 BDRVNVMeState *s = bs->opaque; in nvme_detach_aio_context()
1555 for (unsigned i = 0; i < s->queue_count; i++) { in nvme_detach_aio_context()
1556 NVMeQueuePair *q = s->queues[i]; in nvme_detach_aio_context()
1558 qemu_bh_delete(q->completion_bh); in nvme_detach_aio_context()
1559 q->completion_bh = NULL; in nvme_detach_aio_context()
1563 &s->irq_notifier[MSIX_SHARED_IRQ_IDX], in nvme_detach_aio_context()
1570 BDRVNVMeState *s = bs->opaque; in nvme_attach_aio_context()
1572 s->aio_context = new_context; in nvme_attach_aio_context()
1573 aio_set_event_notifier(new_context, &s->irq_notifier[MSIX_SHARED_IRQ_IDX], in nvme_attach_aio_context()
1577 for (unsigned i = 0; i < s->queue_count; i++) { in nvme_attach_aio_context()
1578 NVMeQueuePair *q = s->queues[i]; in nvme_attach_aio_context()
1580 q->completion_bh = in nvme_attach_aio_context()
1589 BDRVNVMeState *s = bs->opaque; in nvme_register_buf()
1596 ret = qemu_vfio_dma_map(s->vfio, host, size, false, NULL, errp); in nvme_register_buf()
1602 BDRVNVMeState *s = bs->opaque; in nvme_unregister_buf()
1604 qemu_vfio_dma_unmap(s->vfio, host); in nvme_unregister_buf()
1610 BDRVNVMeState *s = bs->opaque; in nvme_get_specific_stats()
1612 stats->driver = BLOCKDEV_DRIVER_NVME; in nvme_get_specific_stats()
1613 stats->u.nvme = (BlockStatsSpecificNvme) { in nvme_get_specific_stats()
1614 .completion_errors = s->stats.completion_errors, in nvme_get_specific_stats()
1615 .aligned_accesses = s->stats.aligned_accesses, in nvme_get_specific_stats()
1616 .unaligned_accesses = s->stats.unaligned_accesses, in nvme_get_specific_stats()