341     struct gdma_queue *queue)  in mana_gd_create_hw_eq()  argument
347 	if (queue->type != GDMA_EQ)  in mana_gd_create_hw_eq()
353 	req.hdr.dev_id = queue->gdma_dev->dev_id;  in mana_gd_create_hw_eq()
354 	req.type = queue->type;  in mana_gd_create_hw_eq()
355 	req.pdid = queue->gdma_dev->pdid;  in mana_gd_create_hw_eq()
356 	req.doolbell_id = queue->gdma_dev->doorbell;  in mana_gd_create_hw_eq()
357 	req.gdma_region = queue->mem_info.dma_region_handle;  in mana_gd_create_hw_eq()
358 	req.queue_size = queue->queue_size;  in mana_gd_create_hw_eq()
359 	req.log2_throttle_limit = queue->eq.log2_throttle_limit;  in mana_gd_create_hw_eq()
360 	req.eq_pci_msix_index = queue->eq.msix_index;  in mana_gd_create_hw_eq()
370 	queue->id = resp.queue_index;  in mana_gd_create_hw_eq()
371 	queue->eq.disable_needed = true;  in mana_gd_create_hw_eq()
372 	queue->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION;  in mana_gd_create_hw_eq()
377 int mana_gd_disable_queue(struct gdma_queue *queue)  in mana_gd_disable_queue()  argument
379 	struct gdma_context *gc = queue->gdma_dev->gdma_context;  in mana_gd_disable_queue()
384 	if (queue->type != GDMA_EQ)  in mana_gd_disable_queue()
386 		    queue->type);  in mana_gd_disable_queue()
391 	req.hdr.dev_id = queue->gdma_dev->dev_id;  in mana_gd_disable_queue()
392 	req.type = queue->type;  in mana_gd_disable_queue()
393 	req.queue_index =  queue->id;  in mana_gd_disable_queue()
471 mana_gd_wq_ring_doorbell(struct gdma_context *gc, struct gdma_queue *queue)  in mana_gd_wq_ring_doorbell()  argument
473 	mana_gd_ring_doorbell(gc, queue->gdma_dev->doorbell, queue->type,  in mana_gd_wq_ring_doorbell()
474 	    queue->id, queue->head * GDMA_WQE_BU_SIZE, 0);  in mana_gd_wq_ring_doorbell()
629 mana_gd_register_irq(struct gdma_queue *queue,  in mana_gd_register_irq()  argument
632 	struct gdma_dev *gd = queue->gdma_dev;  in mana_gd_register_irq()
649 		queue->eq.msix_index = msi_index;  in mana_gd_register_irq()
673 	gic->arg = queue;  in mana_gd_register_irq()
684 mana_gd_deregiser_irq(struct gdma_queue *queue)  in mana_gd_deregiser_irq()  argument
686 	struct gdma_dev *gd = queue->gdma_dev;  in mana_gd_deregiser_irq()
696 	msix_index = queue->eq.msix_index;  in mana_gd_deregiser_irq()
708 	queue->eq.msix_index = INVALID_PCI_MSIX_INDEX;  in mana_gd_deregiser_irq()
769     struct gdma_queue *queue)  in mana_gd_destroy_eq()  argument
774 		err = mana_gd_test_eq(gc, queue);  in mana_gd_destroy_eq()
780 	mana_gd_deregiser_irq(queue);  in mana_gd_destroy_eq()
782 	if (queue->eq.disable_needed)  in mana_gd_destroy_eq()
783 		mana_gd_disable_queue(queue);  in mana_gd_destroy_eq()
788     bool create_hwq, struct gdma_queue *queue)  in mana_gd_create_eq()  argument
795 	queue->eq.msix_index = INVALID_PCI_MSIX_INDEX;  in mana_gd_create_eq()
797 	log2_num_entries = ilog2(queue->queue_size / GDMA_EQE_SIZE);  in mana_gd_create_eq()
806 	err = mana_gd_register_irq(queue, spec);  in mana_gd_create_eq()
812 	queue->eq.callback = spec->eq.callback;  in mana_gd_create_eq()
813 	queue->eq.context = spec->eq.context;  in mana_gd_create_eq()
814 	queue->head |= INITIALIZED_OWNER_BIT(log2_num_entries);  in mana_gd_create_eq()
815 	queue->eq.log2_throttle_limit = spec->eq.log2_throttle_limit ?: 1;  in mana_gd_create_eq()
818 		err = mana_gd_create_hw_eq(gc, queue);  in mana_gd_create_eq()
822 		err = mana_gd_test_eq(gc, queue);  in mana_gd_create_eq()
830 	mana_gd_destroy_eq(gc, false, queue);  in mana_gd_create_eq()
836     struct gdma_queue *queue)  in mana_gd_create_cq()  argument
840 	queue->head |= INITIALIZED_OWNER_BIT(log2_num_entries);  in mana_gd_create_cq()
841 	queue->cq.parent = spec->cq.parent_eq;  in mana_gd_create_cq()
842 	queue->cq.context = spec->cq.context;  in mana_gd_create_cq()
843 	queue->cq.callback = spec->cq.callback;  in mana_gd_create_cq()
848     struct gdma_queue *queue)  in mana_gd_destroy_cq()  argument
850 	uint32_t id = queue->id;  in mana_gd_destroy_cq()
867 	struct gdma_queue *queue;  in mana_gd_create_hwc_queue()  local
870 	queue = malloc(sizeof(*queue), M_DEVBUF, M_WAITOK | M_ZERO);  in mana_gd_create_hwc_queue()
871 	gmi = &queue->mem_info;  in mana_gd_create_hwc_queue()
876 	queue->head = 0;  in mana_gd_create_hwc_queue()
877 	queue->tail = 0;  in mana_gd_create_hwc_queue()
878 	queue->queue_mem_ptr = gmi->virt_addr;  in mana_gd_create_hwc_queue()
879 	queue->queue_size = spec->queue_size;  in mana_gd_create_hwc_queue()
880 	queue->monitor_avl_buf = spec->monitor_avl_buf;  in mana_gd_create_hwc_queue()
881 	queue->type = spec->type;  in mana_gd_create_hwc_queue()
882 	queue->gdma_dev = gd;  in mana_gd_create_hwc_queue()
885 		err = mana_gd_create_eq(gd, spec, false, queue);  in mana_gd_create_hwc_queue()
887 		mana_gd_create_cq(spec, queue);  in mana_gd_create_hwc_queue()
892 	*queue_ptr = queue;  in mana_gd_create_hwc_queue()
897 	free(queue, M_DEVBUF);  in mana_gd_create_hwc_queue()
998 	struct gdma_queue *queue;  in mana_gd_create_mana_eq()  local
1004 	queue = malloc(sizeof(*queue),  M_DEVBUF, M_WAITOK | M_ZERO);  in mana_gd_create_mana_eq()
1005 	gmi = &queue->mem_info;  in mana_gd_create_mana_eq()
1014 	queue->head = 0;  in mana_gd_create_mana_eq()
1015 	queue->tail = 0;  in mana_gd_create_mana_eq()
1016 	queue->queue_mem_ptr = gmi->virt_addr;  in mana_gd_create_mana_eq()
1017 	queue->queue_size = spec->queue_size;  in mana_gd_create_mana_eq()
1018 	queue->monitor_avl_buf = spec->monitor_avl_buf;  in mana_gd_create_mana_eq()
1019 	queue->type = spec->type;  in mana_gd_create_mana_eq()
1020 	queue->gdma_dev = gd;  in mana_gd_create_mana_eq()
1022 	err = mana_gd_create_eq(gd, spec, true, queue);  in mana_gd_create_mana_eq()
1026 	*queue_ptr = queue;  in mana_gd_create_mana_eq()
1032 	free(queue, M_DEVBUF);  in mana_gd_create_mana_eq()
1042 	struct gdma_queue *queue;  in mana_gd_create_mana_wq_cq()  local
1049 	queue = malloc(sizeof(*queue), M_DEVBUF, M_WAITOK | M_ZERO);  in mana_gd_create_mana_wq_cq()
1050 	gmi = &queue->mem_info;  in mana_gd_create_mana_wq_cq()
1059 	queue->head = 0;  in mana_gd_create_mana_wq_cq()
1060 	queue->tail = 0;  in mana_gd_create_mana_wq_cq()
1061 	queue->queue_mem_ptr = gmi->virt_addr;  in mana_gd_create_mana_wq_cq()
1062 	queue->queue_size = spec->queue_size;  in mana_gd_create_mana_wq_cq()
1063 	queue->monitor_avl_buf = spec->monitor_avl_buf;  in mana_gd_create_mana_wq_cq()
1064 	queue->type = spec->type;  in mana_gd_create_mana_wq_cq()
1065 	queue->gdma_dev = gd;  in mana_gd_create_mana_wq_cq()
1068 		mana_gd_create_cq(spec, queue);  in mana_gd_create_mana_wq_cq()
1070 	*queue_ptr = queue;  in mana_gd_create_mana_wq_cq()
1076 	free(queue, M_DEVBUF);  in mana_gd_create_mana_wq_cq()
1081 mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue)  in mana_gd_destroy_queue()  argument
1083 	struct gdma_mem_info *gmi = &queue->mem_info;  in mana_gd_destroy_queue()
1085 	switch (queue->type) {  in mana_gd_destroy_queue()
1087 		mana_gd_destroy_eq(gc, queue->eq.disable_needed, queue);  in mana_gd_destroy_queue()
1091 		mana_gd_destroy_cq(gc, queue);  in mana_gd_destroy_queue()
1103 		    queue->type);  in mana_gd_destroy_queue()
1109 	free(queue, M_DEVBUF);  in mana_gd_destroy_queue()
1377 mana_gd_post_and_ring(struct gdma_queue *queue,  in mana_gd_post_and_ring()  argument
1381 	struct gdma_context *gc = queue->gdma_dev->gdma_context;  in mana_gd_post_and_ring()
1384 	err = mana_gd_post_work_request(queue, wqe_req, wqe_info);  in mana_gd_post_and_ring()
1388 	mana_gd_wq_ring_doorbell(gc, queue);  in mana_gd_post_and_ring()