19 static void ufshci_req_queue_submit_tracker(struct ufshci_req_queue *req_queue,
55 	struct ufshci_req_queue *req_queue;  in ufshci_utmr_req_queue_construct()  local
62 	req_queue = &ctrlr->task_mgmt_req_queue;  in ufshci_utmr_req_queue_construct()
63 	req_queue->queue_mode = UFSHCI_Q_MODE_SDB;  in ufshci_utmr_req_queue_construct()
64 	req_queue->qops = sdb_utmr_qops;  in ufshci_utmr_req_queue_construct()
66 	error = req_queue->qops.construct(ctrlr, req_queue, UFSHCI_UTRM_ENTRIES,  in ufshci_utmr_req_queue_construct()
96 	struct ufshci_req_queue *req_queue;  in ufshci_utr_req_queue_construct()  local
104 	req_queue = &ctrlr->transfer_req_queue;  in ufshci_utr_req_queue_construct()
105 	req_queue->queue_mode = UFSHCI_Q_MODE_SDB;  in ufshci_utr_req_queue_construct()
106 	req_queue->qops = sdb_utr_qops;  in ufshci_utr_req_queue_construct()
108 	error = req_queue->qops.construct(ctrlr, req_queue, UFSHCI_UTR_ENTRIES,  in ufshci_utr_req_queue_construct()
136 ufshci_req_queue_response_is_error(struct ufshci_req_queue *req_queue,  in ufshci_req_queue_response_is_error()  argument
143 		ufshci_printf(req_queue->ctrlr, "Invalid OCS = 0x%x\n", ocs);  in ufshci_req_queue_response_is_error()
149 		ufshci_printf(req_queue->ctrlr,  in ufshci_req_queue_response_is_error()
178 ufshci_req_queue_manual_complete_request(struct ufshci_req_queue *req_queue,  in ufshci_req_queue_manual_complete_request()  argument
186 	error = ufshci_req_queue_response_is_error(req_queue, ocs,  in ufshci_req_queue_manual_complete_request()
190 		ufshci_printf(req_queue->ctrlr,  in ufshci_req_queue_manual_complete_request()
202     struct ufshci_req_queue *req_queue)  in ufshci_req_queue_fail()  argument
204 	struct ufshci_hw_queue *hwq = req_queue->qops.get_hw_queue(req_queue);  in ufshci_req_queue_fail()
213 	for (i = 0; i < req_queue->num_trackers; i++) {  in ufshci_req_queue_fail()
218 			ufshci_req_queue_manual_complete_request(req_queue,  in ufshci_req_queue_fail()
241 	struct ufshci_req_queue *req_queue = tr->req_queue;  in ufshci_req_queue_complete_tracker()  local
253 	if (req_queue->is_task_mgmt) {  in ufshci_req_queue_complete_tracker()
259 		bus_dmamap_sync(req_queue->dma_tag_ucd, req_queue->ucdmem_map,  in ufshci_req_queue_complete_tracker()
268 	error = ufshci_req_queue_response_is_error(req_queue, ocs,  in ufshci_req_queue_complete_tracker()
274 	    req->retries < req_queue->ctrlr->retry_count;  in ufshci_req_queue_complete_tracker()
277 	if (error && req->retries >= req_queue->ctrlr->retry_count && retriable)  in ufshci_req_queue_complete_tracker()
287 			bus_dmamap_sync(req_queue->dma_tag_payload,  in ufshci_req_queue_complete_tracker()
299 	req_queue->qops.clear_cpl_ntf(req_queue->ctrlr, tr);  in ufshci_req_queue_complete_tracker()
303 		ufshci_req_queue_submit_tracker(req_queue, tr,  in ufshci_req_queue_complete_tracker()
307 			bus_dmamap_unload(req_queue->dma_tag_payload,  in ufshci_req_queue_complete_tracker()
324 ufshci_req_queue_process_completions(struct ufshci_req_queue *req_queue)  in ufshci_req_queue_process_completions()  argument
329 	hwq = req_queue->qops.get_hw_queue(req_queue);  in ufshci_req_queue_process_completions()
332 	done = req_queue->qops.process_cpl(req_queue);  in ufshci_req_queue_process_completions()
351 		ufshci_printf(tr->req_queue->ctrlr,  in ufshci_payload_map()
370 	bus_dmamap_sync(tr->req_queue->dma_tag_payload, tr->payload_dma_map,  in ufshci_payload_map()
386 	error = bus_dmamap_load_mem(tr->req_queue->dma_tag_payload,  in ufshci_req_queue_prepare_prdt()
397 		ufshci_printf(tr->req_queue->ctrlr,  in ufshci_req_queue_prepare_prdt()
585 		hwq->req_queue->qops.process_cpl(hwq->req_queue);  in ufshci_req_queue_timeout()
620 			    !hwq->req_queue->is_task_mgmt &&  in ufshci_req_queue_timeout()
676 ufshci_req_queue_submit_tracker(struct ufshci_req_queue *req_queue,  in ufshci_req_queue_submit_tracker()  argument
679 	struct ufshci_controller *ctrlr = req_queue->ctrlr;  in ufshci_req_queue_submit_tracker()
687 	hwq = req_queue->qops.get_hw_queue(req_queue);  in ufshci_req_queue_submit_tracker()
706 	if (req_queue->is_task_mgmt) {  in ufshci_req_queue_submit_tracker()
729 		bus_dmamap_sync(req_queue->dma_tag_ucd, req_queue->ucdmem_map,  in ufshci_req_queue_submit_tracker()
739 	req_queue->qops.ring_doorbell(ctrlr, tr);  in ufshci_req_queue_submit_tracker()
743 _ufshci_req_queue_submit_request(struct ufshci_req_queue *req_queue,  in _ufshci_req_queue_submit_request()  argument
749 	mtx_assert(&req_queue->qops.get_hw_queue(req_queue)->qlock, MA_OWNED);  in _ufshci_req_queue_submit_request()
751 	error = req_queue->qops.reserve_slot(req_queue, &tr);  in _ufshci_req_queue_submit_request()
753 		ufshci_printf(req_queue->ctrlr, "Failed to get tracker");  in _ufshci_req_queue_submit_request()
773 	ufshci_req_queue_submit_tracker(req_queue, tr, req->data_direction);  in _ufshci_req_queue_submit_request()
779 ufshci_req_queue_submit_request(struct ufshci_req_queue *req_queue,  in ufshci_req_queue_submit_request()  argument
787 	hwq = req_queue->qops.get_hw_queue(req_queue);  in ufshci_req_queue_submit_request()
791 	error = _ufshci_req_queue_submit_request(req_queue, req);  in ufshci_req_queue_submit_request()