99     struct mpr_command *cm);
102 static void mpr_config_complete(struct mpr_softc *sc, struct mpr_command *cm);
105 static void mpr_enqueue_request(struct mpr_softc *sc, struct mpr_command *cm);
757 	struct mpr_command *cm;  in mpr_iocfacts_free()  local
815 			cm = &sc->commands[i];  in mpr_iocfacts_free()
816 			bus_dmamap_destroy(sc->buffer_dmat, cm->cm_dmamap);  in mpr_iocfacts_free()
1126 mpr_enqueue_request(struct mpr_softc *sc, struct mpr_command *cm)  in mpr_enqueue_request()  argument
1132 	    cm->cm_desc.Default.SMID, cm, cm->cm_ccb);  in mpr_enqueue_request()
1141 	KASSERT(cm->cm_state == MPR_CM_STATE_BUSY,  in mpr_enqueue_request()
1142 	    ("command not busy, state = %u\n", cm->cm_state));  in mpr_enqueue_request()
1143 	cm->cm_state = MPR_CM_STATE_INQUEUE;  in mpr_enqueue_request()
1146 		rd.u.low = cm->cm_desc.Words.Low;  in mpr_enqueue_request()
1150 		rd.u.low = htole32(cm->cm_desc.Words.Low);  in mpr_enqueue_request()
1151 		rd.u.high = htole32(cm->cm_desc.Words.High);  in mpr_enqueue_request()
1483 	struct mpr_command *cm;  in mpr_alloc_requests()  local
1582 		cm = &sc->commands[i];  in mpr_alloc_requests()
1583 		cm->cm_req = sc->req_frames + i * sc->reqframesz;  in mpr_alloc_requests()
1584 		cm->cm_req_busaddr = sc->req_busaddr + i * sc->reqframesz;  in mpr_alloc_requests()
1585 		cm->cm_sense = &sc->sense_frames[i];  in mpr_alloc_requests()
1586 		cm->cm_sense_busaddr = sc->sense_busaddr + i * MPR_SENSE_LEN;  in mpr_alloc_requests()
1587 		cm->cm_desc.Default.SMID = htole16(i);  in mpr_alloc_requests()
1588 		cm->cm_sc = sc;  in mpr_alloc_requests()
1589 		cm->cm_state = MPR_CM_STATE_BUSY;  in mpr_alloc_requests()
1590 		TAILQ_INIT(&cm->cm_chain_list);  in mpr_alloc_requests()
1591 		TAILQ_INIT(&cm->cm_prp_page_list);  in mpr_alloc_requests()
1592 		callout_init_mtx(&cm->cm_callout, &sc->mpr_mtx, 0);  in mpr_alloc_requests()
1595 		if (bus_dmamap_create(sc->buffer_dmat, 0, &cm->cm_dmamap)  in mpr_alloc_requests()
1598 				mpr_free_high_priority_command(sc, cm);  in mpr_alloc_requests()
1600 				mpr_free_command(sc, cm);  in mpr_alloc_requests()
2103 	struct mpr_command *cm;  in mpr_dump_reqs()  local
2131 		cm = &sc->commands[i];  in mpr_dump_reqs()
2132 		if ((sc->dump_reqs_alltypes == 0) && (cm->cm_state != state))  in mpr_dump_reqs()
2135 		hdr.state = cm->cm_state;  in mpr_dump_reqs()
2137 		hdr.deschi = cm->cm_desc.Words.High;  in mpr_dump_reqs()
2138 		hdr.desclo = cm->cm_desc.Words.Low;  in mpr_dump_reqs()
2139 		TAILQ_FOREACH_SAFE(chain, &cm->cm_chain_list, chain_link,  in mpr_dump_reqs()
2143 		sbuf_bcat(sb, cm->cm_req, 128);  in mpr_dump_reqs()
2144 		TAILQ_FOREACH_SAFE(chain, &cm->cm_chain_list, chain_link,  in mpr_dump_reqs()
2388 mpr_complete_command(struct mpr_softc *sc, struct mpr_command *cm)  in mpr_complete_command()  argument
2392 	if (cm == NULL) {  in mpr_complete_command()
2397 	KASSERT(cm->cm_state == MPR_CM_STATE_INQUEUE,  in mpr_complete_command()
2398 	    ("command not inqueue, state = %u\n", cm->cm_state));  in mpr_complete_command()
2399 	cm->cm_state = MPR_CM_STATE_BUSY;  in mpr_complete_command()
2400 	if (cm->cm_flags & MPR_CM_FLAGS_POLLED)  in mpr_complete_command()
2401 		cm->cm_flags |= MPR_CM_FLAGS_COMPLETE;  in mpr_complete_command()
2403 	if (cm->cm_complete != NULL) {  in mpr_complete_command()
2406 		    __func__, cm, cm->cm_complete, cm->cm_complete_data,  in mpr_complete_command()
2407 		    cm->cm_reply);  in mpr_complete_command()
2408 		cm->cm_complete(sc, cm);  in mpr_complete_command()
2411 	if (cm->cm_flags & MPR_CM_FLAGS_WAKEUP) {  in mpr_complete_command()
2412 		mpr_dprint(sc, MPR_TRACE, "waking up %p\n", cm);  in mpr_complete_command()
2413 		wakeup(cm);  in mpr_complete_command()
2532 	struct mpr_command *cm = NULL;  in mpr_intr_locked()  local
2544 		cm = NULL;  in mpr_intr_locked()
2577 			cm = &sc->commands[le16toh(desc->SCSIIOSuccess.SMID)];  in mpr_intr_locked()
2578 			cm->cm_reply = NULL;  in mpr_intr_locked()
2645 				cm = &sc->commands[  in mpr_intr_locked()
2647 				if (cm->cm_state == MPR_CM_STATE_INQUEUE) {  in mpr_intr_locked()
2648 					cm->cm_reply = reply;  in mpr_intr_locked()
2649 					cm->cm_reply_data =  in mpr_intr_locked()
2656 					    cm->cm_state, cm);  in mpr_intr_locked()
2668 			cm = NULL;  in mpr_intr_locked()
2672 		if (cm != NULL) {  in mpr_intr_locked()
2674 			if (cm->cm_reply)  in mpr_intr_locked()
2675 				mpr_display_reply_info(sc,cm->cm_reply);  in mpr_intr_locked()
2676 			mpr_complete_command(sc, cm);  in mpr_intr_locked()
2718 mpr_reregister_events_complete(struct mpr_softc *sc, struct mpr_command *cm)  in mpr_reregister_events_complete()  argument
2722 	if (cm->cm_reply)  in mpr_reregister_events_complete()
2724 			(MPI2_EVENT_NOTIFICATION_REPLY *)cm->cm_reply);  in mpr_reregister_events_complete()
2726 	mpr_free_command(sc, cm);  in mpr_reregister_events_complete()
2761 	struct mpr_command *cm = NULL;  in mpr_update_events()  local
2776 	if ((cm = mpr_alloc_command(sc)) == NULL)  in mpr_update_events()
2778 	evtreq = (MPI2_EVENT_NOTIFICATION_REQUEST *)cm->cm_req;  in mpr_update_events()
2793 	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;  in mpr_update_events()
2794 	cm->cm_data = NULL;  in mpr_update_events()
2796 	error = mpr_request_polled(sc, &cm);  in mpr_update_events()
2797 	if (cm != NULL)  in mpr_update_events()
2798 		reply = (MPI2_EVENT_NOTIFICATION_REPLY *)cm->cm_reply;  in mpr_update_events()
2808 	if (cm != NULL)  in mpr_update_events()
2809 		mpr_free_command(sc, cm);  in mpr_update_events()
2817 	struct mpr_command *cm;  in mpr_reregister_events()  local
2832 	if ((cm = mpr_alloc_command(sc)) == NULL)  in mpr_reregister_events()
2834 	evtreq = (MPI2_EVENT_NOTIFICATION_REQUEST *)cm->cm_req;  in mpr_reregister_events()
2849 	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;  in mpr_reregister_events()
2850 	cm->cm_data = NULL;  in mpr_reregister_events()
2851 	cm->cm_complete = mpr_reregister_events_complete;  in mpr_reregister_events()
2853 	error = mpr_map_command(sc, cm);  in mpr_reregister_events()
2918 mpr_build_nvme_prp(struct mpr_softc *sc, struct mpr_command *cm,  in mpr_build_nvme_prp()  argument
2964 	TAILQ_INSERT_TAIL(&cm->cm_prp_page_list, prp_page_info, prp_page_link);  in mpr_build_nvme_prp()
3114 mpr_check_pcie_native_sgl(struct mpr_softc *sc, struct mpr_command *cm,  in mpr_check_pcie_native_sgl()  argument
3140 	if ((cm->cm_targ->MDTS > 0) && (buff_len > cm->cm_targ->MDTS))  in mpr_check_pcie_native_sgl()
3211 	main_chain_element = (pMpi25IeeeSgeChain64_t)cm->cm_sge;  in mpr_check_pcie_native_sgl()
3236 	TAILQ_INSERT_TAIL(&cm->cm_prp_page_list, prp_page_info, prp_page_link);  in mpr_check_pcie_native_sgl()
3278 	ptr_first_sgl = (uint32_t *)cm->cm_sge;  in mpr_check_pcie_native_sgl()
3388 mpr_add_chain(struct mpr_command *cm, int segsleft)  in mpr_add_chain()  argument
3390 	struct mpr_softc *sc = cm->cm_sc;  in mpr_add_chain()
3402 	if (cm->cm_flags & MPR_CM_FLAGS_SGE_SIMPLE) {  in mpr_add_chain()
3409 	if (cm->cm_sglsize < sgc_size)  in mpr_add_chain()
3412 	chain = mpr_alloc_chain(cm->cm_sc);  in mpr_add_chain()
3420 	TAILQ_INSERT_TAIL(&cm->cm_chain_list, chain, chain_link);  in mpr_add_chain()
3428 	if (cm->cm_sglsize < (sgc_size * segsleft)) {  in mpr_add_chain()
3439 		current_segs = (cm->cm_sglsize / sgc_size) - 1;  in mpr_add_chain()
3446 	ieee_sgc = &((MPI25_SGE_IO_UNION *)cm->cm_sge)->IeeeChain;  in mpr_add_chain()
3455 	cm->cm_sge = &((MPI25_SGE_IO_UNION *)chain->chain)->IeeeSimple;  in mpr_add_chain()
3456 	req = (MPI2_REQUEST_HEADER *)cm->cm_req;  in mpr_add_chain()
3459 	cm->cm_sglsize = sc->chain_frame_size;  in mpr_add_chain()
3470 mpr_push_sge(struct mpr_command *cm, MPI2_SGE_SIMPLE64 *sge, size_t len,  in mpr_push_sge()  argument
3481 	if (cm->cm_sglsize < (segsleft * sizeof(MPI2_SGE_SIMPLE64))) {  in mpr_push_sge()
3482 		mpr_dprint(cm->cm_sc, MPR_ERROR,  in mpr_push_sge()
3509 	if (cm->cm_out_len) {  in mpr_push_sge()
3510 		sge->FlagsLength = cm->cm_out_len |  in mpr_push_sge()
3516 		cm->cm_sglsize -= len;  in mpr_push_sge()
3520 		bcopy(sge, cm->cm_sge, len);  in mpr_push_sge()
3521 		cm->cm_sge = (MPI2_SGE_IO_UNION *)((uintptr_t)cm->cm_sge + len);  in mpr_push_sge()
3530 	if (cm->cm_flags & MPR_CM_FLAGS_DATAIN) {  in mpr_push_sge()
3542 	cm->cm_sglsize -= len;  in mpr_push_sge()
3546 	bcopy(sge, cm->cm_sge, len);  in mpr_push_sge()
3547 	cm->cm_sge = (MPI2_SGE_IO_UNION *)((uintptr_t)cm->cm_sge + len);  in mpr_push_sge()
3557 mpr_push_ieee_sge(struct mpr_command *cm, void *sgep, int segsleft)  in mpr_push_ieee_sge()  argument
3574 	if (cm->cm_sglsize < ieee_sge_size)  in mpr_push_ieee_sge()
3577 	if ((segsleft >= 2) && (cm->cm_sglsize < (ieee_sge_size * 2))) {  in mpr_push_ieee_sge()
3578 		if ((error = mpr_add_chain(cm, segsleft)) != 0)  in mpr_push_ieee_sge()
3596 		if (cm->cm_out_len) {  in mpr_push_ieee_sge()
3597 			sge->Length = cm->cm_out_len;  in mpr_push_ieee_sge()
3600 			cm->cm_sglsize -= ieee_sge_size;  in mpr_push_ieee_sge()
3604 			bcopy(sgep, cm->cm_sge, ieee_sge_size);  in mpr_push_ieee_sge()
3605 			cm->cm_sge =  in mpr_push_ieee_sge()
3606 			    (MPI25_SGE_IO_UNION *)((uintptr_t)cm->cm_sge +  in mpr_push_ieee_sge()
3617 	cm->cm_sglsize -= ieee_sge_size;  in mpr_push_ieee_sge()
3621 	bcopy(sgep, cm->cm_sge, ieee_sge_size);  in mpr_push_ieee_sge()
3622 	cm->cm_sge = (MPI25_SGE_IO_UNION *)((uintptr_t)cm->cm_sge +  in mpr_push_ieee_sge()
3631 mpr_add_dmaseg(struct mpr_command *cm, vm_paddr_t pa, size_t len, u_int flags,  in mpr_add_dmaseg()  argument
3637 	if (!(cm->cm_flags & MPR_CM_FLAGS_SGE_SIMPLE)) {  in mpr_add_dmaseg()
3643 		return (mpr_push_ieee_sge(cm, &ieee_sge, segsleft));  in mpr_add_dmaseg()
3655 		return (mpr_push_sge(cm, &sge, sizeof sge, segsleft));  in mpr_add_dmaseg()
3663 	struct mpr_command *cm;  in mpr_data_cb()  local
3666 	cm = (struct mpr_command *)arg;  in mpr_data_cb()
3667 	sc = cm->cm_sc;  in mpr_data_cb()
3673 	if ((cm->cm_max_segs != 0) && (nsegs > cm->cm_max_segs)) {  in mpr_data_cb()
3676 		    cm->cm_max_segs);  in mpr_data_cb()
3684 	if (cm->cm_flags & MPR_CM_FLAGS_SMP_PASS) {  in mpr_data_cb()
3706 	} else if (cm->cm_flags & MPR_CM_FLAGS_DATAOUT) {  in mpr_data_cb()
3713 	if (cm->cm_targ && cm->cm_targ->is_nvme &&  in mpr_data_cb()
3714 	    mpr_check_pcie_native_sgl(sc, cm, segs, nsegs) == 0) {  in mpr_data_cb()
3720 		if ((cm->cm_flags & MPR_CM_FLAGS_SMP_PASS) && (i != 0)) {  in mpr_data_cb()
3723 		error = mpr_add_dmaseg(cm, segs[i].ds_addr, segs[i].ds_len,  in mpr_data_cb()
3730 			cm->cm_flags |= MPR_CM_FLAGS_CHAIN_FAILED;  in mpr_data_cb()
3737 			cm->cm_state = MPR_CM_STATE_INQUEUE;  in mpr_data_cb()
3738 			mpr_complete_command(sc, cm);  in mpr_data_cb()
3744 	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir);  in mpr_data_cb()
3745 	mpr_enqueue_request(sc, cm);  in mpr_data_cb()
3765 mpr_map_command(struct mpr_softc *sc, struct mpr_command *cm)  in mpr_map_command()  argument
3769 	if (cm->cm_flags & MPR_CM_FLAGS_USE_UIO) {  in mpr_map_command()
3770 		error = bus_dmamap_load_uio(sc->buffer_dmat, cm->cm_dmamap,  in mpr_map_command()
3771 		    &cm->cm_uio, mpr_data_cb2, cm, 0);  in mpr_map_command()
3772 	} else if (cm->cm_flags & MPR_CM_FLAGS_USE_CCB) {  in mpr_map_command()
3773 		error = bus_dmamap_load_ccb(sc->buffer_dmat, cm->cm_dmamap,  in mpr_map_command()
3774 		    cm->cm_data, mpr_data_cb, cm, 0);  in mpr_map_command()
3775 	} else if ((cm->cm_data != NULL) && (cm->cm_length != 0)) {  in mpr_map_command()
3776 		error = bus_dmamap_load(sc->buffer_dmat, cm->cm_dmamap,  in mpr_map_command()
3777 		    cm->cm_data, cm->cm_length, mpr_data_cb, cm, 0);  in mpr_map_command()
3780 		if (cm->cm_sge != NULL)  in mpr_map_command()
3781 			mpr_add_dmaseg(cm, 0, 0, 0, 1);  in mpr_map_command()
3782 		mpr_enqueue_request(sc, cm);  in mpr_map_command()
3799 	struct mpr_command *cm = *cmp;  in mpr_wait_command()  local
3804 	cm->cm_complete = NULL;  in mpr_wait_command()
3805 	cm->cm_flags |= (MPR_CM_FLAGS_WAKEUP + MPR_CM_FLAGS_POLLED);  in mpr_wait_command()
3806 	error = mpr_map_command(sc, cm);  in mpr_wait_command()
3817 		error = msleep(cm, &sc->mpr_mtx, 0, "mprwait", timeout*hz);  in mpr_wait_command()
3827 		while ((cm->cm_flags & MPR_CM_FLAGS_COMPLETE) == 0) {  in mpr_wait_command()
3844 		if (cm->cm_timeout_handler == NULL) {  in mpr_wait_command()
3852 			cm->cm_timeout_handler(sc, cm);  in mpr_wait_command()
3874 	struct mpr_command *cm = *cmp;  in mpr_request_polled()  local
3878 	cm->cm_flags |= MPR_CM_FLAGS_POLLED;  in mpr_request_polled()
3879 	cm->cm_complete = NULL;  in mpr_request_polled()
3880 	mpr_map_command(sc, cm);  in mpr_request_polled()
3883 	while ((cm->cm_flags & MPR_CM_FLAGS_COMPLETE) == 0) {  in mpr_request_polled()
3903 	cm->cm_state = MPR_CM_STATE_BUSY;  in mpr_request_polled()
3929 	struct mpr_command *cm;  in mpr_read_config_page()  local
3936 	cm = mpr_alloc_command(sc);  in mpr_read_config_page()
3937 	if (cm == NULL) {  in mpr_read_config_page()
3941 	req = (MPI2_CONFIG_REQUEST *)cm->cm_req;  in mpr_read_config_page()
3967 	cm->cm_data = params->buffer;  in mpr_read_config_page()
3968 	cm->cm_length = params->length;  in mpr_read_config_page()
3969 	if (cm->cm_data != NULL) {  in mpr_read_config_page()
3970 		cm->cm_sge = &req->PageBufferSGE;  in mpr_read_config_page()
3971 		cm->cm_sglsize = sizeof(MPI2_SGE_IO_UNION);  in mpr_read_config_page()
3972 		cm->cm_flags = MPR_CM_FLAGS_SGE_SIMPLE | MPR_CM_FLAGS_DATAIN;  in mpr_read_config_page()
3974 		cm->cm_sge = NULL;  in mpr_read_config_page()
3975 	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;  in mpr_read_config_page()
3977 	cm->cm_complete_data = params;  in mpr_read_config_page()
3979 		cm->cm_complete = mpr_config_complete;  in mpr_read_config_page()
3980 		return (mpr_map_command(sc, cm));  in mpr_read_config_page()
3982 		error = mpr_wait_command(sc, &cm, 0, CAN_SLEEP);  in mpr_read_config_page()
3986 			if (cm != NULL)  in mpr_read_config_page()
3987 				mpr_free_command(sc, cm);  in mpr_read_config_page()
3990 		mpr_config_complete(sc, cm);  in mpr_read_config_page()
4003 mpr_config_complete(struct mpr_softc *sc, struct mpr_command *cm)  in mpr_config_complete()  argument
4009 	params = cm->cm_complete_data;  in mpr_config_complete()
4011 	if (cm->cm_data != NULL) {  in mpr_config_complete()
4012 		bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap,  in mpr_config_complete()
4014 		bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);  in mpr_config_complete()
4021 	if ((cm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) {  in mpr_config_complete()
4026 	reply = (MPI2_CONFIG_REPLY *)cm->cm_reply;  in mpr_config_complete()
4046 	mpr_free_command(sc, cm);  in mpr_config_complete()