27 nvmft_printf(struct nvmft_controller *ctrlr, const char *fmt, ...)  in nvmft_printf()  argument
37 	sbuf_printf(&sb, "nvmft%u: ", ctrlr->cntlid);  in nvmft_printf()
53 	struct nvmft_controller *ctrlr;  in nvmft_controller_alloc()  local
55 	ctrlr = malloc(sizeof(*ctrlr), M_NVMFT, M_WAITOK | M_ZERO);  in nvmft_controller_alloc()
56 	ctrlr->cntlid = cntlid;  in nvmft_controller_alloc()
57 	ctrlr->np = np;  in nvmft_controller_alloc()
58 	mtx_init(&ctrlr->lock, "nvmft controller", NULL, MTX_DEF);  in nvmft_controller_alloc()
59 	callout_init(&ctrlr->ka_timer, 1);  in nvmft_controller_alloc()
60 	TASK_INIT(&ctrlr->shutdown_task, 0, nvmft_controller_shutdown, ctrlr);  in nvmft_controller_alloc()
61 	TIMEOUT_TASK_INIT(taskqueue_thread, &ctrlr->terminate_task, 0,  in nvmft_controller_alloc()
62 	    nvmft_controller_terminate, ctrlr);  in nvmft_controller_alloc()
64 	ctrlr->cdata = np->cdata;  in nvmft_controller_alloc()
65 	ctrlr->cdata.ctrlr_id = htole16(cntlid);  in nvmft_controller_alloc()
66 	memcpy(ctrlr->hostid, data->hostid, sizeof(ctrlr->hostid));  in nvmft_controller_alloc()
67 	memcpy(ctrlr->hostnqn, data->hostnqn, sizeof(ctrlr->hostnqn));  in nvmft_controller_alloc()
68 	ctrlr->hip.power_cycles[0] = 1;  in nvmft_controller_alloc()
69 	ctrlr->create_time = sbinuptime();  in nvmft_controller_alloc()
71 	ctrlr->changed_ns = malloc(sizeof(*ctrlr->changed_ns), M_NVMFT,  in nvmft_controller_alloc()
74 	return (ctrlr);  in nvmft_controller_alloc()
78 nvmft_controller_free(struct nvmft_controller *ctrlr)  in nvmft_controller_free()  argument
80 	mtx_destroy(&ctrlr->lock);  in nvmft_controller_free()
81 	MPASS(ctrlr->io_qpairs == NULL);  in nvmft_controller_free()
82 	free(ctrlr->changed_ns, M_NVMFT);  in nvmft_controller_free()
83 	free(ctrlr, M_NVMFT);  in nvmft_controller_free()
89 	struct nvmft_controller *ctrlr = arg;  in nvmft_keep_alive_timer()  local
92 	if (ctrlr->shutdown)  in nvmft_keep_alive_timer()
95 	traffic = atomic_readandclear_int(&ctrlr->ka_active_traffic);  in nvmft_keep_alive_timer()
97 		nvmft_printf(ctrlr,  in nvmft_keep_alive_timer()
99 		nvmft_controller_error(ctrlr, NULL, ETIMEDOUT);  in nvmft_keep_alive_timer()
103 	callout_schedule_sbt(&ctrlr->ka_timer, ctrlr->ka_sbt, 0, C_HARDCLOCK);  in nvmft_keep_alive_timer()
107 nvmft_update_cdata(struct nvmft_controller *ctrlr)  in nvmft_update_cdata()  argument
111 	val = nvmft_max_ioccsz(ctrlr->admin);  in nvmft_update_cdata()
113 		ioccsz = le32toh(ctrlr->cdata.ioccsz) * 16;  in nvmft_update_cdata()
115 			ctrlr->cdata.ioccsz = htole32(val / 16);  in nvmft_update_cdata()
124 	struct nvmft_controller *ctrlr;  in nvmft_handoff_admin_queue()  local
152 	TAILQ_FOREACH(ctrlr, &np->controllers, link) {  in nvmft_handoff_admin_queue()
153 		KASSERT(ctrlr->cntlid != cntlid,  in nvmft_handoff_admin_queue()
159 	ctrlr = nvmft_controller_alloc(np, cntlid, data);  in nvmft_handoff_admin_queue()
164 		nvmft_controller_free(ctrlr);  in nvmft_handoff_admin_queue()
170 	TAILQ_INSERT_TAIL(&np->controllers, ctrlr, link);  in nvmft_handoff_admin_queue()
172 	nvmft_printf(ctrlr, "associated with %.*s\n",  in nvmft_handoff_admin_queue()
174 	ctrlr->admin = qp;  in nvmft_handoff_admin_queue()
175 	ctrlr->trtype = trtype;  in nvmft_handoff_admin_queue()
176 	nvmft_update_cdata(ctrlr);  in nvmft_handoff_admin_queue()
188 		ctrlr->ka_sbt = mstosbt(roundup(kato, 1000));  in nvmft_handoff_admin_queue()
189 		callout_reset_sbt(&ctrlr->ka_timer, ctrlr->ka_sbt, 0,  in nvmft_handoff_admin_queue()
190 		    nvmft_keep_alive_timer, ctrlr, C_HARDCLOCK);  in nvmft_handoff_admin_queue()
194 	nvmft_finish_accept(qp, cmd, ctrlr);  in nvmft_handoff_admin_queue()
204 	struct nvmft_controller *ctrlr;  in nvmft_handoff_io_queue()  local
223 	TAILQ_FOREACH(ctrlr, &np->controllers, link) {  in nvmft_handoff_io_queue()
224 		if (ctrlr->cntlid == cntlid)  in nvmft_handoff_io_queue()
227 	if (ctrlr == NULL) {  in nvmft_handoff_io_queue()
230 		    ctrlr->cntlid, qid, (int)sizeof(data->hostnqn),  in nvmft_handoff_io_queue()
238 	if (memcmp(ctrlr->hostid, data->hostid, sizeof(ctrlr->hostid)) != 0) {  in nvmft_handoff_io_queue()
240 		nvmft_printf(ctrlr,  in nvmft_handoff_io_queue()
248 	if (memcmp(ctrlr->hostnqn, data->hostnqn, sizeof(ctrlr->hostnqn)) != 0) {  in nvmft_handoff_io_queue()
250 		nvmft_printf(ctrlr,  in nvmft_handoff_io_queue()
261 	mtx_lock(&ctrlr->lock);  in nvmft_handoff_io_queue()
262 	if (ctrlr->shutdown) {  in nvmft_handoff_io_queue()
263 		mtx_unlock(&ctrlr->lock);  in nvmft_handoff_io_queue()
265 		nvmft_printf(ctrlr,  in nvmft_handoff_io_queue()
273 	if (ctrlr->num_io_queues == 0) {  in nvmft_handoff_io_queue()
274 		mtx_unlock(&ctrlr->lock);  in nvmft_handoff_io_queue()
276 		nvmft_printf(ctrlr,  in nvmft_handoff_io_queue()
284 	if (cmd->qid > ctrlr->num_io_queues) {  in nvmft_handoff_io_queue()
285 		mtx_unlock(&ctrlr->lock);  in nvmft_handoff_io_queue()
287 		nvmft_printf(ctrlr,  in nvmft_handoff_io_queue()
295 	if (ctrlr->io_qpairs[qid - 1].qp != NULL) {  in nvmft_handoff_io_queue()
296 		mtx_unlock(&ctrlr->lock);  in nvmft_handoff_io_queue()
298 		nvmft_printf(ctrlr,  in nvmft_handoff_io_queue()
307 	ctrlr->io_qpairs[qid - 1].qp = qp;  in nvmft_handoff_io_queue()
308 	mtx_unlock(&ctrlr->lock);  in nvmft_handoff_io_queue()
310 	nvmft_finish_accept(qp, cmd, ctrlr);  in nvmft_handoff_io_queue()
318 	struct nvmft_controller *ctrlr = arg;  in nvmft_controller_shutdown()  local
326 	mtx_lock(&ctrlr->lock);  in nvmft_controller_shutdown()
327 	for (u_int i = 0; i < ctrlr->num_io_queues; i++) {  in nvmft_controller_shutdown()
328 		if (ctrlr->io_qpairs[i].qp != NULL) {  in nvmft_controller_shutdown()
329 			ctrlr->io_qpairs[i].shutdown = true;  in nvmft_controller_shutdown()
330 			mtx_unlock(&ctrlr->lock);  in nvmft_controller_shutdown()
331 			nvmft_qpair_shutdown(ctrlr->io_qpairs[i].qp);  in nvmft_controller_shutdown()
332 			mtx_lock(&ctrlr->lock);  in nvmft_controller_shutdown()
335 	mtx_unlock(&ctrlr->lock);  in nvmft_controller_shutdown()
338 	nvmft_terminate_commands(ctrlr);  in nvmft_controller_shutdown()
341 	mtx_lock(&ctrlr->lock);  in nvmft_controller_shutdown()
342 	while (ctrlr->pending_commands != 0)  in nvmft_controller_shutdown()
343 		mtx_sleep(&ctrlr->pending_commands, &ctrlr->lock, 0, "nvmftsh",  in nvmft_controller_shutdown()
345 	mtx_unlock(&ctrlr->lock);  in nvmft_controller_shutdown()
348 	for (u_int i = 0; i < ctrlr->num_io_queues; i++) {  in nvmft_controller_shutdown()
349 		if (ctrlr->io_qpairs[i].qp != NULL)  in nvmft_controller_shutdown()
350 			nvmft_qpair_destroy(ctrlr->io_qpairs[i].qp);  in nvmft_controller_shutdown()
352 	free(ctrlr->io_qpairs, M_NVMFT);  in nvmft_controller_shutdown()
353 	ctrlr->io_qpairs = NULL;  in nvmft_controller_shutdown()
355 	mtx_lock(&ctrlr->lock);  in nvmft_controller_shutdown()
356 	ctrlr->num_io_queues = 0;  in nvmft_controller_shutdown()
359 	if (NVMEV(NVME_CSTS_REG_SHST, ctrlr->csts) == NVME_SHST_OCCURRING) {  in nvmft_controller_shutdown()
360 		ctrlr->csts &= ~NVMEM(NVME_CSTS_REG_SHST);  in nvmft_controller_shutdown()
361 		ctrlr->csts |= NVMEF(NVME_CSTS_REG_SHST, NVME_SHST_COMPLETE);  in nvmft_controller_shutdown()
364 	if (NVMEV(NVME_CSTS_REG_CFS, ctrlr->csts) == 0) {  in nvmft_controller_shutdown()
365 		ctrlr->csts &= ~NVMEM(NVME_CSTS_REG_RDY);  in nvmft_controller_shutdown()
366 		ctrlr->shutdown = false;  in nvmft_controller_shutdown()
368 	mtx_unlock(&ctrlr->lock);  in nvmft_controller_shutdown()
376 	if (ctrlr->admin_closed || NVMEV(NVME_CSTS_REG_CFS, ctrlr->csts) != 0)  in nvmft_controller_shutdown()
377 		nvmft_controller_terminate(ctrlr, 0);  in nvmft_controller_shutdown()
380 		    &ctrlr->terminate_task, hz * 60 * 2);  in nvmft_controller_shutdown()
386 	struct nvmft_controller *ctrlr = arg;  in nvmft_controller_terminate()  local
391 	mtx_lock(&ctrlr->lock);  in nvmft_controller_terminate()
392 	if (NVMEV(NVME_CC_REG_EN, ctrlr->cc) != 0) {  in nvmft_controller_terminate()
393 		mtx_unlock(&ctrlr->lock);  in nvmft_controller_terminate()
395 		if (ctrlr->ka_sbt != 0)  in nvmft_controller_terminate()
396 			callout_schedule_sbt(&ctrlr->ka_timer, ctrlr->ka_sbt, 0,  in nvmft_controller_terminate()
402 	ctrlr->shutdown = true;  in nvmft_controller_terminate()
403 	mtx_unlock(&ctrlr->lock);  in nvmft_controller_terminate()
405 	nvmft_qpair_destroy(ctrlr->admin);  in nvmft_controller_terminate()
408 	np = ctrlr->np;  in nvmft_controller_terminate()
410 	TAILQ_REMOVE(&np->controllers, ctrlr, link);  in nvmft_controller_terminate()
413 	free_unr(np->ids, ctrlr->cntlid);  in nvmft_controller_terminate()
417 	callout_drain(&ctrlr->ka_timer);  in nvmft_controller_terminate()
419 	nvmft_printf(ctrlr, "association terminated\n");  in nvmft_controller_terminate()
420 	nvmft_controller_free(ctrlr);  in nvmft_controller_terminate()
425 nvmft_controller_error(struct nvmft_controller *ctrlr, struct nvmft_qpair *qp,  in nvmft_controller_error()  argument
439 		if (qp != ctrlr->admin)  in nvmft_controller_error()
442 		mtx_lock(&ctrlr->lock);  in nvmft_controller_error()
443 		if (ctrlr->shutdown) {  in nvmft_controller_error()
444 			ctrlr->admin_closed = true;  in nvmft_controller_error()
445 			mtx_unlock(&ctrlr->lock);  in nvmft_controller_error()
449 		if (NVMEV(NVME_CC_REG_EN, ctrlr->cc) == 0) {  in nvmft_controller_error()
450 			MPASS(ctrlr->num_io_queues == 0);  in nvmft_controller_error()
451 			mtx_unlock(&ctrlr->lock);  in nvmft_controller_error()
469 			    &ctrlr->terminate_task, NULL) == 0)  in nvmft_controller_error()
471 				    &ctrlr->terminate_task, 0);  in nvmft_controller_error()
480 		ctrlr->admin_closed = true;  in nvmft_controller_error()
482 		mtx_lock(&ctrlr->lock);  in nvmft_controller_error()
485 	if (ctrlr->shutdown) {  in nvmft_controller_error()
486 		mtx_unlock(&ctrlr->lock);  in nvmft_controller_error()
490 	ctrlr->csts |= NVMEF(NVME_CSTS_REG_CFS, 1);  in nvmft_controller_error()
491 	ctrlr->cc &= ~NVMEM(NVME_CC_REG_EN);  in nvmft_controller_error()
492 	ctrlr->shutdown = true;  in nvmft_controller_error()
493 	mtx_unlock(&ctrlr->lock);  in nvmft_controller_error()
495 	callout_stop(&ctrlr->ka_timer);  in nvmft_controller_error()
496 	taskqueue_enqueue(taskqueue_thread, &ctrlr->shutdown_task);  in nvmft_controller_error()
550 handle_get_log_page(struct nvmft_controller *ctrlr,  in handle_get_log_page()  argument
595 		mtx_lock(&ctrlr->lock);  in handle_get_log_page()
596 		hip = ctrlr->hip;  in handle_get_log_page()
598 		    sbintime_getsec(ctrlr->busy_total) / 60;  in handle_get_log_page()
600 		    sbintime_getsec(sbinuptime() - ctrlr->create_time) / 3600;  in handle_get_log_page()
601 		mtx_unlock(&ctrlr->lock);  in handle_get_log_page()
612 		if (offset >= sizeof(ctrlr->np->fp)) {  in handle_get_log_page()
616 		todo = sizeof(ctrlr->np->fp) - offset;  in handle_get_log_page()
621 		m_copyback(m, 0, todo, (char *)&ctrlr->np->fp + offset);  in handle_get_log_page()
628 		if (offset >= sizeof(*ctrlr->changed_ns)) {  in handle_get_log_page()
632 		todo = sizeof(*ctrlr->changed_ns) - offset;  in handle_get_log_page()
637 		mtx_lock(&ctrlr->lock);  in handle_get_log_page()
638 		m_copyback(m, 0, todo, (char *)ctrlr->changed_ns + offset);  in handle_get_log_page()
639 		if (offset == 0 && len == sizeof(*ctrlr->changed_ns))  in handle_get_log_page()
640 			memset(ctrlr->changed_ns, 0,  in handle_get_log_page()
641 			    sizeof(*ctrlr->changed_ns));  in handle_get_log_page()
643 			ctrlr->changed_ns_reported = false;  in handle_get_log_page()
644 		mtx_unlock(&ctrlr->lock);  in handle_get_log_page()
651 		nvmft_printf(ctrlr, "Unsupported page %#x for GET_LOG_PAGE\n",  in handle_get_log_page()
659 		nvmft_command_completed(ctrlr->admin, nc);  in handle_get_log_page()
661 		nvmft_send_generic_error(ctrlr->admin, nc, status);  in handle_get_log_page()
672 handle_identify_command(struct nvmft_controller *ctrlr,  in handle_identify_command()  argument
682 	if (data_len != sizeof(ctrlr->cdata)) {  in handle_identify_command()
683 		nvmft_printf(ctrlr,  in handle_identify_command()
686 		nvmft_send_generic_error(ctrlr->admin, nc,  in handle_identify_command()
695 		nvmft_dispatch_command(ctrlr->admin, nc, true);  in handle_identify_command()
699 		m = m_getml(sizeof(ctrlr->cdata), M_WAITOK);  in handle_identify_command()
700 		m_copyback(m, 0, sizeof(ctrlr->cdata), (void *)&ctrlr->cdata);  in handle_identify_command()
702 		    sizeof(ctrlr->cdata));  in handle_identify_command()
718 		nvmft_populate_active_nslist(ctrlr->np, nsid, nslist);  in handle_identify_command()
728 		nvmft_printf(ctrlr, "Unsupported CNS %#x for IDENTIFY\n", cns);  in handle_identify_command()
734 		nvmft_command_completed(ctrlr->admin, nc);  in handle_identify_command()
736 		nvmft_send_generic_error(ctrlr->admin, nc, status);  in handle_identify_command()
741 handle_set_features(struct nvmft_controller *ctrlr,  in handle_set_features()  argument
770 		mtx_lock(&ctrlr->lock);  in handle_set_features()
771 		if (ctrlr->num_io_queues != 0) {  in handle_set_features()
772 			mtx_unlock(&ctrlr->lock);  in handle_set_features()
774 			nvmft_send_generic_error(ctrlr->admin, nc,  in handle_set_features()
780 		ctrlr->num_io_queues = num_queues;  in handle_set_features()
781 		ctrlr->io_qpairs = io_qpairs;  in handle_set_features()
782 		mtx_unlock(&ctrlr->lock);  in handle_set_features()
786 		nvmft_send_response(ctrlr->admin, &cqe);  in handle_set_features()
800 		mtx_lock(&ctrlr->lock);  in handle_set_features()
801 		ctrlr->aer_mask = aer_mask;  in handle_set_features()
802 		mtx_unlock(&ctrlr->lock);  in handle_set_features()
803 		nvmft_send_success(ctrlr->admin, nc);  in handle_set_features()
808 		nvmft_printf(ctrlr,  in handle_set_features()
814 	nvmft_send_generic_error(ctrlr->admin, nc, NVME_SC_INVALID_FIELD);  in handle_set_features()
819 update_cc(struct nvmft_controller *ctrlr, uint32_t new_cc, bool *need_shutdown)  in update_cc()  argument
821 	struct nvmft_port *np = ctrlr->np;  in update_cc()
826 	mtx_lock(&ctrlr->lock);  in update_cc()
829 	if (ctrlr->shutdown) {  in update_cc()
830 		mtx_unlock(&ctrlr->lock);  in update_cc()
834 	if (!_nvmf_validate_cc(np->max_io_qsize, np->cap, ctrlr->cc, new_cc)) {  in update_cc()
835 		mtx_unlock(&ctrlr->lock);  in update_cc()
839 	changes = ctrlr->cc ^ new_cc;  in update_cc()
840 	ctrlr->cc = new_cc;  in update_cc()
845 		ctrlr->csts &= ~NVMEM(NVME_CSTS_REG_SHST);  in update_cc()
846 		ctrlr->csts |= NVMEF(NVME_CSTS_REG_SHST, NVME_SHST_OCCURRING);  in update_cc()
847 		ctrlr->cc &= ~NVMEM(NVME_CC_REG_EN);  in update_cc()
848 		ctrlr->shutdown = true;  in update_cc()
850 		nvmft_printf(ctrlr, "shutdown requested\n");  in update_cc()
856 			nvmft_printf(ctrlr, "reset requested\n");  in update_cc()
857 			ctrlr->shutdown = true;  in update_cc()
860 			ctrlr->csts |= NVMEF(NVME_CSTS_REG_RDY, 1);  in update_cc()
862 	mtx_unlock(&ctrlr->lock);  in update_cc()
868 handle_property_get(struct nvmft_controller *ctrlr, struct nvmf_capsule *nc,  in handle_property_get()  argument
879 		rsp.value.u64 = htole64(ctrlr->np->cap);  in handle_property_get()
884 		rsp.value.u32.low = ctrlr->cdata.ver;  in handle_property_get()
889 		rsp.value.u32.low = htole32(ctrlr->cc);  in handle_property_get()
894 		rsp.value.u32.low = htole32(ctrlr->csts);  in handle_property_get()
900 	nvmft_send_response(ctrlr->admin, &rsp);  in handle_property_get()
903 	nvmft_send_generic_error(ctrlr->admin, nc, NVME_SC_INVALID_FIELD);  in handle_property_get()
907 handle_property_set(struct nvmft_controller *ctrlr, struct nvmf_capsule *nc,  in handle_property_set()  argument
917 		if (!update_cc(ctrlr, le32toh(pset->value.u32.low),  in handle_property_set()
925 	nvmft_send_success(ctrlr->admin, nc);  in handle_property_set()
927 		callout_stop(&ctrlr->ka_timer);  in handle_property_set()
928 		taskqueue_enqueue(taskqueue_thread, &ctrlr->shutdown_task);  in handle_property_set()
932 	nvmft_send_generic_error(ctrlr->admin, nc, NVME_SC_INVALID_FIELD);  in handle_property_set()
936 handle_admin_fabrics_command(struct nvmft_controller *ctrlr,  in handle_admin_fabrics_command()  argument
941 		handle_property_get(ctrlr, nc,  in handle_admin_fabrics_command()
945 		handle_property_set(ctrlr, nc,  in handle_admin_fabrics_command()
949 		nvmft_printf(ctrlr,  in handle_admin_fabrics_command()
951 		nvmft_send_generic_error(ctrlr->admin, nc,  in handle_admin_fabrics_command()
955 		nvmft_printf(ctrlr, "DISCONNECT command on admin queue\n");  in handle_admin_fabrics_command()
956 		nvmft_send_error(ctrlr->admin, nc, NVME_SCT_COMMAND_SPECIFIC,  in handle_admin_fabrics_command()
960 		nvmft_printf(ctrlr, "Unsupported fabrics command %#x\n",  in handle_admin_fabrics_command()
962 		nvmft_send_generic_error(ctrlr->admin, nc,  in handle_admin_fabrics_command()
970 nvmft_handle_admin_command(struct nvmft_controller *ctrlr,  in nvmft_handle_admin_command()  argument
976 	if (NVMEV(NVME_CC_REG_EN, ctrlr->cc) == 0 &&  in nvmft_handle_admin_command()
978 		nvmft_printf(ctrlr,  in nvmft_handle_admin_command()
980 		nvmft_send_generic_error(ctrlr->admin, nc,  in nvmft_handle_admin_command()
986 	atomic_store_int(&ctrlr->ka_active_traffic, 1);  in nvmft_handle_admin_command()
990 		handle_get_log_page(ctrlr, nc, cmd);  in nvmft_handle_admin_command()
993 		handle_identify_command(ctrlr, nc, cmd);  in nvmft_handle_admin_command()
996 		handle_set_features(ctrlr, nc, cmd);  in nvmft_handle_admin_command()
999 		mtx_lock(&ctrlr->lock);  in nvmft_handle_admin_command()
1000 		if (ctrlr->aer_pending == NVMFT_NUM_AER) {  in nvmft_handle_admin_command()
1001 			mtx_unlock(&ctrlr->lock);  in nvmft_handle_admin_command()
1002 			nvmft_send_error(ctrlr->admin, nc,  in nvmft_handle_admin_command()
1007 			ctrlr->aer_cids[ctrlr->aer_pidx] = cmd->cid;  in nvmft_handle_admin_command()
1008 			ctrlr->aer_pending++;  in nvmft_handle_admin_command()
1009 			ctrlr->aer_pidx = (ctrlr->aer_pidx + 1) % NVMFT_NUM_AER;  in nvmft_handle_admin_command()
1010 			mtx_unlock(&ctrlr->lock);  in nvmft_handle_admin_command()
1015 		nvmft_send_success(ctrlr->admin, nc);  in nvmft_handle_admin_command()
1019 		handle_admin_fabrics_command(ctrlr, nc,  in nvmft_handle_admin_command()
1023 		nvmft_printf(ctrlr, "Unsupported admin opcode %#x\n", cmd->opc);  in nvmft_handle_admin_command()
1024 		nvmft_send_generic_error(ctrlr->admin, nc,  in nvmft_handle_admin_command()
1035 	struct nvmft_controller *ctrlr = nvmft_qpair_ctrlr(qp);  in nvmft_handle_io_command()  local
1038 	atomic_store_int(&ctrlr->ka_active_traffic, 1);  in nvmft_handle_io_command()
1059 		nvmft_printf(ctrlr, "Unsupported I/O opcode %#x\n", cmd->opc);  in nvmft_handle_io_command()
1068 nvmft_report_aer(struct nvmft_controller *ctrlr, uint32_t aer_mask,  in nvmft_report_aer()  argument
1076 	mtx_lock(&ctrlr->lock);  in nvmft_report_aer()
1077 	if ((ctrlr->aer_mask & aer_mask) == 0) {  in nvmft_report_aer()
1078 		mtx_unlock(&ctrlr->lock);  in nvmft_report_aer()
1086 	if (ctrlr->aer_pending == 0) {  in nvmft_report_aer()
1087 		mtx_unlock(&ctrlr->lock);  in nvmft_report_aer()
1088 		nvmft_printf(ctrlr,  in nvmft_report_aer()
1095 	cpl.cid = ctrlr->aer_cids[ctrlr->aer_cidx];  in nvmft_report_aer()
1096 	ctrlr->aer_pending--;  in nvmft_report_aer()
1097 	ctrlr->aer_cidx = (ctrlr->aer_cidx + 1) % NVMFT_NUM_AER;  in nvmft_report_aer()
1098 	mtx_unlock(&ctrlr->lock);  in nvmft_report_aer()
1104 	nvmft_send_response(ctrlr->admin, &cpl);  in nvmft_report_aer()
1108 nvmft_controller_lun_changed(struct nvmft_controller *ctrlr, int lun_id)  in nvmft_controller_lun_changed()  argument
1116 	mtx_lock(&ctrlr->lock);  in nvmft_controller_lun_changed()
1117 	nslist = ctrlr->changed_ns;  in nvmft_controller_lun_changed()
1126 				mtx_unlock(&ctrlr->lock);  in nvmft_controller_lun_changed()
1136 			memset(ctrlr->changed_ns, 0,  in nvmft_controller_lun_changed()
1137 			    sizeof(*ctrlr->changed_ns));  in nvmft_controller_lun_changed()
1138 			ctrlr->changed_ns->ns[0] = 0xffffffff;  in nvmft_controller_lun_changed()
1153 	if (ctrlr->changed_ns_reported) {  in nvmft_controller_lun_changed()
1154 		mtx_unlock(&ctrlr->lock);  in nvmft_controller_lun_changed()
1157 	ctrlr->changed_ns_reported = true;  in nvmft_controller_lun_changed()
1158 	mtx_unlock(&ctrlr->lock);  in nvmft_controller_lun_changed()
1160 	nvmft_report_aer(ctrlr, NVME_ASYNC_EVENT_NS_ATTRIBUTE, 0x2, 0x0,  in nvmft_controller_lun_changed()