36 static void cdx_mcdi_start_or_queue(struct cdx_mcdi_iface *mcdi,
38 static void cdx_mcdi_cmd_start_or_queue(struct cdx_mcdi_iface *mcdi,
40 static bool cdx_mcdi_complete_cmd(struct cdx_mcdi_iface *mcdi,
45 static void cdx_mcdi_timeout_cmd(struct cdx_mcdi_iface *mcdi,
68 static void _cdx_mcdi_remove_cmd(struct cdx_mcdi_iface *mcdi,  in _cdx_mcdi_remove_cmd()  argument
78 		++mcdi->outstanding_cleanups;  in _cdx_mcdi_remove_cmd()
83 static void cdx_mcdi_remove_cmd(struct cdx_mcdi_iface *mcdi,  in cdx_mcdi_remove_cmd()  argument
88 	_cdx_mcdi_remove_cmd(mcdi, cmd, cleanup_list);  in cdx_mcdi_remove_cmd()
91 	if (list_empty(&mcdi->cmd_list))  in cdx_mcdi_remove_cmd()
92 		wake_up(&mcdi->cmd_complete_wq);  in cdx_mcdi_remove_cmd()
118 	struct cdx_mcdi_iface *mcdi;  in cdx_mcdi_init()  local
121 	cdx->mcdi = kzalloc_obj(*cdx->mcdi);  in cdx_mcdi_init()
122 	if (!cdx->mcdi)  in cdx_mcdi_init()
125 	mcdi = cdx_mcdi_if(cdx);  in cdx_mcdi_init()
126 	mcdi->cdx = cdx;  in cdx_mcdi_init()
128 	mcdi->workqueue = alloc_ordered_workqueue("mcdi_wq", 0);  in cdx_mcdi_init()
129 	if (!mcdi->workqueue)  in cdx_mcdi_init()
131 	mutex_init(&mcdi->iface_lock);  in cdx_mcdi_init()
132 	mcdi->mode = MCDI_MODE_EVENTS;  in cdx_mcdi_init()
133 	INIT_LIST_HEAD(&mcdi->cmd_list);  in cdx_mcdi_init()
134 	init_waitqueue_head(&mcdi->cmd_complete_wq);  in cdx_mcdi_init()
136 	mcdi->new_epoch = true;  in cdx_mcdi_init()
140 	kfree(cdx->mcdi);  in cdx_mcdi_init()
141 	cdx->mcdi = NULL;  in cdx_mcdi_init()
157 	struct cdx_mcdi_iface *mcdi;  in cdx_mcdi_finish()  local
159 	mcdi = cdx_mcdi_if(cdx);  in cdx_mcdi_finish()
160 	if (!mcdi)  in cdx_mcdi_finish()
165 	destroy_workqueue(mcdi->workqueue);  in cdx_mcdi_finish()
166 	kfree(cdx->mcdi);  in cdx_mcdi_finish()
167 	cdx->mcdi = NULL;  in cdx_mcdi_finish()
171 static bool cdx_mcdi_flushed(struct cdx_mcdi_iface *mcdi, bool ignore_cleanups)  in cdx_mcdi_flushed()  argument
175 	mutex_lock(&mcdi->iface_lock);  in cdx_mcdi_flushed()
176 	flushed = list_empty(&mcdi->cmd_list) &&  in cdx_mcdi_flushed()
177 		  (ignore_cleanups || !mcdi->outstanding_cleanups);  in cdx_mcdi_flushed()
178 	mutex_unlock(&mcdi->iface_lock);  in cdx_mcdi_flushed()
185 	struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);  in cdx_mcdi_wait_for_cleanup()  local
187 	if (!mcdi)  in cdx_mcdi_wait_for_cleanup()
190 	wait_event(mcdi->cmd_complete_wq,  in cdx_mcdi_wait_for_cleanup()
191 		   cdx_mcdi_flushed(mcdi, false));  in cdx_mcdi_wait_for_cleanup()
197 	struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);  in cdx_mcdi_wait_for_quiescence()  local
201 	if (!mcdi)  in cdx_mcdi_wait_for_quiescence()
204 	flush_workqueue(mcdi->workqueue);  in cdx_mcdi_wait_for_quiescence()
206 	add_wait_queue(&mcdi->cmd_complete_wq, &wait);  in cdx_mcdi_wait_for_quiescence()
208 	while (!cdx_mcdi_flushed(mcdi, true)) {  in cdx_mcdi_wait_for_quiescence()
215 	remove_wait_queue(&mcdi->cmd_complete_wq, &wait);  in cdx_mcdi_wait_for_quiescence()
245 	struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);  in cdx_mcdi_send_request()  local
253 	if (!mcdi)  in cdx_mcdi_send_request()
256 	mcdi->prev_seq = cmd->seq;  in cdx_mcdi_send_request()
257 	mcdi->seq_held_by[cmd->seq] = cmd;  in cdx_mcdi_send_request()
258 	mcdi->db_held_by = cmd;  in cdx_mcdi_send_request()
261 	not_epoch = !mcdi->new_epoch;  in cdx_mcdi_send_request()
289 	mcdi->new_epoch = false;  in cdx_mcdi_send_request()
342 	struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);  in cdx_mcdi_process_cleanup_list()  local
345 	if (!mcdi)  in cdx_mcdi_process_cleanup_list()
362 		mutex_lock(&mcdi->iface_lock);  in cdx_mcdi_process_cleanup_list()
363 		CDX_WARN_ON_PARANOID(cleanups > mcdi->outstanding_cleanups);  in cdx_mcdi_process_cleanup_list()
364 		all_done = (mcdi->outstanding_cleanups -= cleanups) == 0;  in cdx_mcdi_process_cleanup_list()
365 		mutex_unlock(&mcdi->iface_lock);  in cdx_mcdi_process_cleanup_list()
367 			wake_up(&mcdi->cmd_complete_wq);  in cdx_mcdi_process_cleanup_list()
371 static void _cdx_mcdi_cancel_cmd(struct cdx_mcdi_iface *mcdi,  in _cdx_mcdi_cancel_cmd()  argument
377 	list_for_each_entry(cmd, &mcdi->cmd_list, list)  in _cdx_mcdi_cancel_cmd()
386 				cdx_mcdi_remove_cmd(mcdi, cmd, cleanup_list);  in _cdx_mcdi_cancel_cmd()
401 	struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);  in cdx_mcdi_cancel_cmd()  local
404 	if (!mcdi)  in cdx_mcdi_cancel_cmd()
407 	mutex_lock(&mcdi->iface_lock);  in cdx_mcdi_cancel_cmd()
408 	cdx_mcdi_timeout_cmd(mcdi, cmd, &cleanup_list);  in cdx_mcdi_cancel_cmd()
409 	mutex_unlock(&mcdi->iface_lock);  in cdx_mcdi_cancel_cmd()
513 static bool cdx_mcdi_get_seq(struct cdx_mcdi_iface *mcdi, unsigned char *seq)  in cdx_mcdi_get_seq()  argument
515 	*seq = mcdi->prev_seq;  in cdx_mcdi_get_seq()
517 		*seq = (*seq + 1) % ARRAY_SIZE(mcdi->seq_held_by);  in cdx_mcdi_get_seq()
518 	} while (mcdi->seq_held_by[*seq] && *seq != mcdi->prev_seq);  in cdx_mcdi_get_seq()
519 	return !mcdi->seq_held_by[*seq];  in cdx_mcdi_get_seq()
526 	struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);  in cdx_mcdi_rpc_async_internal()  local
529 	if (!mcdi) {  in cdx_mcdi_rpc_async_internal()
534 	if (mcdi->mode == MCDI_MODE_FAIL) {  in cdx_mcdi_rpc_async_internal()
539 	cmd->mcdi = mcdi;  in cdx_mcdi_rpc_async_internal()
547 	queue_work(mcdi->workqueue, &cmd->work);  in cdx_mcdi_rpc_async_internal()
551 static void cdx_mcdi_cmd_start_or_queue(struct cdx_mcdi_iface *mcdi,  in cdx_mcdi_cmd_start_or_queue()  argument
554 	struct cdx_mcdi *cdx = mcdi->cdx;  in cdx_mcdi_cmd_start_or_queue()
557 	if (!mcdi->db_held_by &&  in cdx_mcdi_cmd_start_or_queue()
558 	    cdx_mcdi_get_seq(mcdi, &seq)) {  in cdx_mcdi_cmd_start_or_queue()
569 static void cdx_mcdi_start_or_queue(struct cdx_mcdi_iface *mcdi,  in cdx_mcdi_start_or_queue()  argument
574 	list_for_each_entry_safe(cmd, tmp, &mcdi->cmd_list, list)  in cdx_mcdi_start_or_queue()
577 			cdx_mcdi_cmd_start_or_queue(mcdi, cmd);  in cdx_mcdi_start_or_queue()
595 	struct cdx_mcdi_iface *mcdi;  in cdx_mcdi_process_cmd()  local
605 	mcdi = cdx_mcdi_if(cdx);  in cdx_mcdi_process_cmd()
606 	if (!mcdi)  in cdx_mcdi_process_cmd()
611 	mutex_lock(&mcdi->iface_lock);  in cdx_mcdi_process_cmd()
612 	cmd = mcdi->seq_held_by[respseq];  in cdx_mcdi_process_cmd()
616 			mutex_unlock(&mcdi->iface_lock);  in cdx_mcdi_process_cmd()
621 		cdx_mcdi_complete_cmd(mcdi, cmd, outbuf, len, &cleanup_list);  in cdx_mcdi_process_cmd()
626 	mutex_unlock(&mcdi->iface_lock);  in cdx_mcdi_process_cmd()
628 	cdx_mcdi_process_cleanup_list(mcdi->cdx, &cleanup_list);  in cdx_mcdi_process_cmd()
636 	struct cdx_mcdi_iface *mcdi = cmd->mcdi;  in cdx_mcdi_cmd_work()  local
638 	mutex_lock(&mcdi->iface_lock);  in cdx_mcdi_cmd_work()
640 	cmd->handle = mcdi->prev_handle++;  in cdx_mcdi_cmd_work()
641 	list_add_tail(&cmd->list, &mcdi->cmd_list);  in cdx_mcdi_cmd_work()
642 	cdx_mcdi_cmd_start_or_queue(mcdi, cmd);  in cdx_mcdi_cmd_work()
644 	mutex_unlock(&mcdi->iface_lock);  in cdx_mcdi_cmd_work()
652 static bool cdx_mcdi_complete_cmd(struct cdx_mcdi_iface *mcdi,  in cdx_mcdi_complete_cmd()  argument
659 	struct cdx_mcdi *cdx = mcdi->cdx;  in cdx_mcdi_complete_cmd()
720 	if (mcdi->db_held_by == cmd)  in cdx_mcdi_complete_cmd()
721 		mcdi->db_held_by = NULL;  in cdx_mcdi_complete_cmd()
733 		cdx_mcdi_remove_cmd(mcdi, cmd, cleanup_list);  in cdx_mcdi_complete_cmd()
738 	mcdi->seq_held_by[cmd->seq] = NULL;  in cdx_mcdi_complete_cmd()
740 	cdx_mcdi_start_or_queue(mcdi, rc != MC_CMD_ERR_QUEUE_FULL);  in cdx_mcdi_complete_cmd()
743 	wake_up(&mcdi->cmd_complete_wq);  in cdx_mcdi_complete_cmd()
750 static void cdx_mcdi_timeout_cmd(struct cdx_mcdi_iface *mcdi,  in cdx_mcdi_timeout_cmd()  argument
754 	struct cdx_mcdi *cdx = mcdi->cdx;  in cdx_mcdi_timeout_cmd()
761 	cdx_mcdi_remove_cmd(mcdi, cmd, cleanup_list);  in cdx_mcdi_timeout_cmd()
856 	struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);  in cdx_mcdi_mode_fail()  local
858 	if (!mcdi)  in cdx_mcdi_mode_fail()
861 	mcdi->mode = MCDI_MODE_FAIL;  in cdx_mcdi_mode_fail()
863 	while (!list_empty(&mcdi->cmd_list)) {  in cdx_mcdi_mode_fail()
866 		cmd = list_first_entry(&mcdi->cmd_list, struct cdx_mcdi_cmd,  in cdx_mcdi_mode_fail()
868 		_cdx_mcdi_cancel_cmd(mcdi, cdx_mcdi_cmd_handle(cmd), cleanup_list);  in cdx_mcdi_mode_fail()