35 channel_to_ops(struct tegra_bpmp_channel *channel)  in channel_to_ops()  argument
37 	struct tegra_bpmp *bpmp = channel->bpmp;  in channel_to_ops()
79 tegra_bpmp_channel_get_thread_index(struct tegra_bpmp_channel *channel)  in tegra_bpmp_channel_get_thread_index()  argument
81 	struct tegra_bpmp *bpmp = channel->bpmp;  in tegra_bpmp_channel_get_thread_index()
87 	index = channel - channel->bpmp->threaded_channels;  in tegra_bpmp_channel_get_thread_index()
102 static bool tegra_bpmp_is_response_ready(struct tegra_bpmp_channel *channel)  in tegra_bpmp_is_response_ready()  argument
104 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_is_response_ready()
106 	return ops->is_response_ready(channel);  in tegra_bpmp_is_response_ready()
109 static bool tegra_bpmp_is_request_ready(struct tegra_bpmp_channel *channel)  in tegra_bpmp_is_request_ready()  argument
111 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_is_request_ready()
113 	return ops->is_request_ready(channel);  in tegra_bpmp_is_request_ready()
116 static int tegra_bpmp_wait_response(struct tegra_bpmp_channel *channel)  in tegra_bpmp_wait_response()  argument
118 	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;  in tegra_bpmp_wait_response()
124 		if (tegra_bpmp_is_response_ready(channel))  in tegra_bpmp_wait_response()
131 static int tegra_bpmp_ack_response(struct tegra_bpmp_channel *channel)  in tegra_bpmp_ack_response()  argument
133 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_ack_response()
135 	return ops->ack_response(channel);  in tegra_bpmp_ack_response()
138 static int tegra_bpmp_ack_request(struct tegra_bpmp_channel *channel)  in tegra_bpmp_ack_request()  argument
140 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_ack_request()
142 	return ops->ack_request(channel);  in tegra_bpmp_ack_request()
146 tegra_bpmp_is_request_channel_free(struct tegra_bpmp_channel *channel)  in tegra_bpmp_is_request_channel_free()  argument
148 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_is_request_channel_free()
150 	return ops->is_request_channel_free(channel);  in tegra_bpmp_is_request_channel_free()
154 tegra_bpmp_is_response_channel_free(struct tegra_bpmp_channel *channel)  in tegra_bpmp_is_response_channel_free()  argument
156 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_is_response_channel_free()
158 	return ops->is_response_channel_free(channel);  in tegra_bpmp_is_response_channel_free()
162 tegra_bpmp_wait_request_channel_free(struct tegra_bpmp_channel *channel)  in tegra_bpmp_wait_request_channel_free()  argument
164 	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;  in tegra_bpmp_wait_request_channel_free()
170 		if (tegra_bpmp_is_request_channel_free(channel))  in tegra_bpmp_wait_request_channel_free()
179 static int tegra_bpmp_post_request(struct tegra_bpmp_channel *channel)  in tegra_bpmp_post_request()  argument
181 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_post_request()
183 	return ops->post_request(channel);  in tegra_bpmp_post_request()
186 static int tegra_bpmp_post_response(struct tegra_bpmp_channel *channel)  in tegra_bpmp_post_response()  argument
188 	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);  in tegra_bpmp_post_response()
190 	return ops->post_response(channel);  in tegra_bpmp_post_response()
198 static ssize_t __tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,  in __tegra_bpmp_channel_read()  argument
204 		memcpy(data, channel->ib->data, size);  in __tegra_bpmp_channel_read()
206 	err = tegra_bpmp_ack_response(channel);  in __tegra_bpmp_channel_read()
210 	*ret = channel->ib->code;  in __tegra_bpmp_channel_read()
215 static ssize_t tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,  in tegra_bpmp_channel_read()  argument
218 	struct tegra_bpmp *bpmp = channel->bpmp;  in tegra_bpmp_channel_read()
223 	index = tegra_bpmp_channel_get_thread_index(channel);  in tegra_bpmp_channel_read()
230 	err = __tegra_bpmp_channel_read(channel, data, size, ret);  in tegra_bpmp_channel_read()
240 static ssize_t __tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,  in __tegra_bpmp_channel_write()  argument
244 	channel->ob->code = mrq;  in __tegra_bpmp_channel_write()
245 	channel->ob->flags = flags;  in __tegra_bpmp_channel_write()
248 		memcpy(channel->ob->data, data, size);  in __tegra_bpmp_channel_write()
250 	return tegra_bpmp_post_request(channel);  in __tegra_bpmp_channel_write()
259 	struct tegra_bpmp_channel *channel;  in tegra_bpmp_write_threaded()  local
276 	channel = &bpmp->threaded_channels[index];  in tegra_bpmp_write_threaded()
278 	if (!tegra_bpmp_is_request_channel_free(channel)) {  in tegra_bpmp_write_threaded()
285 	err = __tegra_bpmp_channel_write(channel, mrq, MSG_ACK | MSG_RING,  in tegra_bpmp_write_threaded()
293 	return channel;  in tegra_bpmp_write_threaded()
304 static ssize_t tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,  in tegra_bpmp_channel_write()  argument
310 	err = tegra_bpmp_wait_request_channel_free(channel);  in tegra_bpmp_channel_write()
314 	return __tegra_bpmp_channel_write(channel, mrq, flags, data, size);  in tegra_bpmp_channel_write()
320 	struct tegra_bpmp_channel *channel;  in tegra_bpmp_transfer_atomic()  local
329 	channel = bpmp->tx_channel;  in tegra_bpmp_transfer_atomic()
333 	err = tegra_bpmp_channel_write(channel, msg->mrq, MSG_ACK,  in tegra_bpmp_transfer_atomic()
346 	err = tegra_bpmp_wait_response(channel);  in tegra_bpmp_transfer_atomic()
350 	return __tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,  in tegra_bpmp_transfer_atomic()
358 	struct tegra_bpmp_channel *channel;  in tegra_bpmp_transfer()  local
368 	channel = tegra_bpmp_write_threaded(bpmp, msg->mrq, msg->tx.data,  in tegra_bpmp_transfer()
370 	if (IS_ERR(channel))  in tegra_bpmp_transfer()
371 		return PTR_ERR(channel);  in tegra_bpmp_transfer()
379 	err = wait_for_completion_timeout(&channel->completion, timeout);  in tegra_bpmp_transfer()
383 	return tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,  in tegra_bpmp_transfer()
400 void tegra_bpmp_mrq_return(struct tegra_bpmp_channel *channel, int code,  in tegra_bpmp_mrq_return()  argument
403 	unsigned long flags = channel->ib->flags;  in tegra_bpmp_mrq_return()
404 	struct tegra_bpmp *bpmp = channel->bpmp;  in tegra_bpmp_mrq_return()
410 	err = tegra_bpmp_ack_request(channel);  in tegra_bpmp_mrq_return()
417 	if (WARN_ON(!tegra_bpmp_is_response_channel_free(channel)))  in tegra_bpmp_mrq_return()
420 	channel->ob->code = code;  in tegra_bpmp_mrq_return()
423 		memcpy(channel->ob->data, data, size);  in tegra_bpmp_mrq_return()
425 	err = tegra_bpmp_post_response(channel);  in tegra_bpmp_mrq_return()
439 				  struct tegra_bpmp_channel *channel)  in tegra_bpmp_handle_mrq()  argument
449 		tegra_bpmp_mrq_return(channel, -EINVAL, &zero, sizeof(zero));  in tegra_bpmp_handle_mrq()
453 	entry->handler(mrq, channel, entry->data);  in tegra_bpmp_handle_mrq()
529 				       struct tegra_bpmp_channel *channel,  in tegra_bpmp_mrq_handle_ping()  argument
535 	request = (struct mrq_ping_request *)channel->ib->data;  in tegra_bpmp_mrq_handle_ping()
540 	tegra_bpmp_mrq_return(channel, 0, &response, sizeof(response));  in tegra_bpmp_mrq_handle_ping()
649 static void tegra_bpmp_channel_signal(struct tegra_bpmp_channel *channel)  in tegra_bpmp_channel_signal()  argument
651 	unsigned long flags = channel->ob->flags;  in tegra_bpmp_channel_signal()
656 	complete(&channel->completion);  in tegra_bpmp_channel_signal()
661 	struct tegra_bpmp_channel *channel;  in tegra_bpmp_handle_rx()  local
665 	channel = bpmp->rx_channel;  in tegra_bpmp_handle_rx()
669 	if (tegra_bpmp_is_request_ready(channel))  in tegra_bpmp_handle_rx()
670 		tegra_bpmp_handle_mrq(bpmp, channel->ib->code, channel);  in tegra_bpmp_handle_rx()
675 		struct tegra_bpmp_channel *channel;  in tegra_bpmp_handle_rx()  local
677 		channel = &bpmp->threaded_channels[i];  in tegra_bpmp_handle_rx()
679 		if (tegra_bpmp_is_response_ready(channel)) {  in tegra_bpmp_handle_rx()
680 			tegra_bpmp_channel_signal(channel);  in tegra_bpmp_handle_rx()