drm/xe/xe_guc_relay.c

39 static struct xe_guc *relay_to_guc(struct xe_guc_relay *relay)
41 	return container_of(relay, struct xe_guc, relay);
44 static struct xe_guc_ct *relay_to_ct(struct xe_guc_relay *relay)
46 	return &relay_to_guc(relay)->ct;
49 static struct xe_gt *relay_to_gt(struct xe_guc_relay *relay)
51 	return guc_to_gt(relay_to_guc(relay));
54 static struct xe_device *relay_to_xe(struct xe_guc_relay *relay)
56 	return gt_to_xe(relay_to_gt(relay));
59 #define relay_assert(relay, condition)	xe_gt_assert(relay_to_gt(relay), condition)
60 #define relay_notice(relay, msg...)	xe_gt_sriov_notice(relay_to_gt(relay), "relay: " msg)
61 #define relay_debug(relay, msg...)	xe_gt_sriov_dbg_verbose(relay_to_gt(relay), "relay: " msg)
63 static int relay_get_totalvfs(struct xe_guc_relay *relay)
65 	struct xe_device *xe = relay_to_xe(relay);
68 	KUNIT_STATIC_STUB_REDIRECT(relay_get_totalvfs, relay);
72 static bool relay_is_ready(struct xe_guc_relay *relay)
74 	return mempool_initialized(&relay->pool);
77 static u32 relay_get_next_rid(struct xe_guc_relay *relay)
81 	spin_lock(&relay->lock);
82 	rid = ++relay->last_rid;
83 	spin_unlock(&relay->lock);
122 	 * @remote: PF/VF identifier of the origin (or target) of the relay
127 	/** @rid: identifier of the VF/PF relay message. */
156 	 * @offset: offset to the start of the inner VF/PF relay message inside
158 	 *          relay header message.
209 __relay_get_transaction(struct xe_guc_relay *relay, bool incoming, u32 remote, u32 rid,
214 	relay_assert(relay, action_len >= GUC_RELAY_MSG_MIN_LEN);
215 	relay_assert(relay, action_len <= GUC_RELAY_MSG_MAX_LEN);
216 	relay_assert(relay, !(!!resp ^ !!resp_size));
217 	relay_assert(relay, resp_size <= GUC_RELAY_MSG_MAX_LEN);
218 	relay_assert(relay, resp_size == 0 || resp_size >= GUC_RELAY_MSG_MIN_LEN);
220 	if (unlikely(!relay_is_ready(relay)))
228 	txn = mempool_alloc(&relay->pool, incoming ? GFP_ATOMIC : GFP_NOWAIT);
239 	relay_assert(relay, txn->offset);
240 	relay_assert(relay, txn->offset + GUC_RELAY_MSG_MAX_LEN <= ARRAY_SIZE(txn->request_buf));
241 	relay_assert(relay, txn->offset + GUC_RELAY_MSG_MAX_LEN <= ARRAY_SIZE(txn->response_buf));
257 relay_new_transaction(struct xe_guc_relay *relay, u32 target, const u32 *action, u32 len,
260 	u32 rid = relay_get_next_rid(relay);
262 	return __relay_get_transaction(relay, false, target, rid, action, len, resp, resp_size);
266 relay_new_incoming_transaction(struct xe_guc_relay *relay, u32 origin, u32 rid,
269 	return __relay_get_transaction(relay, true, origin, rid, action, len, NULL, 0);
272 static void relay_release_transaction(struct xe_guc_relay *relay, struct relay_transaction *txn)
274 	relay_assert(relay, list_empty(&txn->link));
279 	mempool_free(txn, &relay->pool);
282 static int relay_send_transaction(struct xe_guc_relay *relay, struct relay_transaction *txn)
289 	relay_assert(relay, txn->offset);
290 	relay_assert(relay, txn->offset + len <= GUC_CTB_MAX_DWORDS);
291 	relay_assert(relay, len >= GUC_RELAY_MSG_MIN_LEN);
292 	relay_assert(relay, len <= GUC_RELAY_MSG_MAX_LEN);
294 	relay_debug(relay, "sending %s.%u to %u = %*ph\n",
298 	ret = xe_guc_ct_send_block(relay_to_ct(relay), buf, len + txn->offset);
301 		relay_notice(relay, "Unexpected data=%d from GuC, wrong ABI?\n", ret);
305 		relay_notice(relay, "Failed to send %s.%x to GuC (%pe) %*ph ...\n",
309 		relay_notice(relay, "Failed to send %s.%u to %u (%pe) %*ph\n",
319 	struct xe_guc_relay *relay = arg;
321 	mempool_exit(&relay->pool);
326  * @relay: the &xe_guc_relay to initialize
333 int xe_guc_relay_init(struct xe_guc_relay *relay)
336 	struct xe_device *xe = relay_to_xe(relay);
339 	relay_assert(relay, !relay_is_ready(relay));
344 	spin_lock_init(&relay->lock);
345 	INIT_WORK(&relay->worker, relays_worker_fn);
346 	INIT_LIST_HEAD(&relay->pending_relays);
347 	INIT_LIST_HEAD(&relay->incoming_actions);
349 	err = mempool_init_kmalloc_pool(&relay->pool, XE_RELAY_MEMPOOL_MIN_NUM +
350 					relay_get_totalvfs(relay),
355 	relay_debug(relay, "using mempool with %d elements\n", relay->pool.min_nr);
357 	return drmm_add_action_or_reset(&xe->drm, __fini_relay, relay);
363 	/* XXX: assume that relay errors match errno codes */
369 	/* XXX: assume that relay errors match errno codes */
402 static void relay_testonly_nop(struct xe_guc_relay *relay)
404 	KUNIT_STATIC_STUB_REDIRECT(relay_testonly_nop, relay);
407 static int relay_send_message_and_wait(struct xe_guc_relay *relay,
422 	relay_debug(relay, "%s.%u to %u action %#x:%u\n",
427 	spin_lock(&relay->lock);
428 	list_add_tail(&txn->link, &relay->pending_relays);
429 	spin_unlock(&relay->lock);
432 	ret = relay_send_transaction(relay, txn);
443 	relay_debug(relay, "%u.%u reply %d after %u msec\n",
450 			relay_testonly_nop(relay);
460 	relay_debug(relay, "%u.%u response %*ph\n", txn->remote, txn->rid,
462 	relay_assert(relay, txn->response_len >= GUC_RELAY_MSG_MIN_LEN);
466 	spin_lock(&relay->lock);
468 	spin_unlock(&relay->lock);
471 		relay_notice(relay, "Unsuccessful %s.%u %#x:%u to %u (%pe) %*ph\n",
480 static int relay_send_to(struct xe_guc_relay *relay, u32 target,
486 	relay_assert(relay, len >= GUC_RELAY_MSG_MIN_LEN);
487 	relay_assert(relay, len <= GUC_RELAY_MSG_MAX_LEN);
488 	relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) == GUC_HXG_ORIGIN_HOST);
489 	relay_assert(relay, guc_hxg_type_is_action(FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0])));
491 	if (unlikely(!relay_is_ready(relay)))
494 	txn = relay_new_transaction(relay, target, msg, len, buf, buf_size);
500 		ret = relay_send_message_and_wait(relay, txn, buf, buf_size);
503 		relay_assert(relay, !GUC_HXG_TYPE_FAST_REQUEST);
506 		ret = relay_send_transaction(relay, txn);
513 	relay_release_transaction(relay, txn);
520  * @relay: the &xe_guc_relay which will send the message
532 int xe_guc_relay_send_to_vf(struct xe_guc_relay *relay, u32 target,
535 	relay_assert(relay, IS_SRIOV_PF(relay_to_xe(relay)));
537 	return relay_send_to(relay, target, msg, len, buf, buf_size);
543  * @relay: the &xe_guc_relay which will send the message
554 int xe_guc_relay_send_to_pf(struct xe_guc_relay *relay,
557 	relay_assert(relay, IS_SRIOV_VF(relay_to_xe(relay)));
559 	return relay_send_to(relay, PFID, msg, len, buf, buf_size);
562 static int relay_handle_reply(struct xe_guc_relay *relay, u32 origin,
568 	spin_lock(&relay->lock);
569 	list_for_each_entry(pending, &relay->pending_relays, link) {
571 			relay_debug(relay, "%u.%u still awaits response\n",
589 	spin_unlock(&relay->lock);
594 static int relay_handle_failure(struct xe_guc_relay *relay, u32 origin,
600 	relay_assert(relay, len);
601 	relay_debug(relay, "%u.%u error %#x (%pe) hint %u debug %*ph\n",
604 	return relay_handle_reply(relay, origin, rid, error ?: -EREMOTEIO, NULL, 0);
607 static int relay_testloop_action_handler(struct xe_guc_relay *relay, u32 origin,
618 	relay_assert(relay, guc_hxg_type_is_action(type));
619 	relay_assert(relay, action == GUC_RELAY_ACTION_VFXPF_TESTLOOP);
663 	relay_notice(relay, "Unexpected action %#x opcode %#x\n", action, opcode);
667 static int relay_action_handler(struct xe_guc_relay *relay, u32 origin,
670 	struct xe_gt *gt = relay_to_gt(relay);
674 	relay_assert(relay, len >= GUC_HXG_MSG_MIN_LEN);
677 		return relay_testloop_action_handler(relay, origin, msg, len, response, size);
681 	if (IS_SRIOV_PF(relay_to_xe(relay)))
687 		relay_assert(relay, ret <= 0);
692 static struct relay_transaction *relay_dequeue_transaction(struct xe_guc_relay *relay)
696 	spin_lock(&relay->lock);
697 	txn = list_first_entry_or_null(&relay->incoming_actions, struct relay_transaction, link);
700 	spin_unlock(&relay->lock);
705 static void relay_process_incoming_action(struct xe_guc_relay *relay)
712 	txn = relay_dequeue_transaction(relay);
718 	ret = relay_action_handler(relay, txn->remote,
730 		ret = relay_send_transaction(relay, txn);
736 		relay_notice(relay, "Failed to handle %s.%u from %u (%pe) %*ph\n",
745 		ret = relay_send_transaction(relay, txn);
750 		spin_lock(&relay->lock);
751 		list_add(&txn->link, &relay->incoming_actions);
752 		spin_unlock(&relay->lock);
757 		relay_notice(relay, "Failed to process action.%u (%pe) %*ph\n",
761 	relay_release_transaction(relay, txn);
764 static bool relay_needs_worker(struct xe_guc_relay *relay)
768 	spin_lock(&relay->lock);
769 	is_empty = list_empty(&relay->incoming_actions);
770 	spin_unlock(&relay->lock);
776 static void relay_kick_worker(struct xe_guc_relay *relay)
778 	KUNIT_STATIC_STUB_REDIRECT(relay_kick_worker, relay);
779 	queue_work(relay_to_xe(relay)->sriov.wq, &relay->worker);
784 	struct xe_guc_relay *relay = container_of(w, struct xe_guc_relay, worker);
786 	relay_process_incoming_action(relay);
788 	if (relay_needs_worker(relay))
789 		relay_kick_worker(relay);
792 static int relay_queue_action_msg(struct xe_guc_relay *relay, u32 origin, u32 rid,
797 	txn = relay_new_incoming_transaction(relay, origin, rid, msg, len);
801 	spin_lock(&relay->lock);
802 	list_add_tail(&txn->link, &relay->incoming_actions);
803 	spin_unlock(&relay->lock);
805 	relay_kick_worker(relay);
809 static int relay_process_msg(struct xe_guc_relay *relay, u32 origin, u32 rid,
822 	relay_debug(relay, "received %s.%u from %u = %*ph\n",
829 		err = relay_queue_action_msg(relay, origin, rid, msg, len);
832 		err = relay_handle_reply(relay, origin, rid, 0, msg, len);
835 		err = relay_handle_reply(relay, origin, rid, -EBUSY, NULL, 0);
838 		err = relay_handle_reply(relay, origin, rid, -EAGAIN, NULL, 0);
841 		err = relay_handle_failure(relay, origin, rid, msg, len);
848 		relay_notice(relay, "Failed to process %s.%u from %u (%pe) %*ph\n",
856  * xe_guc_relay_process_guc2vf - Handle relay notification message from the GuC.
857  * @relay: the &xe_guc_relay which will handle the message
861  * This function will handle relay messages received from the GuC.
867 int xe_guc_relay_process_guc2vf(struct xe_guc_relay *relay, const u32 *msg, u32 len)
871 	relay_assert(relay, len >= GUC_HXG_MSG_MIN_LEN);
872 	relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) == GUC_HXG_ORIGIN_GUC);
873 	relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]) == GUC_HXG_TYPE_EVENT);
874 	relay_assert(relay, FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[0]) ==
877 	if (unlikely(!IS_SRIOV_VF(relay_to_xe(relay)) && !kunit_get_current_test()))
880 	if (unlikely(!relay_is_ready(relay)))
894 	return relay_process_msg(relay, PFID, rid,
901  * xe_guc_relay_process_guc2pf - Handle relay notification message from the GuC.
902  * @relay: the &xe_guc_relay which will handle the message
906  * This function will handle relay messages received from the GuC.
912 int xe_guc_relay_process_guc2pf(struct xe_guc_relay *relay, const u32 *msg, u32 len)
917 	relay_assert(relay, len >= GUC_HXG_EVENT_MSG_MIN_LEN);
918 	relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) == GUC_HXG_ORIGIN_GUC);
919 	relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]) == GUC_HXG_TYPE_EVENT);
920 	relay_assert(relay, FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[0]) ==
923 	if (unlikely(!IS_SRIOV_PF(relay_to_xe(relay)) && !kunit_get_current_test()))
926 	if (unlikely(!relay_is_ready(relay)))
941 	if (unlikely(origin > relay_get_totalvfs(relay)))
944 	err = relay_process_msg(relay, origin, rid,