Lines Matching +full:rpc +full:- +full:if
50 #include <nvrm/535.113.01/nvidia/generated/g_rpc-structures.h>
69 …* https://github.com/NVIDIA/open-gpu-kernel-modules/blob/535/src/nvidia/inc/kernel/gpu/gsp/message…
73 * RPC via the GSP command queue, GSP writes the status of the submitted
74 * RPC in the status queue.
78 * - message element header (struct r535_gsp_msg), which mostly maintains
81 * - RPC message header (struct nvfw_gsp_rpc), which maintains the info
82 * of the RPC. E.g., the RPC function number.
84 * - The payload, where the RPC message stays. E.g. the params of a
85 * specific RPC function. Some RPC functions also have their headers
90 * +------------------------+
96 * |----------V-------------|
97 * | GSP RPC Header |
102 * |----------V-------------|
107 * +------------------------+
122 * - gsp_msg(msg): GSP message element (element header + GSP RPC header +
124 * - gsp_rpc(rpc): GSP RPC (RPC header + payload)
125 * - gsp_rpc_buf: buffer for (GSP RPC header + payload)
126 * - gsp_rpc_len: size of (GSP RPC header + payload)
127 * - params_size: size of params in the payload
128 * - payload_size: size of (header if exists + params) in the payload
170 return -EBUSY; in r535_rpc_status_to_errno()
172 return -ENOMEM; in r535_rpc_status_to_errno()
174 return -EINVAL; in r535_rpc_status_to_errno()
181 u32 size, rptr = *gsp->msgq.rptr; in r535_gsp_msgq_wait()
186 if (WARN_ON(!size || size >= gsp->msgq.cnt)) in r535_gsp_msgq_wait()
187 return -EINVAL; in r535_gsp_msgq_wait()
190 u32 wptr = *gsp->msgq.wptr; in r535_gsp_msgq_wait()
192 used = wptr + gsp->msgq.cnt - rptr; in r535_gsp_msgq_wait()
193 if (used >= gsp->msgq.cnt) in r535_gsp_msgq_wait()
194 used -= gsp->msgq.cnt; in r535_gsp_msgq_wait()
195 if (used >= size) in r535_gsp_msgq_wait()
199 } while (--(*ptime)); in r535_gsp_msgq_wait()
201 if (WARN_ON(!*ptime)) in r535_gsp_msgq_wait()
202 return -ETIMEDOUT; in r535_gsp_msgq_wait()
210 u32 rptr = *gsp->msgq.rptr; in r535_gsp_msgq_get_entry()
213 return (void *)((u8 *)gsp->shm.msgq.ptr + GSP_PAGE_SIZE + in r535_gsp_msgq_get_entry()
223 * - Peek the element from the queue: r535_gsp_msgq_peek().
227 * - Allocate memory for the message.
232 * - Receive the message: r535_gsp_msgq_recv().
234 * If the message is a large GSP message, r535_gsp_msgq_recv() calls
240 * - Free the allocated memory: r535_gsp_msg_done().
251 if (ret < 0) in r535_gsp_msgq_peek()
256 return mqe->data; in r535_gsp_msgq_peek()
273 u8 *buf = info->gsp_rpc_buf; in r535_gsp_msgq_recv_one_elem()
274 u32 rptr = *gsp->msgq.rptr; in r535_gsp_msgq_recv_one_elem()
279 expected = info->gsp_rpc_len; in r535_gsp_msgq_recv_one_elem()
281 ret = r535_gsp_msgq_wait(gsp, expected, info->retries); in r535_gsp_msgq_recv_one_elem()
282 if (ret < 0) in r535_gsp_msgq_recv_one_elem()
287 if (info->continuation) { in r535_gsp_msgq_recv_one_elem()
288 struct nvfw_gsp_rpc *rpc = (struct nvfw_gsp_rpc *)mqe->data; in r535_gsp_msgq_recv_one_elem() local
290 if (rpc->function != NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD) { in r535_gsp_msgq_recv_one_elem()
291 nvkm_error(&gsp->subdev, in r535_gsp_msgq_recv_one_elem()
292 "Not a continuation of a large RPC\n"); in r535_gsp_msgq_recv_one_elem()
293 r535_gsp_msg_dump(gsp, rpc, NV_DBG_ERROR); in r535_gsp_msgq_recv_one_elem()
294 return ERR_PTR(-EIO); in r535_gsp_msgq_recv_one_elem()
300 len = ((gsp->msgq.cnt - rptr) * GSP_PAGE_SIZE) - sizeof(*mqe); in r535_gsp_msgq_recv_one_elem()
303 if (info->continuation) in r535_gsp_msgq_recv_one_elem()
304 memcpy(buf, mqe->data + sizeof(struct nvfw_gsp_rpc), in r535_gsp_msgq_recv_one_elem()
305 len - sizeof(struct nvfw_gsp_rpc)); in r535_gsp_msgq_recv_one_elem()
307 memcpy(buf, mqe->data, len); in r535_gsp_msgq_recv_one_elem()
309 expected -= len; in r535_gsp_msgq_recv_one_elem()
311 if (expected) { in r535_gsp_msgq_recv_one_elem()
312 mqe = (void *)((u8 *)gsp->shm.msgq.ptr + 0x1000 + 0 * 0x1000); in r535_gsp_msgq_recv_one_elem()
316 rptr = (rptr + DIV_ROUND_UP(size, GSP_PAGE_SIZE)) % gsp->msgq.cnt; in r535_gsp_msgq_recv_one_elem()
319 (*gsp->msgq.rptr) = rptr; in r535_gsp_msgq_recv_one_elem()
327 const u32 max_rpc_size = GSP_MSG_MAX_SIZE - sizeof(*mqe); in r535_gsp_msgq_recv()
328 struct nvfw_gsp_rpc *rpc; in r535_gsp_msgq_recv() local
334 rpc = (struct nvfw_gsp_rpc *)mqe->data; in r535_gsp_msgq_recv()
336 if (WARN_ON(rpc->length > max_rpc_size)) in r535_gsp_msgq_recv()
339 buf = kvmalloc(max_t(u32, rpc->length, expected), GFP_KERNEL); in r535_gsp_msgq_recv()
340 if (!buf) in r535_gsp_msgq_recv()
341 return ERR_PTR(-ENOMEM); in r535_gsp_msgq_recv()
345 info.gsp_rpc_len = rpc->length; in r535_gsp_msgq_recv()
348 if (IS_ERR(buf)) { in r535_gsp_msgq_recv()
354 if (expected <= max_rpc_size) in r535_gsp_msgq_recv()
358 expected -= info.gsp_rpc_len; in r535_gsp_msgq_recv()
363 rpc = r535_gsp_msgq_peek(gsp, sizeof(*rpc), info.retries); in r535_gsp_msgq_recv()
364 if (IS_ERR_OR_NULL(rpc)) { in r535_gsp_msgq_recv()
366 return rpc; in r535_gsp_msgq_recv()
369 info.gsp_rpc_len = rpc->length; in r535_gsp_msgq_recv()
372 rpc = r535_gsp_msgq_recv_one_elem(gsp, &info); in r535_gsp_msgq_recv()
373 if (IS_ERR_OR_NULL(rpc)) { in r535_gsp_msgq_recv()
375 return rpc; in r535_gsp_msgq_recv()
378 size = info.gsp_rpc_len - sizeof(*rpc); in r535_gsp_msgq_recv()
379 expected -= size; in r535_gsp_msgq_recv()
383 rpc = buf; in r535_gsp_msgq_recv()
384 rpc->length = gsp_rpc_len; in r535_gsp_msgq_recv()
389 r535_gsp_cmdq_push(struct nvkm_gsp *gsp, void *rpc) in r535_gsp_cmdq_push() argument
391 struct r535_gsp_msg *msg = to_gsp_hdr(rpc, msg); in r535_gsp_cmdq_push()
393 u32 gsp_rpc_len = msg->checksum; in r535_gsp_cmdq_push()
404 msg->pad = 0; in r535_gsp_cmdq_push()
405 msg->checksum = 0; in r535_gsp_cmdq_push()
406 msg->sequence = gsp->cmdq.seq++; in r535_gsp_cmdq_push()
407 msg->elem_count = DIV_ROUND_UP(len, 0x1000); in r535_gsp_cmdq_push()
412 msg->checksum = upper_32_bits(csum) ^ lower_32_bits(csum); in r535_gsp_cmdq_push()
414 wptr = *gsp->cmdq.wptr; in r535_gsp_cmdq_push()
417 free = *gsp->cmdq.rptr + gsp->cmdq.cnt - wptr - 1; in r535_gsp_cmdq_push()
418 if (free >= gsp->cmdq.cnt) in r535_gsp_cmdq_push()
419 free -= gsp->cmdq.cnt; in r535_gsp_cmdq_push()
420 if (free >= 1) in r535_gsp_cmdq_push()
424 } while(--time); in r535_gsp_cmdq_push()
426 if (WARN_ON(!time)) { in r535_gsp_cmdq_push()
428 return -ETIMEDOUT; in r535_gsp_cmdq_push()
431 cqe = (void *)((u8 *)gsp->shm.cmdq.ptr + 0x1000 + wptr * 0x1000); in r535_gsp_cmdq_push()
432 step = min_t(u32, free, (gsp->cmdq.cnt - wptr)); in r535_gsp_cmdq_push()
438 if (wptr == gsp->cmdq.cnt) in r535_gsp_cmdq_push()
442 len -= size; in r535_gsp_cmdq_push()
445 nvkm_trace(&gsp->subdev, "cmdq: wptr %d\n", wptr); in r535_gsp_cmdq_push()
447 (*gsp->cmdq.wptr) = wptr; in r535_gsp_cmdq_push()
450 nvkm_falcon_wr32(&gsp->falcon, 0xc00, 0x00000000); in r535_gsp_cmdq_push()
464 if (!msg) in r535_gsp_cmdq_get()
465 return ERR_PTR(-ENOMEM); in r535_gsp_cmdq_get()
467 msg->checksum = gsp_rpc_len; in r535_gsp_cmdq_get()
468 return msg->data; in r535_gsp_cmdq_get()
480 if (gsp->subdev.debug >= lvl) { in r535_gsp_msg_dump()
481 nvkm_printk__(&gsp->subdev, lvl, info, in r535_gsp_msg_dump()
483 msg->function, msg->length, msg->length - sizeof(*msg), in r535_gsp_msg_dump()
484 msg->rpc_result, msg->rpc_result_private); in r535_gsp_msg_dump()
486 msg->data, msg->length - sizeof(*msg), true); in r535_gsp_msg_dump()
493 struct nvkm_subdev *subdev = &gsp->subdev; in r535_gsp_msg_recv()
494 struct nvfw_gsp_rpc *rpc; in r535_gsp_msg_recv() local
498 rpc = r535_gsp_msgq_peek(gsp, sizeof(*rpc), &retries); in r535_gsp_msg_recv()
499 if (IS_ERR_OR_NULL(rpc)) in r535_gsp_msg_recv()
500 return rpc; in r535_gsp_msg_recv()
502 rpc = r535_gsp_msgq_recv(gsp, gsp_rpc_len, &retries); in r535_gsp_msg_recv()
503 if (IS_ERR_OR_NULL(rpc)) in r535_gsp_msg_recv()
504 return rpc; in r535_gsp_msg_recv()
506 if (rpc->rpc_result) { in r535_gsp_msg_recv()
507 r535_gsp_msg_dump(gsp, rpc, NV_DBG_ERROR); in r535_gsp_msg_recv()
508 r535_gsp_msg_done(gsp, rpc); in r535_gsp_msg_recv()
509 return ERR_PTR(-EINVAL); in r535_gsp_msg_recv()
512 r535_gsp_msg_dump(gsp, rpc, NV_DBG_TRACE); in r535_gsp_msg_recv()
514 if (fn && rpc->function == fn) { in r535_gsp_msg_recv()
515 if (gsp_rpc_len) { in r535_gsp_msg_recv()
516 if (rpc->length < gsp_rpc_len) { in r535_gsp_msg_recv()
517 nvkm_error(subdev, "rpc len %d < %d\n", in r535_gsp_msg_recv()
518 rpc->length, gsp_rpc_len); in r535_gsp_msg_recv()
519 r535_gsp_msg_dump(gsp, rpc, NV_DBG_ERROR); in r535_gsp_msg_recv()
520 r535_gsp_msg_done(gsp, rpc); in r535_gsp_msg_recv()
521 return ERR_PTR(-EIO); in r535_gsp_msg_recv()
524 return rpc; in r535_gsp_msg_recv()
527 r535_gsp_msg_done(gsp, rpc); in r535_gsp_msg_recv()
531 for (i = 0; i < gsp->msgq.ntfy_nr; i++) { in r535_gsp_msg_recv()
532 struct nvkm_gsp_msgq_ntfy *ntfy = &gsp->msgq.ntfy[i]; in r535_gsp_msg_recv()
534 if (ntfy->fn == rpc->function) { in r535_gsp_msg_recv()
535 if (ntfy->func) in r535_gsp_msg_recv()
536 ntfy->func(ntfy->priv, ntfy->fn, rpc->data, in r535_gsp_msg_recv()
537 rpc->length - sizeof(*rpc)); in r535_gsp_msg_recv()
542 if (i == gsp->msgq.ntfy_nr) in r535_gsp_msg_recv()
543 r535_gsp_msg_dump(gsp, rpc, NV_DBG_WARN); in r535_gsp_msg_recv()
545 r535_gsp_msg_done(gsp, rpc); in r535_gsp_msg_recv()
546 if (fn) in r535_gsp_msg_recv()
549 if (*gsp->msgq.rptr != *gsp->msgq.wptr) in r535_gsp_msg_recv()
560 mutex_lock(&gsp->msgq.mutex); in r535_gsp_msg_ntfy_add()
561 if (WARN_ON(gsp->msgq.ntfy_nr >= ARRAY_SIZE(gsp->msgq.ntfy))) { in r535_gsp_msg_ntfy_add()
562 ret = -ENOSPC; in r535_gsp_msg_ntfy_add()
564 gsp->msgq.ntfy[gsp->msgq.ntfy_nr].fn = fn; in r535_gsp_msg_ntfy_add()
565 gsp->msgq.ntfy[gsp->msgq.ntfy_nr].func = func; in r535_gsp_msg_ntfy_add()
566 gsp->msgq.ntfy[gsp->msgq.ntfy_nr].priv = priv; in r535_gsp_msg_ntfy_add()
567 gsp->msgq.ntfy_nr++; in r535_gsp_msg_ntfy_add()
569 mutex_unlock(&gsp->msgq.mutex); in r535_gsp_msg_ntfy_add()
578 mutex_lock(&gsp->cmdq.mutex); in r535_gsp_rpc_poll()
580 mutex_unlock(&gsp->cmdq.mutex); in r535_gsp_rpc_poll()
581 if (IS_ERR(repv)) in r535_gsp_rpc_poll()
591 struct nvfw_gsp_rpc *rpc = to_gsp_hdr(payload, rpc); in r535_gsp_rpc_send() local
593 u32 fn = rpc->function; in r535_gsp_rpc_send()
597 if (gsp->subdev.debug >= NV_DBG_TRACE) { in r535_gsp_rpc_send()
598 nvkm_trace(&gsp->subdev, "rpc fn:%d len:0x%x/0x%zx\n", rpc->function, in r535_gsp_rpc_send()
599 rpc->length, rpc->length - sizeof(*rpc)); in r535_gsp_rpc_send()
600 print_hex_dump(KERN_INFO, "rpc: ", DUMP_PREFIX_OFFSET, 16, 1, in r535_gsp_rpc_send()
601 rpc->data, rpc->length - sizeof(*rpc), true); in r535_gsp_rpc_send()
604 ret = r535_gsp_cmdq_push(gsp, rpc); in r535_gsp_rpc_send()
605 if (ret) in r535_gsp_rpc_send()
608 if (wait) { in r535_gsp_rpc_send()
610 if (!IS_ERR_OR_NULL(msg)) in r535_gsp_rpc_send()
611 repv = msg->data; in r535_gsp_rpc_send()
622 struct nvkm_gsp_device *device = event->device; in r535_gsp_event_dtor()
623 struct nvkm_gsp_client *client = device->object.client; in r535_gsp_event_dtor()
624 struct nvkm_gsp *gsp = client->gsp; in r535_gsp_event_dtor()
626 mutex_lock(&gsp->client_id.mutex); in r535_gsp_event_dtor()
627 if (event->func) { in r535_gsp_event_dtor()
628 list_del(&event->head); in r535_gsp_event_dtor()
629 event->func = NULL; in r535_gsp_event_dtor()
631 mutex_unlock(&gsp->client_id.mutex); in r535_gsp_event_dtor()
633 nvkm_gsp_rm_free(&event->object); in r535_gsp_event_dtor()
634 event->device = NULL; in r535_gsp_event_dtor()
640 struct nvkm_gsp_device *device = event->device; in r535_gsp_device_event_get()
643 ctrl = nvkm_gsp_rm_ctrl_get(&device->subdevice, in r535_gsp_device_event_get()
645 if (IS_ERR(ctrl)) in r535_gsp_device_event_get()
648 ctrl->event = event->id; in r535_gsp_device_event_get()
649 ctrl->action = NV2080_CTRL_EVENT_SET_NOTIFICATION_ACTION_REPEAT; in r535_gsp_device_event_get()
650 return nvkm_gsp_rm_ctrl_wr(&device->subdevice, ctrl); in r535_gsp_device_event_get()
657 struct nvkm_gsp_client *client = device->object.client; in r535_gsp_device_event_ctor()
658 struct nvkm_gsp *gsp = client->gsp; in r535_gsp_device_event_ctor()
662 args = nvkm_gsp_rm_alloc_get(&device->subdevice, handle, in r535_gsp_device_event_ctor()
664 &event->object); in r535_gsp_device_event_ctor()
665 if (IS_ERR(args)) in r535_gsp_device_event_ctor()
668 args->hParentClient = client->object.handle; in r535_gsp_device_event_ctor()
669 args->hSrcResource = 0; in r535_gsp_device_event_ctor()
670 args->hClass = NV01_EVENT_KERNEL_CALLBACK_EX; in r535_gsp_device_event_ctor()
671 args->notifyIndex = NV01_EVENT_CLIENT_RM | id; in r535_gsp_device_event_ctor()
672 args->data = NULL; in r535_gsp_device_event_ctor()
674 ret = nvkm_gsp_rm_alloc_wr(&event->object, args); in r535_gsp_device_event_ctor()
675 if (ret) in r535_gsp_device_event_ctor()
678 event->device = device; in r535_gsp_device_event_ctor()
679 event->id = id; in r535_gsp_device_event_ctor()
682 if (ret) { in r535_gsp_device_event_ctor()
687 mutex_lock(&gsp->client_id.mutex); in r535_gsp_device_event_ctor()
688 event->func = func; in r535_gsp_device_event_ctor()
689 list_add(&event->head, &client->events); in r535_gsp_device_event_ctor()
690 mutex_unlock(&gsp->client_id.mutex); in r535_gsp_device_event_ctor()
697 nvkm_gsp_rm_free(&device->subdevice); in r535_gsp_device_dtor()
698 nvkm_gsp_rm_free(&device->object); in r535_gsp_device_dtor()
706 return nvkm_gsp_rm_alloc(&device->object, 0x5d1d0000, NV20_SUBDEVICE_0, sizeof(*args), in r535_gsp_subdevice_ctor()
707 &device->subdevice); in r535_gsp_subdevice_ctor()
716 args = nvkm_gsp_rm_alloc_get(&client->object, 0xde1d0000, NV01_DEVICE_0, sizeof(*args), in r535_gsp_device_ctor()
717 &device->object); in r535_gsp_device_ctor()
718 if (IS_ERR(args)) in r535_gsp_device_ctor()
721 args->hClientShare = client->object.handle; in r535_gsp_device_ctor()
723 ret = nvkm_gsp_rm_alloc_wr(&device->object, args); in r535_gsp_device_ctor()
724 if (ret) in r535_gsp_device_ctor()
728 if (ret) in r535_gsp_device_ctor()
729 nvkm_gsp_rm_free(&device->object); in r535_gsp_device_ctor()
737 struct nvkm_gsp *gsp = client->gsp; in r535_gsp_client_dtor()
739 nvkm_gsp_rm_free(&client->object); in r535_gsp_client_dtor()
741 mutex_lock(&gsp->client_id.mutex); in r535_gsp_client_dtor()
742 idr_remove(&gsp->client_id.idr, client->object.handle & 0xffff); in r535_gsp_client_dtor()
743 mutex_unlock(&gsp->client_id.mutex); in r535_gsp_client_dtor()
745 client->gsp = NULL; in r535_gsp_client_dtor()
754 mutex_lock(&gsp->client_id.mutex); in r535_gsp_client_ctor()
755 ret = idr_alloc(&gsp->client_id.idr, client, 0, 0xffff + 1, GFP_KERNEL); in r535_gsp_client_ctor()
756 mutex_unlock(&gsp->client_id.mutex); in r535_gsp_client_ctor()
757 if (ret < 0) in r535_gsp_client_ctor()
760 client->gsp = gsp; in r535_gsp_client_ctor()
761 client->object.client = client; in r535_gsp_client_ctor()
762 INIT_LIST_HEAD(&client->events); in r535_gsp_client_ctor()
764 args = nvkm_gsp_rm_alloc_get(&client->object, 0xc1d00000 | ret, NV01_ROOT, sizeof(*args), in r535_gsp_client_ctor()
765 &client->object); in r535_gsp_client_ctor()
766 if (IS_ERR(args)) { in r535_gsp_client_ctor()
771 args->hClient = client->object.handle; in r535_gsp_client_ctor()
772 args->processID = ~0; in r535_gsp_client_ctor()
774 ret = nvkm_gsp_rm_alloc_wr(&client->object, args); in r535_gsp_client_ctor()
775 if (ret) { in r535_gsp_client_ctor()
786 struct nvkm_gsp_client *client = object->client; in r535_gsp_rpc_rm_free()
787 struct nvkm_gsp *gsp = client->gsp; in r535_gsp_rpc_rm_free()
788 rpc_free_v03_00 *rpc; in r535_gsp_rpc_rm_free() local
790 nvkm_debug(&gsp->subdev, "cli:0x%08x obj:0x%08x free\n", in r535_gsp_rpc_rm_free()
791 client->object.handle, object->handle); in r535_gsp_rpc_rm_free()
793 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_FREE, sizeof(*rpc)); in r535_gsp_rpc_rm_free()
794 if (WARN_ON(IS_ERR_OR_NULL(rpc))) in r535_gsp_rpc_rm_free()
795 return -EIO; in r535_gsp_rpc_rm_free()
797 rpc->params.hRoot = client->object.handle; in r535_gsp_rpc_rm_free()
798 rpc->params.hObjectParent = 0; in r535_gsp_rpc_rm_free()
799 rpc->params.hObjectOld = object->handle; in r535_gsp_rpc_rm_free()
800 return nvkm_gsp_rpc_wr(gsp, rpc, true); in r535_gsp_rpc_rm_free()
806 rpc_gsp_rm_alloc_v03_00 *rpc = to_payload_hdr(params, rpc); in r535_gsp_rpc_rm_alloc_done() local
808 nvkm_gsp_rpc_done(object->client->gsp, rpc); in r535_gsp_rpc_rm_alloc_done()
814 rpc_gsp_rm_alloc_v03_00 *rpc = to_payload_hdr(params, rpc); in r535_gsp_rpc_rm_alloc_push() local
815 struct nvkm_gsp *gsp = object->client->gsp; in r535_gsp_rpc_rm_alloc_push()
818 rpc = nvkm_gsp_rpc_push(gsp, rpc, true, sizeof(*rpc)); in r535_gsp_rpc_rm_alloc_push()
819 if (IS_ERR_OR_NULL(rpc)) in r535_gsp_rpc_rm_alloc_push()
820 return rpc; in r535_gsp_rpc_rm_alloc_push()
822 if (rpc->status) { in r535_gsp_rpc_rm_alloc_push()
823 ret = ERR_PTR(r535_rpc_status_to_errno(rpc->status)); in r535_gsp_rpc_rm_alloc_push()
824 if (PTR_ERR(ret) != -EAGAIN && PTR_ERR(ret) != -EBUSY) in r535_gsp_rpc_rm_alloc_push()
825 nvkm_error(&gsp->subdev, "RM_ALLOC: 0x%x\n", rpc->status); in r535_gsp_rpc_rm_alloc_push()
828 nvkm_gsp_rpc_done(gsp, rpc); in r535_gsp_rpc_rm_alloc_push()
837 struct nvkm_gsp_client *client = object->client; in r535_gsp_rpc_rm_alloc_get()
838 struct nvkm_gsp *gsp = client->gsp; in r535_gsp_rpc_rm_alloc_get()
839 rpc_gsp_rm_alloc_v03_00 *rpc; in r535_gsp_rpc_rm_alloc_get() local
841 nvkm_debug(&gsp->subdev, "cli:0x%08x obj:0x%08x new obj:0x%08x\n", in r535_gsp_rpc_rm_alloc_get()
842 client->object.handle, object->parent->handle, in r535_gsp_rpc_rm_alloc_get()
843 object->handle); in r535_gsp_rpc_rm_alloc_get()
845 nvkm_debug(&gsp->subdev, "cls:0x%08x params_size:%d\n", oclass, in r535_gsp_rpc_rm_alloc_get()
848 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_GSP_RM_ALLOC, in r535_gsp_rpc_rm_alloc_get()
849 sizeof(*rpc) + params_size); in r535_gsp_rpc_rm_alloc_get()
850 if (IS_ERR(rpc)) in r535_gsp_rpc_rm_alloc_get()
851 return rpc; in r535_gsp_rpc_rm_alloc_get()
853 rpc->hClient = client->object.handle; in r535_gsp_rpc_rm_alloc_get()
854 rpc->hParent = object->parent->handle; in r535_gsp_rpc_rm_alloc_get()
855 rpc->hObject = object->handle; in r535_gsp_rpc_rm_alloc_get()
856 rpc->hClass = oclass; in r535_gsp_rpc_rm_alloc_get()
857 rpc->status = 0; in r535_gsp_rpc_rm_alloc_get()
858 rpc->paramsSize = params_size; in r535_gsp_rpc_rm_alloc_get()
859 return rpc->params; in r535_gsp_rpc_rm_alloc_get()
865 rpc_gsp_rm_control_v03_00 *rpc = to_payload_hdr(params, rpc); in r535_gsp_rpc_rm_ctrl_done() local
867 if (!params) in r535_gsp_rpc_rm_ctrl_done()
869 nvkm_gsp_rpc_done(object->client->gsp, rpc); in r535_gsp_rpc_rm_ctrl_done()
875 rpc_gsp_rm_control_v03_00 *rpc = to_payload_hdr((*params), rpc); in r535_gsp_rpc_rm_ctrl_push() local
876 struct nvkm_gsp *gsp = object->client->gsp; in r535_gsp_rpc_rm_ctrl_push()
879 rpc = nvkm_gsp_rpc_push(gsp, rpc, true, repc); in r535_gsp_rpc_rm_ctrl_push()
880 if (IS_ERR_OR_NULL(rpc)) { in r535_gsp_rpc_rm_ctrl_push()
882 return PTR_ERR(rpc); in r535_gsp_rpc_rm_ctrl_push()
885 if (rpc->status) { in r535_gsp_rpc_rm_ctrl_push()
886 ret = r535_rpc_status_to_errno(rpc->status); in r535_gsp_rpc_rm_ctrl_push()
887 if (ret != -EAGAIN && ret != -EBUSY) in r535_gsp_rpc_rm_ctrl_push()
888 nvkm_error(&gsp->subdev, "cli:0x%08x obj:0x%08x ctrl cmd:0x%08x failed: 0x%08x\n", in r535_gsp_rpc_rm_ctrl_push()
889 object->client->object.handle, object->handle, rpc->cmd, rpc->status); in r535_gsp_rpc_rm_ctrl_push()
892 if (repc) in r535_gsp_rpc_rm_ctrl_push()
893 *params = rpc->params; in r535_gsp_rpc_rm_ctrl_push()
895 nvkm_gsp_rpc_done(gsp, rpc); in r535_gsp_rpc_rm_ctrl_push()
903 struct nvkm_gsp_client *client = object->client; in r535_gsp_rpc_rm_ctrl_get()
904 struct nvkm_gsp *gsp = client->gsp; in r535_gsp_rpc_rm_ctrl_get()
905 rpc_gsp_rm_control_v03_00 *rpc; in r535_gsp_rpc_rm_ctrl_get() local
907 nvkm_debug(&gsp->subdev, "cli:0x%08x obj:0x%08x ctrl cmd:0x%08x params_size:%d\n", in r535_gsp_rpc_rm_ctrl_get()
908 client->object.handle, object->handle, cmd, params_size); in r535_gsp_rpc_rm_ctrl_get()
910 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL, in r535_gsp_rpc_rm_ctrl_get()
911 sizeof(*rpc) + params_size); in r535_gsp_rpc_rm_ctrl_get()
912 if (IS_ERR(rpc)) in r535_gsp_rpc_rm_ctrl_get()
913 return rpc; in r535_gsp_rpc_rm_ctrl_get()
915 rpc->hClient = client->object.handle; in r535_gsp_rpc_rm_ctrl_get()
916 rpc->hObject = object->handle; in r535_gsp_rpc_rm_ctrl_get()
917 rpc->cmd = cmd; in r535_gsp_rpc_rm_ctrl_get()
918 rpc->status = 0; in r535_gsp_rpc_rm_ctrl_get()
919 rpc->paramsSize = params_size; in r535_gsp_rpc_rm_ctrl_get()
920 return rpc->params; in r535_gsp_rpc_rm_ctrl_get()
926 struct nvfw_gsp_rpc *rpc = container_of(repv, typeof(*rpc), data); in r535_gsp_rpc_done() local
928 r535_gsp_msg_done(gsp, rpc); in r535_gsp_rpc_done()
934 struct nvfw_gsp_rpc *rpc; in r535_gsp_rpc_get() local
936 rpc = r535_gsp_cmdq_get(gsp, ALIGN(sizeof(*rpc) + payload_size, in r535_gsp_rpc_get()
938 if (IS_ERR(rpc)) in r535_gsp_rpc_get()
939 return ERR_CAST(rpc); in r535_gsp_rpc_get()
941 rpc->header_version = 0x03000000; in r535_gsp_rpc_get()
942 rpc->signature = ('C' << 24) | ('P' << 16) | ('R' << 8) | 'V'; in r535_gsp_rpc_get()
943 rpc->function = fn; in r535_gsp_rpc_get()
944 rpc->rpc_result = 0xffffffff; in r535_gsp_rpc_get()
945 rpc->rpc_result_private = 0xffffffff; in r535_gsp_rpc_get()
946 rpc->length = sizeof(*rpc) + payload_size; in r535_gsp_rpc_get()
947 return rpc->data; in r535_gsp_rpc_get()
954 struct nvfw_gsp_rpc *rpc = to_gsp_hdr(payload, rpc); in r535_gsp_rpc_push() local
955 struct r535_gsp_msg *msg = to_gsp_hdr(rpc, msg); in r535_gsp_rpc_push()
956 const u32 max_rpc_size = GSP_MSG_MAX_SIZE - sizeof(*msg); in r535_gsp_rpc_push()
957 const u32 max_payload_size = max_rpc_size - sizeof(*rpc); in r535_gsp_rpc_push()
958 u32 payload_size = rpc->length - sizeof(*rpc); in r535_gsp_rpc_push()
961 mutex_lock(&gsp->cmdq.mutex); in r535_gsp_rpc_push()
962 if (payload_size > max_payload_size) { in r535_gsp_rpc_push()
963 const u32 fn = rpc->function; in r535_gsp_rpc_push()
966 /* Adjust length, and send initial RPC. */ in r535_gsp_rpc_push()
967 rpc->length = sizeof(*rpc) + max_payload_size; in r535_gsp_rpc_push()
968 msg->checksum = rpc->length; in r535_gsp_rpc_push()
971 if (IS_ERR(repv)) in r535_gsp_rpc_push()
975 remain_payload_size -= max_payload_size; in r535_gsp_rpc_push()
984 if (IS_ERR(next)) { in r535_gsp_rpc_push()
992 if (IS_ERR(repv)) in r535_gsp_rpc_push()
996 remain_payload_size -= size; in r535_gsp_rpc_push()
1000 rpc = r535_gsp_msg_recv(gsp, fn, payload_size + in r535_gsp_rpc_push()
1001 sizeof(*rpc)); in r535_gsp_rpc_push()
1002 if (!IS_ERR_OR_NULL(rpc)) { in r535_gsp_rpc_push()
1003 if (wait) { in r535_gsp_rpc_push()
1004 repv = rpc->data; in r535_gsp_rpc_push()
1006 nvkm_gsp_rpc_done(gsp, rpc); in r535_gsp_rpc_push()
1010 repv = wait ? rpc : NULL; in r535_gsp_rpc_push()
1017 mutex_unlock(&gsp->cmdq.mutex); in r535_gsp_rpc_push()
1052 mutex_lock(&gsp->cmdq.mutex); in r535_gsp_msgq_work()
1053 if (*gsp->msgq.rptr != *gsp->msgq.wptr) in r535_gsp_msgq_work()
1055 mutex_unlock(&gsp->cmdq.mutex); in r535_gsp_msgq_work()
1062 struct nvkm_subdev *subdev = &gsp->subdev; in r535_gsp_intr()
1063 u32 intr = nvkm_falcon_rd32(&gsp->falcon, 0x0008); in r535_gsp_intr()
1064 u32 inte = nvkm_falcon_rd32(&gsp->falcon, gsp->falcon.func->addr2 + in r535_gsp_intr()
1065 gsp->falcon.func->riscv_irqmask); in r535_gsp_intr()
1068 if (!stat) { in r535_gsp_intr()
1073 if (stat & 0x00000040) { in r535_gsp_intr()
1074 nvkm_falcon_wr32(&gsp->falcon, 0x004, 0x00000040); in r535_gsp_intr()
1075 schedule_work(&gsp->msgq.work); in r535_gsp_intr()
1079 if (stat) { in r535_gsp_intr()
1081 nvkm_falcon_wr32(&gsp->falcon, 0x014, stat); in r535_gsp_intr()
1082 nvkm_falcon_wr32(&gsp->falcon, 0x004, stat); in r535_gsp_intr()
1085 nvkm_falcon_intr_retrigger(&gsp->falcon); in r535_gsp_intr()
1095 ctrl = nvkm_gsp_rm_ctrl_get(&gsp->internal.device.subdevice, in r535_gsp_intr_get_table()
1097 if (IS_ERR(ctrl)) in r535_gsp_intr_get_table()
1100 ret = nvkm_gsp_rm_ctrl_push(&gsp->internal.device.subdevice, &ctrl, sizeof(*ctrl)); in r535_gsp_intr_get_table()
1101 if (WARN_ON(ret)) { in r535_gsp_intr_get_table()
1102 nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl); in r535_gsp_intr_get_table()
1106 for (unsigned i = 0; i < ctrl->tableLen; i++) { in r535_gsp_intr_get_table()
1110 nvkm_debug(&gsp->subdev, in r535_gsp_intr_get_table()
1112 ctrl->table[i].engineIdx, ctrl->table[i].pmcIntrMask, in r535_gsp_intr_get_table()
1113 ctrl->table[i].vectorStall, ctrl->table[i].vectorNonStall); in r535_gsp_intr_get_table()
1115 switch (ctrl->table[i].engineIdx) { in r535_gsp_intr_get_table()
1126 inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_CE0; in r535_gsp_intr_get_table()
1134 inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_NVDEC0; in r535_gsp_intr_get_table()
1138 inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_MSENC; in r535_gsp_intr_get_table()
1142 inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_NVJPEG0; in r535_gsp_intr_get_table()
1152 if (WARN_ON(gsp->intr_nr == ARRAY_SIZE(gsp->intr))) { in r535_gsp_intr_get_table()
1153 ret = -ENOSPC; in r535_gsp_intr_get_table()
1157 gsp->intr[gsp->intr_nr].type = type; in r535_gsp_intr_get_table()
1158 gsp->intr[gsp->intr_nr].inst = inst; in r535_gsp_intr_get_table()
1159 gsp->intr[gsp->intr_nr].stall = ctrl->table[i].vectorStall; in r535_gsp_intr_get_table()
1160 gsp->intr[gsp->intr_nr].nonstall = ctrl->table[i].vectorNonStall; in r535_gsp_intr_get_table()
1161 gsp->intr_nr++; in r535_gsp_intr_get_table()
1164 nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl); in r535_gsp_intr_get_table()
1171 GspStaticConfigInfo *rpc; in r535_gsp_rpc_get_gsp_static_info() local
1172 int last_usable = -1; in r535_gsp_rpc_get_gsp_static_info()
1174 rpc = nvkm_gsp_rpc_rd(gsp, NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO, sizeof(*rpc)); in r535_gsp_rpc_get_gsp_static_info()
1175 if (IS_ERR(rpc)) in r535_gsp_rpc_get_gsp_static_info()
1176 return PTR_ERR(rpc); in r535_gsp_rpc_get_gsp_static_info()
1178 gsp->internal.client.object.client = &gsp->internal.client; in r535_gsp_rpc_get_gsp_static_info()
1179 gsp->internal.client.object.parent = NULL; in r535_gsp_rpc_get_gsp_static_info()
1180 gsp->internal.client.object.handle = rpc->hInternalClient; in r535_gsp_rpc_get_gsp_static_info()
1181 gsp->internal.client.gsp = gsp; in r535_gsp_rpc_get_gsp_static_info()
1183 gsp->internal.device.object.client = &gsp->internal.client; in r535_gsp_rpc_get_gsp_static_info()
1184 gsp->internal.device.object.parent = &gsp->internal.client.object; in r535_gsp_rpc_get_gsp_static_info()
1185 gsp->internal.device.object.handle = rpc->hInternalDevice; in r535_gsp_rpc_get_gsp_static_info()
1187 gsp->internal.device.subdevice.client = &gsp->internal.client; in r535_gsp_rpc_get_gsp_static_info()
1188 gsp->internal.device.subdevice.parent = &gsp->internal.device.object; in r535_gsp_rpc_get_gsp_static_info()
1189 gsp->internal.device.subdevice.handle = rpc->hInternalSubdevice; in r535_gsp_rpc_get_gsp_static_info()
1191 gsp->bar.rm_bar1_pdb = rpc->bar1PdeBase; in r535_gsp_rpc_get_gsp_static_info()
1192 gsp->bar.rm_bar2_pdb = rpc->bar2PdeBase; in r535_gsp_rpc_get_gsp_static_info()
1194 for (int i = 0; i < rpc->fbRegionInfoParams.numFBRegions; i++) { in r535_gsp_rpc_get_gsp_static_info()
1196 &rpc->fbRegionInfoParams.fbRegion[i]; in r535_gsp_rpc_get_gsp_static_info()
1198 nvkm_debug(&gsp->subdev, "fb region %d: " in r535_gsp_rpc_get_gsp_static_info()
1199 "%016llx-%016llx rsvd:%016llx perf:%08x comp:%d iso:%d prot:%d\n", i, in r535_gsp_rpc_get_gsp_static_info()
1200 reg->base, reg->limit, reg->reserved, reg->performance, in r535_gsp_rpc_get_gsp_static_info()
1201 reg->supportCompressed, reg->supportISO, reg->bProtected); in r535_gsp_rpc_get_gsp_static_info()
1203 if (!reg->reserved && !reg->bProtected) { in r535_gsp_rpc_get_gsp_static_info()
1204 if (reg->supportCompressed && reg->supportISO && in r535_gsp_rpc_get_gsp_static_info()
1205 !WARN_ON_ONCE(gsp->fb.region_nr >= ARRAY_SIZE(gsp->fb.region))) { in r535_gsp_rpc_get_gsp_static_info()
1206 const u64 size = (reg->limit + 1) - reg->base; in r535_gsp_rpc_get_gsp_static_info()
1208 gsp->fb.region[gsp->fb.region_nr].addr = reg->base; in r535_gsp_rpc_get_gsp_static_info()
1209 gsp->fb.region[gsp->fb.region_nr].size = size; in r535_gsp_rpc_get_gsp_static_info()
1210 gsp->fb.region_nr++; in r535_gsp_rpc_get_gsp_static_info()
1217 if (last_usable >= 0) { in r535_gsp_rpc_get_gsp_static_info()
1218 u32 rsvd_base = rpc->fbRegionInfoParams.fbRegion[last_usable].limit + 1; in r535_gsp_rpc_get_gsp_static_info()
1220 gsp->fb.rsvd_size = gsp->fb.heap.addr - rsvd_base; in r535_gsp_rpc_get_gsp_static_info()
1223 for (int gpc = 0; gpc < ARRAY_SIZE(rpc->tpcInfo); gpc++) { in r535_gsp_rpc_get_gsp_static_info()
1224 if (rpc->gpcInfo.gpcMask & BIT(gpc)) { in r535_gsp_rpc_get_gsp_static_info()
1225 gsp->gr.tpcs += hweight32(rpc->tpcInfo[gpc].tpcMask); in r535_gsp_rpc_get_gsp_static_info()
1226 gsp->gr.gpcs++; in r535_gsp_rpc_get_gsp_static_info()
1230 nvkm_gsp_rpc_done(gsp, rpc); in r535_gsp_rpc_get_gsp_static_info()
1237 if (mem->data) { in nvkm_gsp_mem_dtor()
1240 * GSP-RM if the buffer was prematurely freed. in nvkm_gsp_mem_dtor()
1242 memset(mem->data, 0xFF, mem->size); in nvkm_gsp_mem_dtor()
1244 dma_free_coherent(mem->dev, mem->size, mem->data, mem->addr); in nvkm_gsp_mem_dtor()
1245 put_device(mem->dev); in nvkm_gsp_mem_dtor()
1252 * nvkm_gsp_mem_ctor - constructor for nvkm_gsp_mem objects
1259 * This memory block can potentially out-live the driver's remove() callback,
1266 mem->data = dma_alloc_coherent(gsp->subdev.device->dev, size, &mem->addr, GFP_KERNEL); in nvkm_gsp_mem_ctor()
1267 if (WARN_ON(!mem->data)) in nvkm_gsp_mem_ctor()
1268 return -ENOMEM; in nvkm_gsp_mem_ctor()
1270 mem->size = size; in nvkm_gsp_mem_ctor()
1271 mem->dev = get_device(gsp->subdev.device->dev); in nvkm_gsp_mem_ctor()
1279 struct nvkm_device *device = gsp->subdev.device; in r535_gsp_postinit()
1283 if (WARN_ON(ret)) in r535_gsp_postinit()
1286 INIT_WORK(&gsp->msgq.work, r535_gsp_msgq_work); in r535_gsp_postinit()
1289 if (WARN_ON(ret)) in r535_gsp_postinit()
1292 ret = nvkm_gsp_intr_stall(gsp, gsp->subdev.type, gsp->subdev.inst); in r535_gsp_postinit()
1293 if (WARN_ON(ret < 0)) in r535_gsp_postinit()
1296 ret = nvkm_inth_add(&device->vfn->intr, ret, NVKM_INTR_PRIO_NORMAL, &gsp->subdev, in r535_gsp_postinit()
1297 r535_gsp_intr, &gsp->subdev.inth); in r535_gsp_postinit()
1298 if (WARN_ON(ret)) in r535_gsp_postinit()
1301 nvkm_inth_allow(&gsp->subdev.inth); in r535_gsp_postinit()
1305 nvkm_gsp_mem_dtor(&gsp->boot.fw); in r535_gsp_postinit()
1306 nvkm_gsp_mem_dtor(&gsp->libos); in r535_gsp_postinit()
1314 rpc_unloading_guest_driver_v1F_07 *rpc; in r535_gsp_rpc_unloading_guest_driver() local
1316 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER, sizeof(*rpc)); in r535_gsp_rpc_unloading_guest_driver()
1317 if (IS_ERR(rpc)) in r535_gsp_rpc_unloading_guest_driver()
1318 return PTR_ERR(rpc); in r535_gsp_rpc_unloading_guest_driver()
1320 if (suspend) { in r535_gsp_rpc_unloading_guest_driver()
1321 rpc->bInPMTransition = 1; in r535_gsp_rpc_unloading_guest_driver()
1322 rpc->bGc6Entering = 0; in r535_gsp_rpc_unloading_guest_driver()
1323 rpc->newLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3; in r535_gsp_rpc_unloading_guest_driver()
1325 rpc->bInPMTransition = 0; in r535_gsp_rpc_unloading_guest_driver()
1326 rpc->bGc6Entering = 0; in r535_gsp_rpc_unloading_guest_driver()
1327 rpc->newLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_0; in r535_gsp_rpc_unloading_guest_driver()
1330 return nvkm_gsp_rpc_wr(gsp, rpc, true); in r535_gsp_rpc_unloading_guest_driver()
1334 REGISTRY_TABLE_ENTRY_TYPE_DWORD = 1, /* 32-bit unsigned integer */
1336 REGISTRY_TABLE_ENTRY_TYPE_STRING = 3, /* Null-terminated string */
1343 * struct registry_list_entry - linked list member for a registry key/value
1349 * @dword: the data, if REGISTRY_TABLE_ENTRY_TYPE_DWORD
1350 * @binary: the data, if TYPE_BINARY or TYPE_STRING
1354 * Type DWORD is a simple 32-bit unsigned integer, and its value is stored in
1357 * Types BINARY and STRING are variable-length binary blobs. The only real
1358 * difference between BINARY and STRING is that STRING is null-terminated and
1362 * but different types, but this is not useful since GSP-RM expects keys to
1376 * add_registry -- adds a registry entry
1387 * RPC data structure.
1390 * It's used to avoid an O(n) calculation of the size when the RPC is built.
1401 if (nlen > REGISTRY_MAX_KEY_LENGTH) in add_registry()
1402 return -EINVAL; in add_registry()
1407 if (!reg) in add_registry()
1408 return -ENOMEM; in add_registry()
1412 reg->dword = *(const u32 *)(data); in add_registry()
1416 memcpy(reg->binary, data, alloc_size); in add_registry()
1419 nvkm_error(&gsp->subdev, "unrecognized registry type %u for '%s'\n", in add_registry()
1422 return -EINVAL; in add_registry()
1425 memcpy(reg->key, key, nlen); in add_registry()
1426 reg->klen = nlen; in add_registry()
1427 reg->vlen = length; in add_registry()
1428 reg->type = type; in add_registry()
1430 list_add_tail(®->head, &gsp->registry_list); in add_registry()
1431 gsp->registry_rpc_size += sizeof(PACKED_REGISTRY_ENTRY) + nlen + alloc_size; in add_registry()
1449 * build_registry -- create the registry RPC data
1451 * @registry: pointer to the RPC payload to fill
1454 * build the RPC.
1456 * The registry RPC looks like this:
1458 * +-----------------+
1461 * +-----------------+
1462 * +----------------------------------------+
1464 * +----------------------------------------+
1465 * |Null-terminated key (string) for entry 0|
1466 * +----------------------------------------+
1467 * |Binary/string data value for entry 0 | (only if necessary)
1468 * +----------------------------------------+
1470 * +----------------------------------------+
1472 * +----------------------------------------+
1473 * |Null-terminated key (string) for entry 1|
1474 * +----------------------------------------+
1475 * |Binary/string data value for entry 1 | (only if necessary)
1476 * +----------------------------------------+
1480 * The 'data' field of an entry is either a 32-bit integer (for type DWORD)
1491 registry->numEntries = list_count_nodes(&gsp->registry_list); in build_registry()
1492 str_offset = struct_size(registry, entries, registry->numEntries); in build_registry()
1494 list_for_each_entry_safe(reg, n, &gsp->registry_list, head) { in build_registry()
1495 registry->entries[i].type = reg->type; in build_registry()
1496 registry->entries[i].length = reg->vlen; in build_registry()
1499 registry->entries[i].nameOffset = str_offset; in build_registry()
1500 memcpy((void *)registry + str_offset, reg->key, reg->klen); in build_registry()
1501 str_offset += reg->klen; in build_registry()
1503 switch (reg->type) { in build_registry()
1505 registry->entries[i].data = reg->dword; in build_registry()
1509 /* If the type is binary or string, also append the value */ in build_registry()
1510 memcpy((void *)registry + str_offset, reg->binary, reg->vlen); in build_registry()
1511 registry->entries[i].data = str_offset; in build_registry()
1512 str_offset += reg->vlen; in build_registry()
1519 list_del(®->head); in build_registry()
1523 /* Double-check that we calculated the sizes correctly */ in build_registry()
1524 WARN_ON(gsp->registry_rpc_size != str_offset); in build_registry()
1526 registry->size = gsp->registry_rpc_size; in build_registry()
1530 * clean_registry -- clean up registry memory in case of error
1540 list_for_each_entry_safe(reg, n, &gsp->registry_list, head) { in clean_registry()
1541 list_del(®->head); in clean_registry()
1545 gsp->registry_rpc_size = sizeof(PACKED_REGISTRY_TABLE); in clean_registry()
1549 "A semicolon-separated list of key=integer pairs of GSP-RM registry keys");
1560 * r535_registry_entries - required registry entries for GSP-RM
1562 * This array lists registry entries that are required for GSP-RM to
1565 * RMSecBusResetEnable - enables PCI secondary bus reset
1566 * RMForcePcieConfigSave - forces GSP-RM to preserve PCI configuration
1576 * strip - strips all characters in 'reject' from 's'
1602 * r535_gsp_rpc_set_registry - build registry RPC and call GSP-RM
1605 * The GSP-RM registry is a set of key/value pairs that configure some aspects
1606 * of GSP-RM. The keys are strings, and the values are 32-bit integers.
1608 * The registry is built from a combination of a static hard-coded list (see
1614 PACKED_REGISTRY_TABLE *rpc; in r535_gsp_rpc_set_registry() local
1618 INIT_LIST_HEAD(&gsp->registry_list); in r535_gsp_rpc_set_registry()
1619 gsp->registry_rpc_size = sizeof(PACKED_REGISTRY_TABLE); in r535_gsp_rpc_set_registry()
1624 if (ret) in r535_gsp_rpc_set_registry()
1634 if (NVreg_RegistryDwords) { in r535_gsp_rpc_set_registry()
1638 if (!p) { in r535_gsp_rpc_set_registry()
1639 ret = -ENOMEM; in r535_gsp_rpc_set_registry()
1650 if (!equal || equal == start || equal[1] == 0) { in r535_gsp_rpc_set_registry()
1651 nvkm_error(&gsp->subdev, in r535_gsp_rpc_set_registry()
1661 if (!ret) { in r535_gsp_rpc_set_registry()
1668 if (ret) { in r535_gsp_rpc_set_registry()
1669 nvkm_error(&gsp->subdev, in r535_gsp_rpc_set_registry()
1679 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_SET_REGISTRY, gsp->registry_rpc_size); in r535_gsp_rpc_set_registry()
1680 if (IS_ERR(rpc)) { in r535_gsp_rpc_set_registry()
1681 ret = PTR_ERR(rpc); in r535_gsp_rpc_set_registry()
1685 build_registry(gsp, rpc); in r535_gsp_rpc_set_registry()
1687 return nvkm_gsp_rpc_wr(gsp, rpc, false); in r535_gsp_rpc_set_registry()
1694 #if defined(CONFIG_ACPI) && defined(CONFIG_X86)
1708 caps->status = 0xffff; in r535_gsp_acpi_caps()
1710 if (!acpi_check_dsm(handle, &NVOP_DSM_GUID, NVOP_DSM_REV, BIT_ULL(0x1a))) in r535_gsp_acpi_caps()
1714 if (!obj) in r535_gsp_acpi_caps()
1717 if (WARN_ON(obj->type != ACPI_TYPE_BUFFER) || in r535_gsp_acpi_caps()
1718 WARN_ON(obj->buffer.length != 4)) in r535_gsp_acpi_caps()
1721 caps->status = 0; in r535_gsp_acpi_caps()
1722 caps->optimusCaps = *(u32 *)obj->buffer.pointer; in r535_gsp_acpi_caps()
1743 jt->status = 0xffff; in r535_gsp_acpi_jt()
1746 if (!obj) in r535_gsp_acpi_jt()
1749 if (WARN_ON(obj->type != ACPI_TYPE_BUFFER) || in r535_gsp_acpi_jt()
1750 WARN_ON(obj->buffer.length != 4)) in r535_gsp_acpi_jt()
1753 jt->status = 0; in r535_gsp_acpi_jt()
1754 jt->jtCaps = *(u32 *)obj->buffer.pointer; in r535_gsp_acpi_jt()
1755 jt->jtRevId = (jt->jtCaps & 0xfff00000) >> 20; in r535_gsp_acpi_jt()
1756 jt->bSBIOSCaps = 0; in r535_gsp_acpi_jt()
1773 mode->status = 0xffff; in r535_gsp_acpi_mux_id()
1774 part->status = 0xffff; in r535_gsp_acpi_mux_id()
1778 if (ACPI_FAILURE(status) || !iter) in r535_gsp_acpi_mux_id()
1782 if (ACPI_FAILURE(status) || value != id) in r535_gsp_acpi_mux_id()
1788 if (!handle_mux) in r535_gsp_acpi_mux_id()
1791 /* I -think- 0 means "acquire" according to nvidia's driver source */ in r535_gsp_acpi_mux_id()
1792 input.pointer->integer.type = ACPI_TYPE_INTEGER; in r535_gsp_acpi_mux_id()
1793 input.pointer->integer.value = 0; in r535_gsp_acpi_mux_id()
1796 if (ACPI_SUCCESS(status)) { in r535_gsp_acpi_mux_id()
1797 mode->acpiId = id; in r535_gsp_acpi_mux_id()
1798 mode->mode = value; in r535_gsp_acpi_mux_id()
1799 mode->status = 0; in r535_gsp_acpi_mux_id()
1803 if (ACPI_SUCCESS(status)) { in r535_gsp_acpi_mux_id()
1804 part->acpiId = id; in r535_gsp_acpi_mux_id()
1805 part->mode = value; in r535_gsp_acpi_mux_id()
1806 part->status = 0; in r535_gsp_acpi_mux_id()
1813 mux->tableLen = dod->acpiIdListLen / sizeof(dod->acpiIdList[0]); in r535_gsp_acpi_mux()
1815 for (int i = 0; i < mux->tableLen; i++) { in r535_gsp_acpi_mux()
1816 r535_gsp_acpi_mux_id(handle, dod->acpiIdList[i], &mux->acpiIdMuxModeTable[i], in r535_gsp_acpi_mux()
1817 &mux->acpiIdMuxPartTable[i]); in r535_gsp_acpi_mux()
1828 dod->status = 0xffff; in r535_gsp_acpi_dod()
1831 if (ACPI_FAILURE(status)) in r535_gsp_acpi_dod()
1836 if (WARN_ON(_DOD->type != ACPI_TYPE_PACKAGE) || in r535_gsp_acpi_dod()
1837 WARN_ON(_DOD->package.count > ARRAY_SIZE(dod->acpiIdList))) in r535_gsp_acpi_dod()
1840 for (int i = 0; i < _DOD->package.count; i++) { in r535_gsp_acpi_dod()
1841 if (WARN_ON(_DOD->package.elements[i].type != ACPI_TYPE_INTEGER)) in r535_gsp_acpi_dod()
1844 dod->acpiIdList[i] = _DOD->package.elements[i].integer.value; in r535_gsp_acpi_dod()
1845 dod->acpiIdListLen += sizeof(dod->acpiIdList[0]); in r535_gsp_acpi_dod()
1848 dod->status = 0; in r535_gsp_acpi_dod()
1856 #if defined(CONFIG_ACPI) && defined(CONFIG_X86) in r535_gsp_acpi_info()
1857 acpi_handle handle = ACPI_HANDLE(gsp->subdev.device->dev); in r535_gsp_acpi_info()
1859 if (!handle) in r535_gsp_acpi_info()
1862 acpi->bValid = 1; in r535_gsp_acpi_info()
1864 r535_gsp_acpi_dod(handle, &acpi->dodMethodData); in r535_gsp_acpi_info()
1865 if (acpi->dodMethodData.status == 0) in r535_gsp_acpi_info()
1866 r535_gsp_acpi_mux(handle, &acpi->dodMethodData, &acpi->muxMethodData); in r535_gsp_acpi_info()
1868 r535_gsp_acpi_jt(handle, &acpi->jtMethodData); in r535_gsp_acpi_info()
1869 r535_gsp_acpi_caps(handle, &acpi->capsMethodData); in r535_gsp_acpi_info()
1876 struct nvkm_device *device = gsp->subdev.device; in r535_gsp_rpc_set_system_info()
1880 if (WARN_ON(device->type == NVKM_DEVICE_TEGRA)) in r535_gsp_rpc_set_system_info()
1881 return -ENOSYS; in r535_gsp_rpc_set_system_info()
1884 if (IS_ERR(info)) in r535_gsp_rpc_set_system_info()
1887 info->gpuPhysAddr = device->func->resource_addr(device, 0); in r535_gsp_rpc_set_system_info()
1888 info->gpuPhysFbAddr = device->func->resource_addr(device, 1); in r535_gsp_rpc_set_system_info()
1889 info->gpuPhysInstAddr = device->func->resource_addr(device, 3); in r535_gsp_rpc_set_system_info()
1890 info->nvDomainBusDeviceFunc = pci_dev_id(pdev->pdev); in r535_gsp_rpc_set_system_info()
1891 info->maxUserVa = TASK_SIZE; in r535_gsp_rpc_set_system_info()
1892 info->pciConfigMirrorBase = 0x088000; in r535_gsp_rpc_set_system_info()
1893 info->pciConfigMirrorSize = 0x001000; in r535_gsp_rpc_set_system_info()
1894 r535_gsp_acpi_info(gsp, &info->acpiMethodData); in r535_gsp_rpc_set_system_info()
1903 struct nvkm_subdev *subdev = &gsp->subdev; in r535_gsp_msg_os_error_log()
1906 if (WARN_ON(repc < sizeof(*msg))) in r535_gsp_msg_os_error_log()
1907 return -EINVAL; in r535_gsp_msg_os_error_log()
1909 nvkm_error(subdev, "Xid:%d %s\n", msg->exceptType, msg->errString); in r535_gsp_msg_os_error_log()
1918 struct nvkm_subdev *subdev = &gsp->subdev; in r535_gsp_msg_rc_triggered()
1922 if (WARN_ON(repc < sizeof(*msg))) in r535_gsp_msg_rc_triggered()
1923 return -EINVAL; in r535_gsp_msg_rc_triggered()
1926 msg->nv2080EngineType, msg->chid, msg->exceptType, msg->scope, in r535_gsp_msg_rc_triggered()
1927 msg->partitionAttributionId); in r535_gsp_msg_rc_triggered()
1929 chan = nvkm_chan_get_chid(&subdev->device->fifo->engine, msg->chid / 8, &flags); in r535_gsp_msg_rc_triggered()
1930 if (!chan) { in r535_gsp_msg_rc_triggered()
1931 nvkm_error(subdev, "rc chid:%d not found!\n", msg->chid); in r535_gsp_msg_rc_triggered()
1944 struct nvkm_subdev *subdev = &gsp->subdev; in r535_gsp_msg_mmu_fault_queued()
1957 struct nvkm_subdev *subdev = &gsp->subdev; in r535_gsp_msg_post_event()
1960 if (WARN_ON(repc < sizeof(*msg))) in r535_gsp_msg_post_event()
1961 return -EINVAL; in r535_gsp_msg_post_event()
1962 if (WARN_ON(repc != sizeof(*msg) + msg->eventDataSize)) in r535_gsp_msg_post_event()
1963 return -EINVAL; in r535_gsp_msg_post_event()
1966 msg->hClient, msg->hEvent, msg->notifyIndex, msg->data, in r535_gsp_msg_post_event()
1967 msg->status, msg->eventDataSize, msg->bNotifyList); in r535_gsp_msg_post_event()
1969 mutex_lock(&gsp->client_id.mutex); in r535_gsp_msg_post_event()
1970 client = idr_find(&gsp->client_id.idr, msg->hClient & 0xffff); in r535_gsp_msg_post_event()
1971 if (client) { in r535_gsp_msg_post_event()
1975 list_for_each_entry(event, &client->events, head) { in r535_gsp_msg_post_event()
1976 if (event->object.handle == msg->hEvent) { in r535_gsp_msg_post_event()
1977 event->func(event, msg->eventData, msg->eventDataSize); in r535_gsp_msg_post_event()
1982 if (!handled) { in r535_gsp_msg_post_event()
1984 msg->hClient, msg->hEvent); in r535_gsp_msg_post_event()
1987 nvkm_error(subdev, "event: cid 0x%08x not found!\n", msg->hClient); in r535_gsp_msg_post_event()
1989 mutex_unlock(&gsp->client_id.mutex); in r535_gsp_msg_post_event()
1994 * r535_gsp_msg_run_cpu_sequencer() -- process I/O commands from the GSP
1997 * @repv: pointer to libos print RPC
2002 * perform a special mid-initialization reset.
2008 struct nvkm_subdev *subdev = &gsp->subdev; in r535_gsp_msg_run_cpu_sequencer()
2009 struct nvkm_device *device = subdev->device; in r535_gsp_msg_run_cpu_sequencer()
2013 nvkm_debug(subdev, "seq: %08x %08x\n", seq->bufferSizeDWord, seq->cmdIndex); in r535_gsp_msg_run_cpu_sequencer()
2015 while (ptr < seq->cmdIndex) { in r535_gsp_msg_run_cpu_sequencer()
2016 GSP_SEQUENCER_BUFFER_CMD *cmd = (void *)&seq->commandBuffer[ptr]; in r535_gsp_msg_run_cpu_sequencer()
2019 ptr += GSP_SEQUENCER_PAYLOAD_SIZE_DWORDS(cmd->opCode); in r535_gsp_msg_run_cpu_sequencer()
2021 switch (cmd->opCode) { in r535_gsp_msg_run_cpu_sequencer()
2023 u32 addr = cmd->payload.regWrite.addr; in r535_gsp_msg_run_cpu_sequencer()
2024 u32 data = cmd->payload.regWrite.val; in r535_gsp_msg_run_cpu_sequencer()
2031 u32 addr = cmd->payload.regModify.addr; in r535_gsp_msg_run_cpu_sequencer()
2032 u32 mask = cmd->payload.regModify.mask; in r535_gsp_msg_run_cpu_sequencer()
2033 u32 data = cmd->payload.regModify.val; in r535_gsp_msg_run_cpu_sequencer()
2040 u32 addr = cmd->payload.regPoll.addr; in r535_gsp_msg_run_cpu_sequencer()
2041 u32 mask = cmd->payload.regPoll.mask; in r535_gsp_msg_run_cpu_sequencer()
2042 u32 data = cmd->payload.regPoll.val; in r535_gsp_msg_run_cpu_sequencer()
2043 u32 usec = cmd->payload.regPoll.timeout ?: 4000000; in r535_gsp_msg_run_cpu_sequencer()
2044 //u32 error = cmd->payload.regPoll.error; in r535_gsp_msg_run_cpu_sequencer()
2049 if ((nvkm_rd32(device, addr) & mask) == data) in r535_gsp_msg_run_cpu_sequencer()
2055 u32 usec = cmd->payload.delayUs.val; in r535_gsp_msg_run_cpu_sequencer()
2062 u32 addr = cmd->payload.regStore.addr; in r535_gsp_msg_run_cpu_sequencer()
2063 u32 slot = cmd->payload.regStore.index; in r535_gsp_msg_run_cpu_sequencer()
2065 seq->regSaveArea[slot] = nvkm_rd32(device, addr); in r535_gsp_msg_run_cpu_sequencer()
2066 nvkm_trace(subdev, "seq save %08x -> %d: %08x\n", addr, slot, in r535_gsp_msg_run_cpu_sequencer()
2067 seq->regSaveArea[slot]); in r535_gsp_msg_run_cpu_sequencer()
2072 nvkm_falcon_reset(&gsp->falcon); in r535_gsp_msg_run_cpu_sequencer()
2073 nvkm_falcon_mask(&gsp->falcon, 0x624, 0x00000080, 0x00000080); in r535_gsp_msg_run_cpu_sequencer()
2074 nvkm_falcon_wr32(&gsp->falcon, 0x10c, 0x00000000); in r535_gsp_msg_run_cpu_sequencer()
2078 if (nvkm_falcon_rd32(&gsp->falcon, 0x100) & 0x00000040) in r535_gsp_msg_run_cpu_sequencer()
2079 nvkm_falcon_wr32(&gsp->falcon, 0x130, 0x00000002); in r535_gsp_msg_run_cpu_sequencer()
2081 nvkm_falcon_wr32(&gsp->falcon, 0x100, 0x00000002); in r535_gsp_msg_run_cpu_sequencer()
2086 if (nvkm_falcon_rd32(&gsp->falcon, 0x100) & 0x00000010) in r535_gsp_msg_run_cpu_sequencer()
2091 struct nvkm_sec2 *sec2 = device->sec2; in r535_gsp_msg_run_cpu_sequencer()
2096 ret = gsp->func->reset(gsp); in r535_gsp_msg_run_cpu_sequencer()
2097 if (WARN_ON(ret)) in r535_gsp_msg_run_cpu_sequencer()
2100 nvkm_falcon_wr32(&gsp->falcon, 0x040, lower_32_bits(gsp->libos.addr)); in r535_gsp_msg_run_cpu_sequencer()
2101 nvkm_falcon_wr32(&gsp->falcon, 0x044, upper_32_bits(gsp->libos.addr)); in r535_gsp_msg_run_cpu_sequencer()
2103 nvkm_falcon_start(&sec2->falcon); in r535_gsp_msg_run_cpu_sequencer()
2105 if (nvkm_msec(device, 2000, in r535_gsp_msg_run_cpu_sequencer()
2106 if (nvkm_rd32(device, 0x1180f8) & 0x04000000) in r535_gsp_msg_run_cpu_sequencer()
2109 return -ETIMEDOUT; in r535_gsp_msg_run_cpu_sequencer()
2111 mbox0 = nvkm_falcon_rd32(&sec2->falcon, 0x040); in r535_gsp_msg_run_cpu_sequencer()
2112 if (WARN_ON(mbox0)) { in r535_gsp_msg_run_cpu_sequencer()
2113 nvkm_error(&gsp->subdev, "seq core resume sec2: 0x%x\n", mbox0); in r535_gsp_msg_run_cpu_sequencer()
2114 return -EIO; in r535_gsp_msg_run_cpu_sequencer()
2117 nvkm_falcon_wr32(&gsp->falcon, 0x080, gsp->boot.app_version); in r535_gsp_msg_run_cpu_sequencer()
2119 if (WARN_ON(!nvkm_falcon_riscv_active(&gsp->falcon))) in r535_gsp_msg_run_cpu_sequencer()
2120 return -EIO; in r535_gsp_msg_run_cpu_sequencer()
2124 nvkm_error(subdev, "unknown sequencer opcode %08x\n", cmd->opCode); in r535_gsp_msg_run_cpu_sequencer()
2125 return -EINVAL; in r535_gsp_msg_run_cpu_sequencer()
2135 struct nvkm_subdev *subdev = &gsp->subdev; in r535_gsp_booter_unload()
2136 struct nvkm_device *device = subdev->device; in r535_gsp_booter_unload()
2141 if (!wpr2_hi) { in r535_gsp_booter_unload()
2142 nvkm_debug(subdev, "WPR2 not set - skipping booter unload\n"); in r535_gsp_booter_unload()
2146 ret = nvkm_falcon_fw_boot(&gsp->booter.unload, &gsp->subdev, true, &mbox0, &mbox1, 0, 0); in r535_gsp_booter_unload()
2147 if (WARN_ON(ret)) in r535_gsp_booter_unload()
2151 if (WARN_ON(wpr2_hi)) in r535_gsp_booter_unload()
2152 return -EIO; in r535_gsp_booter_unload()
2162 ret = nvkm_falcon_fw_boot(&gsp->booter.load, &gsp->subdev, true, &mbox0, &mbox1, 0, 0); in r535_gsp_booter_load()
2163 if (ret) in r535_gsp_booter_load()
2166 nvkm_falcon_wr32(&gsp->falcon, 0x080, gsp->boot.app_version); in r535_gsp_booter_load()
2168 if (WARN_ON(!nvkm_falcon_riscv_active(&gsp->falcon))) in r535_gsp_booter_load()
2169 return -EIO; in r535_gsp_booter_load()
2180 ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->wpr_meta); in r535_gsp_wpr_meta_init()
2181 if (ret) in r535_gsp_wpr_meta_init()
2184 meta = gsp->wpr_meta.data; in r535_gsp_wpr_meta_init()
2186 meta->magic = GSP_FW_WPR_META_MAGIC; in r535_gsp_wpr_meta_init()
2187 meta->revision = GSP_FW_WPR_META_REVISION; in r535_gsp_wpr_meta_init()
2189 meta->sysmemAddrOfRadix3Elf = gsp->radix3.lvl0.addr; in r535_gsp_wpr_meta_init()
2190 meta->sizeOfRadix3Elf = gsp->fb.wpr2.elf.size; in r535_gsp_wpr_meta_init()
2192 meta->sysmemAddrOfBootloader = gsp->boot.fw.addr; in r535_gsp_wpr_meta_init()
2193 meta->sizeOfBootloader = gsp->boot.fw.size; in r535_gsp_wpr_meta_init()
2194 meta->bootloaderCodeOffset = gsp->boot.code_offset; in r535_gsp_wpr_meta_init()
2195 meta->bootloaderDataOffset = gsp->boot.data_offset; in r535_gsp_wpr_meta_init()
2196 meta->bootloaderManifestOffset = gsp->boot.manifest_offset; in r535_gsp_wpr_meta_init()
2198 meta->sysmemAddrOfSignature = gsp->sig.addr; in r535_gsp_wpr_meta_init()
2199 meta->sizeOfSignature = gsp->sig.size; in r535_gsp_wpr_meta_init()
2201 meta->gspFwRsvdStart = gsp->fb.heap.addr; in r535_gsp_wpr_meta_init()
2202 meta->nonWprHeapOffset = gsp->fb.heap.addr; in r535_gsp_wpr_meta_init()
2203 meta->nonWprHeapSize = gsp->fb.heap.size; in r535_gsp_wpr_meta_init()
2204 meta->gspFwWprStart = gsp->fb.wpr2.addr; in r535_gsp_wpr_meta_init()
2205 meta->gspFwHeapOffset = gsp->fb.wpr2.heap.addr; in r535_gsp_wpr_meta_init()
2206 meta->gspFwHeapSize = gsp->fb.wpr2.heap.size; in r535_gsp_wpr_meta_init()
2207 meta->gspFwOffset = gsp->fb.wpr2.elf.addr; in r535_gsp_wpr_meta_init()
2208 meta->bootBinOffset = gsp->fb.wpr2.boot.addr; in r535_gsp_wpr_meta_init()
2209 meta->frtsOffset = gsp->fb.wpr2.frts.addr; in r535_gsp_wpr_meta_init()
2210 meta->frtsSize = gsp->fb.wpr2.frts.size; in r535_gsp_wpr_meta_init()
2211 meta->gspFwWprEnd = ALIGN_DOWN(gsp->fb.bios.vga_workspace.addr, 0x20000); in r535_gsp_wpr_meta_init()
2212 meta->fbSize = gsp->fb.size; in r535_gsp_wpr_meta_init()
2213 meta->vgaWorkspaceOffset = gsp->fb.bios.vga_workspace.addr; in r535_gsp_wpr_meta_init()
2214 meta->vgaWorkspaceSize = gsp->fb.bios.vga_workspace.size; in r535_gsp_wpr_meta_init()
2215 meta->bootCount = 0; in r535_gsp_wpr_meta_init()
2216 meta->partitionRpcAddr = 0; in r535_gsp_wpr_meta_init()
2217 meta->partitionRpcRequestOffset = 0; in r535_gsp_wpr_meta_init()
2218 meta->partitionRpcReplyOffset = 0; in r535_gsp_wpr_meta_init()
2219 meta->verified = 0; in r535_gsp_wpr_meta_init()
2232 gsp->shm.cmdq.size = 0x40000; in r535_gsp_shared_init()
2233 gsp->shm.msgq.size = 0x40000; in r535_gsp_shared_init()
2235 gsp->shm.ptes.nr = (gsp->shm.cmdq.size + gsp->shm.msgq.size) >> GSP_PAGE_SHIFT; in r535_gsp_shared_init()
2236 gsp->shm.ptes.nr += DIV_ROUND_UP(gsp->shm.ptes.nr * sizeof(u64), GSP_PAGE_SIZE); in r535_gsp_shared_init()
2237 gsp->shm.ptes.size = ALIGN(gsp->shm.ptes.nr * sizeof(u64), GSP_PAGE_SIZE); in r535_gsp_shared_init()
2239 ret = nvkm_gsp_mem_ctor(gsp, gsp->shm.ptes.size + in r535_gsp_shared_init()
2240 gsp->shm.cmdq.size + in r535_gsp_shared_init()
2241 gsp->shm.msgq.size, in r535_gsp_shared_init()
2242 &gsp->shm.mem); in r535_gsp_shared_init()
2243 if (ret) in r535_gsp_shared_init()
2246 gsp->shm.ptes.ptr = gsp->shm.mem.data; in r535_gsp_shared_init()
2247 gsp->shm.cmdq.ptr = (u8 *)gsp->shm.ptes.ptr + gsp->shm.ptes.size; in r535_gsp_shared_init()
2248 gsp->shm.msgq.ptr = (u8 *)gsp->shm.cmdq.ptr + gsp->shm.cmdq.size; in r535_gsp_shared_init()
2250 for (i = 0; i < gsp->shm.ptes.nr; i++) in r535_gsp_shared_init()
2251 gsp->shm.ptes.ptr[i] = gsp->shm.mem.addr + (i << GSP_PAGE_SHIFT); in r535_gsp_shared_init()
2253 cmdq = gsp->shm.cmdq.ptr; in r535_gsp_shared_init()
2254 cmdq->tx.version = 0; in r535_gsp_shared_init()
2255 cmdq->tx.size = gsp->shm.cmdq.size; in r535_gsp_shared_init()
2256 cmdq->tx.entryOff = GSP_PAGE_SIZE; in r535_gsp_shared_init()
2257 cmdq->tx.msgSize = GSP_PAGE_SIZE; in r535_gsp_shared_init()
2258 cmdq->tx.msgCount = (cmdq->tx.size - cmdq->tx.entryOff) / cmdq->tx.msgSize; in r535_gsp_shared_init()
2259 cmdq->tx.writePtr = 0; in r535_gsp_shared_init()
2260 cmdq->tx.flags = 1; in r535_gsp_shared_init()
2261 cmdq->tx.rxHdrOff = offsetof(typeof(*cmdq), rx.readPtr); in r535_gsp_shared_init()
2263 msgq = gsp->shm.msgq.ptr; in r535_gsp_shared_init()
2265 gsp->cmdq.cnt = cmdq->tx.msgCount; in r535_gsp_shared_init()
2266 gsp->cmdq.wptr = &cmdq->tx.writePtr; in r535_gsp_shared_init()
2267 gsp->cmdq.rptr = &msgq->rx.readPtr; in r535_gsp_shared_init()
2268 gsp->msgq.cnt = cmdq->tx.msgCount; in r535_gsp_shared_init()
2269 gsp->msgq.wptr = &msgq->tx.writePtr; in r535_gsp_shared_init()
2270 gsp->msgq.rptr = &cmdq->rx.readPtr; in r535_gsp_shared_init()
2280 if (!resume) { in r535_gsp_rmargs_init()
2282 if (ret) in r535_gsp_rmargs_init()
2285 ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->rmargs); in r535_gsp_rmargs_init()
2286 if (ret) in r535_gsp_rmargs_init()
2290 args = gsp->rmargs.data; in r535_gsp_rmargs_init()
2291 args->messageQueueInitArguments.sharedMemPhysAddr = gsp->shm.mem.addr; in r535_gsp_rmargs_init()
2292 args->messageQueueInitArguments.pageTableEntryCount = gsp->shm.ptes.nr; in r535_gsp_rmargs_init()
2293 args->messageQueueInitArguments.cmdQueueOffset = in r535_gsp_rmargs_init()
2294 (u8 *)gsp->shm.cmdq.ptr - (u8 *)gsp->shm.mem.data; in r535_gsp_rmargs_init()
2295 args->messageQueueInitArguments.statQueueOffset = in r535_gsp_rmargs_init()
2296 (u8 *)gsp->shm.msgq.ptr - (u8 *)gsp->shm.mem.data; in r535_gsp_rmargs_init()
2298 if (!resume) { in r535_gsp_rmargs_init()
2299 args->srInitArguments.oldLevel = 0; in r535_gsp_rmargs_init()
2300 args->srInitArguments.flags = 0; in r535_gsp_rmargs_init()
2301 args->srInitArguments.bInPMTransition = 0; in r535_gsp_rmargs_init()
2303 args->srInitArguments.oldLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3; in r535_gsp_rmargs_init()
2304 args->srInitArguments.flags = 0; in r535_gsp_rmargs_init()
2305 args->srInitArguments.bInPMTransition = 1; in r535_gsp_rmargs_init()
2314 * If GSP-RM load fails, then the GSP nvkm object will be deleted, the logging
2323 "Migrate the GSP-RM logging debugfs entries upon exit");
2326 * GSP-RM uses a pseudo-class mechanism to define of a variety of per-"engine"
2328 * pre-processor. This is the class ID for the PMU.
2333 * struct rpc_ucode_libos_print_v1e_08 - RPC payload for libos print buffers
2348 * r535_gsp_msg_libos_print - capture log message from the PMU
2351 * @repv: pointer to libos print RPC
2354 * Called when we receive a UCODE_LIBOS_PRINT event RPC from GSP-RM. This RPC
2358 * Technically this RPC can be used to pass print buffers from any number of
2359 * GSP-RM engines, but we only expect to receive them for the PMU.
2361 * For the PMU, the buffer is 4K in size and the RPC always contains the full
2368 struct nvkm_subdev *subdev = &gsp->subdev; in r535_gsp_msg_libos_print()
2369 struct rpc_ucode_libos_print_v1e_08 *rpc = repv; in r535_gsp_msg_libos_print() local
2370 unsigned int class = rpc->ucode_eng_desc >> 8; in r535_gsp_msg_libos_print()
2373 class, rpc->libos_print_buf_size); in r535_gsp_msg_libos_print()
2375 if (class != NV_GSP_MSG_EVENT_UCODE_LIBOS_CLASS_PMU) { in r535_gsp_msg_libos_print()
2379 return -ENOMSG; in r535_gsp_msg_libos_print()
2382 if (rpc->libos_print_buf_size > GSP_PAGE_SIZE) { in r535_gsp_msg_libos_print()
2384 rpc->libos_print_buf_size); in r535_gsp_msg_libos_print()
2385 return -E2BIG; in r535_gsp_msg_libos_print()
2388 memcpy(gsp->blob_pmu.data, rpc->libos_print_buf, rpc->libos_print_buf_size); in r535_gsp_msg_libos_print()
2394 * create_debugfs - create a blob debugfs entry
2406 dent = debugfs_create_blob(name, 0444, gsp->debugfs.parent, blob); in create_debugfs()
2407 if (IS_ERR(dent)) { in create_debugfs()
2408 nvkm_error(&gsp->subdev, in create_debugfs()
2417 * [1] https://lore.kernel.org/r/linux-fsdevel/20240207200619.3354549-1-ttabi@nvidia.com/ in create_debugfs()
2419 i_size_write(d_inode(dent), blob->size); in create_debugfs()
2425 * r535_gsp_libos_debugfs_init - create logging debugfs entries
2432 * RPC sent from GSP-RM and must be only 4KB. We create it here because it's
2433 * only useful if there is a debugfs entry to expose it. If we get the PMU
2434 * logging RPC and there is no debugfs entry, the RPC is just ignored.
2440 * in real-time. We may add that capability in the future, but that
2446 * r535_gsp_booter_load() is called, so that if GSP-RM fails to initialize,
2452 struct device *dev = gsp->subdev.device->dev; in r535_gsp_libos_debugfs_init()
2455 gsp->debugfs.parent = debugfs_create_dir(dev_name(dev), nouveau_debugfs_root); in r535_gsp_libos_debugfs_init()
2456 if (IS_ERR(gsp->debugfs.parent)) { in r535_gsp_libos_debugfs_init()
2457 nvkm_error(&gsp->subdev, in r535_gsp_libos_debugfs_init()
2462 gsp->blob_init.data = gsp->loginit.data; in r535_gsp_libos_debugfs_init()
2463 gsp->blob_init.size = gsp->loginit.size; in r535_gsp_libos_debugfs_init()
2464 gsp->blob_intr.data = gsp->logintr.data; in r535_gsp_libos_debugfs_init()
2465 gsp->blob_intr.size = gsp->logintr.size; in r535_gsp_libos_debugfs_init()
2466 gsp->blob_rm.data = gsp->logrm.data; in r535_gsp_libos_debugfs_init()
2467 gsp->blob_rm.size = gsp->logrm.size; in r535_gsp_libos_debugfs_init()
2469 gsp->debugfs.init = create_debugfs(gsp, "loginit", &gsp->blob_init); in r535_gsp_libos_debugfs_init()
2470 if (!gsp->debugfs.init) in r535_gsp_libos_debugfs_init()
2473 gsp->debugfs.intr = create_debugfs(gsp, "logintr", &gsp->blob_intr); in r535_gsp_libos_debugfs_init()
2474 if (!gsp->debugfs.intr) in r535_gsp_libos_debugfs_init()
2477 gsp->debugfs.rm = create_debugfs(gsp, "logrm", &gsp->blob_rm); in r535_gsp_libos_debugfs_init()
2478 if (!gsp->debugfs.rm) in r535_gsp_libos_debugfs_init()
2482 * Since the PMU buffer is copied from an RPC, it doesn't need to be in r535_gsp_libos_debugfs_init()
2485 gsp->blob_pmu.size = GSP_PAGE_SIZE; in r535_gsp_libos_debugfs_init()
2486 gsp->blob_pmu.data = kzalloc(gsp->blob_pmu.size, GFP_KERNEL); in r535_gsp_libos_debugfs_init()
2487 if (!gsp->blob_pmu.data) in r535_gsp_libos_debugfs_init()
2490 gsp->debugfs.pmu = create_debugfs(gsp, "logpmu", &gsp->blob_pmu); in r535_gsp_libos_debugfs_init()
2491 if (!gsp->debugfs.pmu) { in r535_gsp_libos_debugfs_init()
2492 kfree(gsp->blob_pmu.data); in r535_gsp_libos_debugfs_init()
2496 i_size_write(d_inode(gsp->debugfs.init), gsp->blob_init.size); in r535_gsp_libos_debugfs_init()
2497 i_size_write(d_inode(gsp->debugfs.intr), gsp->blob_intr.size); in r535_gsp_libos_debugfs_init()
2498 i_size_write(d_inode(gsp->debugfs.rm), gsp->blob_rm.size); in r535_gsp_libos_debugfs_init()
2499 i_size_write(d_inode(gsp->debugfs.pmu), gsp->blob_pmu.size); in r535_gsp_libos_debugfs_init()
2504 nvkm_debug(&gsp->subdev, "created debugfs GSP-RM logging entries\n"); in r535_gsp_libos_debugfs_init()
2506 if (keep_gsp_logging) { in r535_gsp_libos_debugfs_init()
2507 nvkm_info(&gsp->subdev, in r535_gsp_libos_debugfs_init()
2514 debugfs_remove(gsp->debugfs.parent); in r535_gsp_libos_debugfs_init()
2515 gsp->debugfs.parent = NULL; in r535_gsp_libos_debugfs_init()
2532 * create_pte_array() - creates a PTE array of a physically contiguous buffer
2537 * GSP-RM sometimes expects physically-contiguous buffers to have an array of
2539 * the buffer to be physically discontiguous, GSP-RM does not currently
2558 * r535_gsp_libos_init() -- create the libos arguments structure
2561 * The logging buffers are byte queues that contain encoded printf-like
2562 * messages from GSP-RM. They need to be decoded by a special application
2565 * The 'loginit' buffer contains logs from early GSP-RM init and
2567 * written to directly by GSP-RM and can be any multiple of GSP_PAGE_SIZE.
2571 * Initially, pp is equal to 0. If the buffer has valid logging data in it,
2573 * be written. Therefore, the logging data is valid if:
2576 * The GSP only understands 4K pages (GSP_PAGE_SIZE), so even if the kernel is
2581 * The buffers must be a multiple of GSP_PAGE_SIZE. GSP-RM also currently
2586 * init_done RPC.
2597 ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->libos); in r535_gsp_libos_init()
2598 if (ret) in r535_gsp_libos_init()
2601 args = gsp->libos.data; in r535_gsp_libos_init()
2603 ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->loginit); in r535_gsp_libos_init()
2604 if (ret) in r535_gsp_libos_init()
2608 args[0].pa = gsp->loginit.addr; in r535_gsp_libos_init()
2609 args[0].size = gsp->loginit.size; in r535_gsp_libos_init()
2612 create_pte_array(gsp->loginit.data + sizeof(u64), gsp->loginit.addr, gsp->loginit.size); in r535_gsp_libos_init()
2614 ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->logintr); in r535_gsp_libos_init()
2615 if (ret) in r535_gsp_libos_init()
2619 args[1].pa = gsp->logintr.addr; in r535_gsp_libos_init()
2620 args[1].size = gsp->logintr.size; in r535_gsp_libos_init()
2623 create_pte_array(gsp->logintr.data + sizeof(u64), gsp->logintr.addr, gsp->logintr.size); in r535_gsp_libos_init()
2625 ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->logrm); in r535_gsp_libos_init()
2626 if (ret) in r535_gsp_libos_init()
2630 args[2].pa = gsp->logrm.addr; in r535_gsp_libos_init()
2631 args[2].size = gsp->logrm.size; in r535_gsp_libos_init()
2634 create_pte_array(gsp->logrm.data + sizeof(u64), gsp->logrm.addr, gsp->logrm.size); in r535_gsp_libos_init()
2637 if (ret) in r535_gsp_libos_init()
2641 args[3].pa = gsp->rmargs.addr; in r535_gsp_libos_init()
2642 args[3].size = gsp->rmargs.size; in r535_gsp_libos_init()
2659 dma_unmap_sgtable(device->dev, sgt, DMA_BIDIRECTIONAL, 0); in nvkm_gsp_sg_free()
2678 if (ret) in nvkm_gsp_sg()
2684 if (!page) { in nvkm_gsp_sg()
2686 return -ENOMEM; in nvkm_gsp_sg()
2692 ret = dma_map_sgtable(device->dev, sgt, DMA_BIDIRECTIONAL, 0); in nvkm_gsp_sg()
2693 if (ret) in nvkm_gsp_sg()
2702 nvkm_gsp_sg_free(gsp->subdev.device, &rx3->lvl2); in nvkm_gsp_radix3_dtor()
2703 nvkm_gsp_mem_dtor(&rx3->lvl1); in nvkm_gsp_radix3_dtor()
2704 nvkm_gsp_mem_dtor(&rx3->lvl0); in nvkm_gsp_radix3_dtor()
2708 * nvkm_gsp_radix3_sg - build a radix3 table from a S/G list
2714 * The GSP uses a three-level page table, called radix3, to map the firmware.
2715 * Each 64-bit "pointer" in the table is either the bus address of an entry in
2733 * result is that GSP-RM must translate the bus addresses in the table to GSP
2750 ret = nvkm_gsp_mem_ctor(gsp, GSP_PAGE_SIZE, &rx3->lvl0); in nvkm_gsp_radix3_sg()
2751 if (ret) in nvkm_gsp_radix3_sg()
2754 ret = nvkm_gsp_mem_ctor(gsp, GSP_PAGE_SIZE, &rx3->lvl1); in nvkm_gsp_radix3_sg()
2755 if (ret) in nvkm_gsp_radix3_sg()
2760 ret = nvkm_gsp_sg(gsp->subdev.device, bufsize, &rx3->lvl2); in nvkm_gsp_radix3_sg()
2761 if (ret) in nvkm_gsp_radix3_sg()
2765 pte = rx3->lvl0.data; in nvkm_gsp_radix3_sg()
2766 *pte = rx3->lvl1.addr; in nvkm_gsp_radix3_sg()
2769 pte = rx3->lvl1.data; in nvkm_gsp_radix3_sg()
2770 for_each_sgtable_dma_page(&rx3->lvl2, &sg_dma_iter, 0) in nvkm_gsp_radix3_sg()
2774 for_each_sgtable_sg(&rx3->lvl2, sg, i) { in nvkm_gsp_radix3_sg()
2778 sgl_end = (void *)pte + sg->length; in nvkm_gsp_radix3_sg()
2784 // Go to the next scatterlist for level 2 if we've reached the end in nvkm_gsp_radix3_sg()
2785 if ((void *)pte >= sgl_end) in nvkm_gsp_radix3_sg()
2790 if (ret) { in nvkm_gsp_radix3_sg()
2792 nvkm_gsp_mem_dtor(&rx3->lvl1); in nvkm_gsp_radix3_sg()
2794 nvkm_gsp_mem_dtor(&rx3->lvl0); in nvkm_gsp_radix3_sg()
2806 if (!gsp->running) in r535_gsp_fini()
2809 if (suspend) { in r535_gsp_fini()
2810 GspFwWprMeta *meta = gsp->wpr_meta.data; in r535_gsp_fini()
2811 u64 len = meta->gspFwWprEnd - meta->gspFwWprStart; in r535_gsp_fini()
2814 ret = nvkm_gsp_sg(gsp->subdev.device, len, &gsp->sr.sgt); in r535_gsp_fini()
2815 if (ret) in r535_gsp_fini()
2818 ret = nvkm_gsp_radix3_sg(gsp, &gsp->sr.sgt, len, &gsp->sr.radix3); in r535_gsp_fini()
2819 if (ret) in r535_gsp_fini()
2822 ret = nvkm_gsp_mem_ctor(gsp, sizeof(*sr), &gsp->sr.meta); in r535_gsp_fini()
2823 if (ret) in r535_gsp_fini()
2826 sr = gsp->sr.meta.data; in r535_gsp_fini()
2827 sr->magic = GSP_FW_SR_META_MAGIC; in r535_gsp_fini()
2828 sr->revision = GSP_FW_SR_META_REVISION; in r535_gsp_fini()
2829 sr->sysmemAddrOfSuspendResumeData = gsp->sr.radix3.lvl0.addr; in r535_gsp_fini()
2830 sr->sizeOfSuspendResumeData = len; in r535_gsp_fini()
2832 mbox0 = lower_32_bits(gsp->sr.meta.addr); in r535_gsp_fini()
2833 mbox1 = upper_32_bits(gsp->sr.meta.addr); in r535_gsp_fini()
2837 if (WARN_ON(ret)) in r535_gsp_fini()
2840 nvkm_msec(gsp->subdev.device, 2000, in r535_gsp_fini()
2841 if (nvkm_falcon_rd32(&gsp->falcon, 0x040) & 0x80000000) in r535_gsp_fini()
2845 nvkm_falcon_reset(&gsp->falcon); in r535_gsp_fini()
2853 gsp->running = false; in r535_gsp_fini()
2863 if (!gsp->sr.meta.data) { in r535_gsp_init()
2864 mbox0 = lower_32_bits(gsp->wpr_meta.addr); in r535_gsp_init()
2865 mbox1 = upper_32_bits(gsp->wpr_meta.addr); in r535_gsp_init()
2869 mbox0 = lower_32_bits(gsp->sr.meta.addr); in r535_gsp_init()
2870 mbox1 = upper_32_bits(gsp->sr.meta.addr); in r535_gsp_init()
2873 /* Execute booter to handle (eventually...) booting GSP-RM. */ in r535_gsp_init()
2875 if (WARN_ON(ret)) in r535_gsp_init()
2879 if (ret) in r535_gsp_init()
2882 gsp->running = true; in r535_gsp_init()
2885 if (gsp->sr.meta.data) { in r535_gsp_init()
2886 nvkm_gsp_mem_dtor(&gsp->sr.meta); in r535_gsp_init()
2887 nvkm_gsp_radix3_dtor(gsp, &gsp->sr.radix3); in r535_gsp_init()
2888 nvkm_gsp_sg_free(gsp->subdev.device, &gsp->sr.sgt); in r535_gsp_init()
2892 if (ret == 0) in r535_gsp_init()
2901 const struct firmware *fw = gsp->fws.bl; in r535_gsp_rm_boot_ctor()
2906 hdr = nvfw_bin_hdr(&gsp->subdev, fw->data); in r535_gsp_rm_boot_ctor()
2907 desc = (void *)fw->data + hdr->header_offset; in r535_gsp_rm_boot_ctor()
2909 ret = nvkm_gsp_mem_ctor(gsp, hdr->data_size, &gsp->boot.fw); in r535_gsp_rm_boot_ctor()
2910 if (ret) in r535_gsp_rm_boot_ctor()
2913 memcpy(gsp->boot.fw.data, fw->data + hdr->data_offset, hdr->data_size); in r535_gsp_rm_boot_ctor()
2915 gsp->boot.code_offset = desc->monitorCodeOffset; in r535_gsp_rm_boot_ctor()
2916 gsp->boot.data_offset = desc->monitorDataOffset; in r535_gsp_rm_boot_ctor()
2917 gsp->boot.manifest_offset = desc->manifestOffset; in r535_gsp_rm_boot_ctor()
2918 gsp->boot.app_version = desc->appVersion; in r535_gsp_rm_boot_ctor()
2930 const u8 *img = gsp->fws.rm->data; in r535_gsp_elf_section()
2932 const struct elf64_shdr *shdr = (const struct elf64_shdr *)&img[ehdr->e_shoff]; in r535_gsp_elf_section()
2933 const char *names = &img[shdr[ehdr->e_shstrndx].sh_offset]; in r535_gsp_elf_section()
2935 for (int i = 0; i < ehdr->e_shnum; i++, shdr++) { in r535_gsp_elf_section()
2936 if (!strcmp(&names[shdr->sh_name], name)) { in r535_gsp_elf_section()
2937 *pdata = &img[shdr->sh_offset]; in r535_gsp_elf_section()
2938 *psize = shdr->sh_size; in r535_gsp_elf_section()
2943 nvkm_error(&gsp->subdev, "section '%s' not found\n", name); in r535_gsp_elf_section()
2944 return -ENOENT; in r535_gsp_elf_section()
2950 nvkm_firmware_put(gsp->fws.bl); in r535_gsp_dtor_fws()
2951 gsp->fws.bl = NULL; in r535_gsp_dtor_fws()
2952 nvkm_firmware_put(gsp->fws.booter.unload); in r535_gsp_dtor_fws()
2953 gsp->fws.booter.unload = NULL; in r535_gsp_dtor_fws()
2954 nvkm_firmware_put(gsp->fws.booter.load); in r535_gsp_dtor_fws()
2955 gsp->fws.booter.load = NULL; in r535_gsp_dtor_fws()
2956 nvkm_firmware_put(gsp->fws.rm); in r535_gsp_dtor_fws()
2957 gsp->fws.rm = NULL; in r535_gsp_dtor_fws()
2977 * r535_debugfs_shutdown - delete GSP-RM logging buffers for one GPU
2980 * Called when the driver is shutting down, to clean up the retained GSP-RM
2987 debugfs_remove(log->debugfs_logging_dir); in r535_debugfs_shutdown()
2989 kfree(log->blob_init.data); in r535_debugfs_shutdown()
2990 kfree(log->blob_intr.data); in r535_debugfs_shutdown()
2991 kfree(log->blob_rm.data); in r535_debugfs_shutdown()
2992 kfree(log->blob_pmu.data); in r535_debugfs_shutdown()
2999 * is_empty - return true if the logging buffer was never written to
3000 * @b: blob wrapper with ->data field pointing to logging buffer
3002 * The first 64-bit field of loginit, and logintr, and logrm is the 'put'
3003 * pointer, and it is initialized to 0. It's a dword-based index into the
3006 * If the pointer is still 0 when GSP-RM is shut down, that means that the
3013 u64 *put = b->data; in is_empty()
3019 * r535_gsp_copy_log - preserve the logging buffers in a blob
3020 * @parent: the top-level dentry for this GPU
3027 * if they actually have data.
3037 if (is_empty(s)) in r535_gsp_copy_log()
3041 p = kmemdup(s->data, s->size, GFP_KERNEL); in r535_gsp_copy_log()
3042 if (!p) in r535_gsp_copy_log()
3043 return -ENOMEM; in r535_gsp_copy_log()
3045 t->data = p; in r535_gsp_copy_log()
3046 t->size = s->size; in r535_gsp_copy_log()
3049 if (IS_ERR(dent)) { in r535_gsp_copy_log()
3055 i_size_write(d_inode(dent), t->size); in r535_gsp_copy_log()
3061 * r535_gsp_retain_logging - copy logging buffers to new debugfs root
3064 * If keep_gsp_logging is enabled, then we want to preserve the GSP-RM logging
3066 * deleted if GSP-RM fails to load.
3076 * If anything fails in this process, then all the dentries need to be
3082 struct device *dev = gsp->subdev.device->dev; in r535_gsp_retain_logging()
3086 if (!keep_gsp_logging || !gsp->debugfs.parent) { in r535_gsp_retain_logging()
3092 if (is_empty(&gsp->blob_init) && is_empty(&gsp->blob_intr) && in r535_gsp_retain_logging()
3093 is_empty(&gsp->blob_rm) && is_empty(&gsp->blob_rm)) { in r535_gsp_retain_logging()
3094 nvkm_warn(&gsp->subdev, "all logging buffers are empty\n"); in r535_gsp_retain_logging()
3099 if (!log) in r535_gsp_retain_logging()
3108 debugfs_remove(gsp->debugfs.init); in r535_gsp_retain_logging()
3109 debugfs_remove(gsp->debugfs.intr); in r535_gsp_retain_logging()
3110 debugfs_remove(gsp->debugfs.rm); in r535_gsp_retain_logging()
3111 debugfs_remove(gsp->debugfs.pmu); in r535_gsp_retain_logging()
3113 ret = r535_gsp_copy_log(gsp->debugfs.parent, "loginit", &gsp->blob_init, &log->blob_init); in r535_gsp_retain_logging()
3114 if (ret) in r535_gsp_retain_logging()
3117 ret = r535_gsp_copy_log(gsp->debugfs.parent, "logintr", &gsp->blob_intr, &log->blob_intr); in r535_gsp_retain_logging()
3118 if (ret) in r535_gsp_retain_logging()
3121 ret = r535_gsp_copy_log(gsp->debugfs.parent, "logrm", &gsp->blob_rm, &log->blob_rm); in r535_gsp_retain_logging()
3122 if (ret) in r535_gsp_retain_logging()
3125 ret = r535_gsp_copy_log(gsp->debugfs.parent, "logpmu", &gsp->blob_pmu, &log->blob_pmu); in r535_gsp_retain_logging()
3126 if (ret) in r535_gsp_retain_logging()
3130 log->debugfs_logging_dir = gsp->debugfs.parent; in r535_gsp_retain_logging()
3132 log->log.shutdown = r535_debugfs_shutdown; in r535_gsp_retain_logging()
3133 list_add(&log->log.entry, &gsp_logs.head); in r535_gsp_retain_logging()
3135 nvkm_warn(&gsp->subdev, in r535_gsp_retain_logging()
3142 nvkm_warn(&gsp->subdev, "failed to migrate logging buffers\n"); in r535_gsp_retain_logging()
3145 debugfs_remove(gsp->debugfs.parent); in r535_gsp_retain_logging()
3147 if (log) { in r535_gsp_retain_logging()
3148 kfree(log->blob_init.data); in r535_gsp_retain_logging()
3149 kfree(log->blob_intr.data); in r535_gsp_retain_logging()
3150 kfree(log->blob_rm.data); in r535_gsp_retain_logging()
3151 kfree(log->blob_pmu.data); in r535_gsp_retain_logging()
3159 * r535_gsp_libos_debugfs_fini - cleanup/retain log buffers on shutdown
3162 * If the log buffers are exposed via debugfs, the data for those entries
3173 * exists only if the debugfs entries were created. in r535_gsp_libos_debugfs_fini()
3175 kfree(gsp->blob_pmu.data); in r535_gsp_libos_debugfs_fini()
3176 gsp->blob_pmu.data = NULL; in r535_gsp_libos_debugfs_fini()
3183 idr_destroy(&gsp->client_id.idr); in r535_gsp_dtor()
3184 mutex_destroy(&gsp->client_id.mutex); in r535_gsp_dtor()
3186 nvkm_gsp_radix3_dtor(gsp, &gsp->radix3); in r535_gsp_dtor()
3187 nvkm_gsp_mem_dtor(&gsp->sig); in r535_gsp_dtor()
3188 nvkm_firmware_dtor(&gsp->fw); in r535_gsp_dtor()
3190 nvkm_falcon_fw_dtor(&gsp->booter.unload); in r535_gsp_dtor()
3191 nvkm_falcon_fw_dtor(&gsp->booter.load); in r535_gsp_dtor()
3193 mutex_destroy(&gsp->msgq.mutex); in r535_gsp_dtor()
3194 mutex_destroy(&gsp->cmdq.mutex); in r535_gsp_dtor()
3198 nvkm_gsp_mem_dtor(&gsp->rmargs); in r535_gsp_dtor()
3199 nvkm_gsp_mem_dtor(&gsp->wpr_meta); in r535_gsp_dtor()
3200 nvkm_gsp_mem_dtor(&gsp->shm.mem); in r535_gsp_dtor()
3204 nvkm_gsp_mem_dtor(&gsp->loginit); in r535_gsp_dtor()
3205 nvkm_gsp_mem_dtor(&gsp->logintr); in r535_gsp_dtor()
3206 nvkm_gsp_mem_dtor(&gsp->logrm); in r535_gsp_dtor()
3212 struct nvkm_device *device = gsp->subdev.device; in r535_gsp_oneinit()
3217 mutex_init(&gsp->cmdq.mutex); in r535_gsp_oneinit()
3218 mutex_init(&gsp->msgq.mutex); in r535_gsp_oneinit()
3220 ret = gsp->func->booter.ctor(gsp, "booter-load", gsp->fws.booter.load, in r535_gsp_oneinit()
3221 &device->sec2->falcon, &gsp->booter.load); in r535_gsp_oneinit()
3222 if (ret) in r535_gsp_oneinit()
3225 ret = gsp->func->booter.ctor(gsp, "booter-unload", gsp->fws.booter.unload, in r535_gsp_oneinit()
3226 &device->sec2->falcon, &gsp->booter.unload); in r535_gsp_oneinit()
3227 if (ret) in r535_gsp_oneinit()
3230 /* Load GSP firmware from ELF image into DMA-accessible memory. */ in r535_gsp_oneinit()
3232 if (ret) in r535_gsp_oneinit()
3235 ret = nvkm_firmware_ctor(&r535_gsp_fw, "gsp-rm", device, data, size, &gsp->fw); in r535_gsp_oneinit()
3236 if (ret) in r535_gsp_oneinit()
3240 ret = r535_gsp_elf_section(gsp, gsp->func->sig_section, &data, &size); in r535_gsp_oneinit()
3241 if (ret) in r535_gsp_oneinit()
3244 ret = nvkm_gsp_mem_ctor(gsp, ALIGN(size, 256), &gsp->sig); in r535_gsp_oneinit()
3245 if (ret) in r535_gsp_oneinit()
3248 memcpy(gsp->sig.data, data, size); in r535_gsp_oneinit()
3251 ret = nvkm_gsp_radix3_sg(gsp, &gsp->fw.mem.sgt, gsp->fw.len, &gsp->radix3); in r535_gsp_oneinit()
3252 if (ret) in r535_gsp_oneinit()
3267 if (ret) in r535_gsp_oneinit()
3270 /* Release FW images - we've copied them to DMA buffers now. */ in r535_gsp_oneinit()
3274 gsp->fb.wpr2.frts.size = 0x100000; in r535_gsp_oneinit()
3275 gsp->fb.wpr2.frts.addr = ALIGN_DOWN(gsp->fb.bios.addr, 0x20000) - gsp->fb.wpr2.frts.size; in r535_gsp_oneinit()
3277 gsp->fb.wpr2.boot.size = gsp->boot.fw.size; in r535_gsp_oneinit()
3278 gsp->fb.wpr2.boot.addr = ALIGN_DOWN(gsp->fb.wpr2.frts.addr - gsp->fb.wpr2.boot.size, 0x1000); in r535_gsp_oneinit()
3280 gsp->fb.wpr2.elf.size = gsp->fw.len; in r535_gsp_oneinit()
3281 gsp->fb.wpr2.elf.addr = ALIGN_DOWN(gsp->fb.wpr2.boot.addr - gsp->fb.wpr2.elf.size, 0x10000); in r535_gsp_oneinit()
3284 u32 fb_size_gb = DIV_ROUND_UP_ULL(gsp->fb.size, 1 << 30); in r535_gsp_oneinit()
3286 gsp->fb.wpr2.heap.size = in r535_gsp_oneinit()
3287 gsp->func->wpr_heap.os_carveout_size + in r535_gsp_oneinit()
3288 gsp->func->wpr_heap.base_size + in r535_gsp_oneinit()
3292 gsp->fb.wpr2.heap.size = max(gsp->fb.wpr2.heap.size, gsp->func->wpr_heap.min_size); in r535_gsp_oneinit()
3295 gsp->fb.wpr2.heap.addr = ALIGN_DOWN(gsp->fb.wpr2.elf.addr - gsp->fb.wpr2.heap.size, 0x100000); in r535_gsp_oneinit()
3296 gsp->fb.wpr2.heap.size = ALIGN_DOWN(gsp->fb.wpr2.elf.addr - gsp->fb.wpr2.heap.addr, 0x100000); in r535_gsp_oneinit()
3298 gsp->fb.wpr2.addr = ALIGN_DOWN(gsp->fb.wpr2.heap.addr - sizeof(GspFwWprMeta), 0x100000); in r535_gsp_oneinit()
3299 gsp->fb.wpr2.size = gsp->fb.wpr2.frts.addr + gsp->fb.wpr2.frts.size - gsp->fb.wpr2.addr; in r535_gsp_oneinit()
3301 gsp->fb.heap.size = 0x100000; in r535_gsp_oneinit()
3302 gsp->fb.heap.addr = gsp->fb.wpr2.addr - gsp->fb.heap.size; in r535_gsp_oneinit()
3305 if (WARN_ON(ret)) in r535_gsp_oneinit()
3309 if (WARN_ON(ret)) in r535_gsp_oneinit()
3313 if (WARN_ON(ret)) in r535_gsp_oneinit()
3317 if (WARN_ON(ret)) in r535_gsp_oneinit()
3321 if (WARN_ON(ret)) in r535_gsp_oneinit()
3324 /* Reset GSP into RISC-V mode. */ in r535_gsp_oneinit()
3325 ret = gsp->func->reset(gsp); in r535_gsp_oneinit()
3326 if (WARN_ON(ret)) in r535_gsp_oneinit()
3329 nvkm_falcon_wr32(&gsp->falcon, 0x040, lower_32_bits(gsp->libos.addr)); in r535_gsp_oneinit()
3330 nvkm_falcon_wr32(&gsp->falcon, 0x044, upper_32_bits(gsp->libos.addr)); in r535_gsp_oneinit()
3332 mutex_init(&gsp->client_id.mutex); in r535_gsp_oneinit()
3333 idr_init(&gsp->client_id.idr); in r535_gsp_oneinit()
3343 snprintf(fwname, sizeof(fwname), "gsp/%s-%s", name, ver); in r535_gsp_load_fw()
3344 return nvkm_firmware_get(&gsp->subdev, fwname, 0, pfw); in r535_gsp_load_fw()
3350 struct nvkm_subdev *subdev = &gsp->subdev; in r535_gsp_load()
3352 bool enable_gsp = fwif->enable; in r535_gsp_load()
3354 #if IS_ENABLED(CONFIG_DRM_NOUVEAU_GSP_DEFAULT) in r535_gsp_load()
3357 if (!nvkm_boolopt(subdev->device->cfgopt, "NvGspRm", enable_gsp)) in r535_gsp_load()
3358 return -EINVAL; in r535_gsp_load()
3360 if ((ret = r535_gsp_load_fw(gsp, "gsp", fwif->ver, &gsp->fws.rm)) || in r535_gsp_load()
3361 (ret = r535_gsp_load_fw(gsp, "booter_load", fwif->ver, &gsp->fws.booter.load)) || in r535_gsp_load()
3362 (ret = r535_gsp_load_fw(gsp, "booter_unload", fwif->ver, &gsp->fws.booter.unload)) || in r535_gsp_load()
3363 (ret = r535_gsp_load_fw(gsp, "bootloader", fwif->ver, &gsp->fws.bl))) { in r535_gsp_load()
3372 MODULE_FIRMWARE("nvidia/"#chip"/gsp/booter_load-535.113.01.bin"); \
3373 MODULE_FIRMWARE("nvidia/"#chip"/gsp/booter_unload-535.113.01.bin"); \
3374 MODULE_FIRMWARE("nvidia/"#chip"/gsp/bootloader-535.113.01.bin"); \
3375 MODULE_FIRMWARE("nvidia/"#chip"/gsp/gsp-535.113.01.bin")