1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * System Control and Management Interface (SCMI) Raw mode support 4 * 5 * Copyright (C) 2022 ARM Ltd. 6 */ 7 /** 8 * DOC: Theory of operation 9 * 10 * When enabled the SCMI Raw mode support exposes a userspace API which allows 11 * to send and receive SCMI commands, replies and notifications from a user 12 * application through injection and snooping of bare SCMI messages in binary 13 * little-endian format. 14 * 15 * Such injected SCMI transactions will then be routed through the SCMI core 16 * stack towards the SCMI backend server using whatever SCMI transport is 17 * currently configured on the system under test. 18 * 19 * It is meant to help in running any sort of SCMI backend server testing, no 20 * matter where the server is placed, as long as it is normally reachable via 21 * the transport configured on the system. 22 * 23 * It is activated by a Kernel configuration option since it is NOT meant to 24 * be used in production but only during development and in CI deployments. 25 * 26 * In order to avoid possible interferences between the SCMI Raw transactions 27 * originated from a test-suite and the normal operations of the SCMI drivers, 28 * when Raw mode is enabled, by default, all the regular SCMI drivers are 29 * inhibited, unless CONFIG_ARM_SCMI_RAW_MODE_SUPPORT_COEX is enabled: in this 30 * latter case the regular SCMI stack drivers will be loaded as usual and it is 31 * up to the user of this interface to take care of manually inhibiting the 32 * regular SCMI drivers in order to avoid interferences during the test runs. 33 * 34 * The exposed API is as follows. 35 * 36 * All SCMI Raw entries are rooted under a common top /raw debugfs top directory 37 * which in turn is rooted under the corresponding underlying SCMI instance. 38 * 39 * /sys/kernel/debug/scmi/ 40 * `-- 0 41 * |-- atomic_threshold_us 42 * |-- instance_name 43 * |-- raw 44 * | |-- channels 45 * | | |-- 0x10 46 * | | | |-- message 47 * | | | `-- message_async 48 * | | `-- 0x13 49 * | | |-- message 50 * | | `-- message_async 51 * | |-- errors 52 * | |-- message 53 * | |-- message_async 54 * | |-- notification 55 * | `-- reset 56 * `-- transport 57 * |-- is_atomic 58 * |-- max_msg_size 59 * |-- max_rx_timeout_ms 60 * |-- rx_max_msg 61 * |-- tx_max_msg 62 * `-- type 63 * 64 * where: 65 * 66 * - errors: used to read back timed-out and unexpected replies 67 * - message*: used to send sync/async commands and read back immediate and 68 * delayed reponses (if any) 69 * - notification: used to read any notification being emitted by the system 70 * (if previously enabled by the user app) 71 * - reset: used to flush the queues of messages (of any kind) still pending 72 * to be read; this is useful at test-suite start/stop to get 73 * rid of any unread messages from the previous run. 74 * 75 * with the per-channel entries rooted at /channels being present only on a 76 * system where multiple transport channels have been configured. 77 * 78 * Such per-channel entries can be used to explicitly choose a specific channel 79 * for SCMI bare message injection, in contrast with the general entries above 80 * where, instead, the selection of the proper channel to use is automatically 81 * performed based the protocol embedded in the injected message and on how the 82 * transport is configured on the system. 83 * 84 * Note that other common general entries are available under transport/ to let 85 * the user applications properly make up their expectations in terms of 86 * timeouts and message characteristics. 87 * 88 * Each write to the message* entries causes one command request to be built 89 * and sent while the replies or delayed response are read back from those same 90 * entries one message at time (receiving an EOF at each message boundary). 91 * 92 * The user application running the test is in charge of handling timeouts 93 * on replies and properly choosing SCMI sequence numbers for the outgoing 94 * requests (using the same sequence number is supported but discouraged). 95 * 96 * Injection of multiple in-flight requests is supported as long as the user 97 * application uses properly distinct sequence numbers for concurrent requests 98 * and takes care to properly manage all the related issues about concurrency 99 * and command/reply pairing. Keep in mind that, anyway, the real level of 100 * parallelism attainable in such scenario is dependent on the characteristics 101 * of the underlying transport being used. 102 * 103 * Since the SCMI core regular stack is partially used to deliver and collect 104 * the messages, late replies arrived after timeouts and any other sort of 105 * unexpected message can be identified by the SCMI core as usual and they will 106 * be reported as messages under "errors" for later analysis. 107 */ 108 109 #include <linux/bitmap.h> 110 #include <linux/debugfs.h> 111 #include <linux/delay.h> 112 #include <linux/device.h> 113 #include <linux/export.h> 114 #include <linux/io.h> 115 #include <linux/kernel.h> 116 #include <linux/fs.h> 117 #include <linux/list.h> 118 #include <linux/module.h> 119 #include <linux/poll.h> 120 #include <linux/of.h> 121 #include <linux/slab.h> 122 #include <linux/xarray.h> 123 124 #include "common.h" 125 126 #include "raw_mode.h" 127 128 #include <trace/events/scmi.h> 129 130 #define SCMI_XFER_RAW_MAX_RETRIES 10 131 132 /** 133 * struct scmi_raw_queue - Generic Raw queue descriptor 134 * 135 * @free_bufs: A freelists listhead used to keep unused raw buffers 136 * @free_bufs_lock: Spinlock used to protect access to @free_bufs 137 * @msg_q: A listhead to a queue of snooped messages waiting to be read out 138 * @msg_q_lock: Spinlock used to protect access to @msg_q 139 * @wq: A waitqueue used to wait and poll on related @msg_q 140 */ 141 struct scmi_raw_queue { 142 struct list_head free_bufs; 143 /* Protect free_bufs[] lists */ 144 spinlock_t free_bufs_lock; 145 struct list_head msg_q; 146 /* Protect msg_q[] lists */ 147 spinlock_t msg_q_lock; 148 wait_queue_head_t wq; 149 }; 150 151 /** 152 * struct scmi_raw_mode_info - Structure holding SCMI Raw instance data 153 * 154 * @id: Sequential Raw instance ID. 155 * @handle: Pointer to SCMI entity handle to use 156 * @desc: Pointer to the transport descriptor to use 157 * @tx_max_msg: Maximum number of concurrent TX in-flight messages 158 * @q: An array of Raw queue descriptors 159 * @chans_q: An XArray mapping optional additional per-channel queues 160 * @free_waiters: Head of freelist for unused waiters 161 * @free_mtx: A mutex to protect the waiters freelist 162 * @active_waiters: Head of list for currently active and used waiters 163 * @active_mtx: A mutex to protect the active waiters list 164 * @waiters_work: A work descriptor to be used with the workqueue machinery 165 * @wait_wq: A workqueue reference to the created workqueue 166 * @dentry: Top debugfs root dentry for SCMI Raw 167 * @gid: A group ID used for devres accounting 168 * 169 * Note that this descriptor is passed back to the core after SCMI Raw is 170 * initialized as an opaque handle to use by subsequent SCMI Raw call hooks. 171 * 172 */ 173 struct scmi_raw_mode_info { 174 unsigned int id; 175 const struct scmi_handle *handle; 176 const struct scmi_desc *desc; 177 int tx_max_msg; 178 struct scmi_raw_queue *q[SCMI_RAW_MAX_QUEUE]; 179 struct xarray chans_q; 180 struct list_head free_waiters; 181 /* Protect free_waiters list */ 182 struct mutex free_mtx; 183 struct list_head active_waiters; 184 /* Protect active_waiters list */ 185 struct mutex active_mtx; 186 struct work_struct waiters_work; 187 struct workqueue_struct *wait_wq; 188 struct dentry *dentry; 189 void *gid; 190 }; 191 192 /** 193 * struct scmi_xfer_raw_waiter - Structure to describe an xfer to be waited for 194 * 195 * @start_jiffies: The timestamp in jiffies of when this structure was queued. 196 * @cinfo: A reference to the channel to use for this transaction 197 * @xfer: A reference to the xfer to be waited for 198 * @async_response: A completion to be, optionally, used for async waits: it 199 * will be setup by @scmi_do_xfer_raw_start, if needed, to be 200 * pointed at by xfer->async_done. 201 * @node: A list node. 202 */ 203 struct scmi_xfer_raw_waiter { 204 unsigned long start_jiffies; 205 struct scmi_chan_info *cinfo; 206 struct scmi_xfer *xfer; 207 struct completion async_response; 208 struct list_head node; 209 }; 210 211 /** 212 * struct scmi_raw_buffer - Structure to hold a full SCMI message 213 * 214 * @max_len: The maximum allowed message size (header included) that can be 215 * stored into @msg 216 * @msg: A message buffer used to collect a full message grabbed from an xfer. 217 * @node: A list node. 218 */ 219 struct scmi_raw_buffer { 220 size_t max_len; 221 struct scmi_msg msg; 222 struct list_head node; 223 }; 224 225 /** 226 * struct scmi_dbg_raw_data - Structure holding data needed by the debugfs 227 * layer 228 * 229 * @chan_id: The preferred channel to use: if zero the channel is automatically 230 * selected based on protocol. 231 * @raw: A reference to the Raw instance. 232 * @tx: A message buffer used to collect TX message on write. 233 * @tx_size: The effective size of the TX message. 234 * @tx_req_size: The final expected size of the complete TX message. 235 * @rx: A message buffer to collect RX message on read. 236 * @rx_size: The effective size of the RX message. 237 */ 238 struct scmi_dbg_raw_data { 239 u8 chan_id; 240 struct scmi_raw_mode_info *raw; 241 struct scmi_msg tx; 242 size_t tx_size; 243 size_t tx_req_size; 244 struct scmi_msg rx; 245 size_t rx_size; 246 }; 247 248 static struct scmi_raw_queue * 249 scmi_raw_queue_select(struct scmi_raw_mode_info *raw, unsigned int idx, 250 unsigned int chan_id) 251 { 252 if (!chan_id) 253 return raw->q[idx]; 254 255 return xa_load(&raw->chans_q, chan_id); 256 } 257 258 static struct scmi_raw_buffer *scmi_raw_buffer_get(struct scmi_raw_queue *q) 259 { 260 unsigned long flags; 261 struct scmi_raw_buffer *rb = NULL; 262 struct list_head *head = &q->free_bufs; 263 264 spin_lock_irqsave(&q->free_bufs_lock, flags); 265 if (!list_empty(head)) { 266 rb = list_first_entry(head, struct scmi_raw_buffer, node); 267 list_del_init(&rb->node); 268 } 269 spin_unlock_irqrestore(&q->free_bufs_lock, flags); 270 271 return rb; 272 } 273 274 static void scmi_raw_buffer_put(struct scmi_raw_queue *q, 275 struct scmi_raw_buffer *rb) 276 { 277 unsigned long flags; 278 279 /* Reset to full buffer length */ 280 rb->msg.len = rb->max_len; 281 282 spin_lock_irqsave(&q->free_bufs_lock, flags); 283 list_add_tail(&rb->node, &q->free_bufs); 284 spin_unlock_irqrestore(&q->free_bufs_lock, flags); 285 } 286 287 static void scmi_raw_buffer_enqueue(struct scmi_raw_queue *q, 288 struct scmi_raw_buffer *rb) 289 { 290 unsigned long flags; 291 292 spin_lock_irqsave(&q->msg_q_lock, flags); 293 list_add_tail(&rb->node, &q->msg_q); 294 spin_unlock_irqrestore(&q->msg_q_lock, flags); 295 296 wake_up_interruptible(&q->wq); 297 } 298 299 static struct scmi_raw_buffer* 300 scmi_raw_buffer_dequeue_unlocked(struct scmi_raw_queue *q) 301 { 302 struct scmi_raw_buffer *rb = NULL; 303 304 if (!list_empty(&q->msg_q)) { 305 rb = list_first_entry(&q->msg_q, struct scmi_raw_buffer, node); 306 list_del_init(&rb->node); 307 } 308 309 return rb; 310 } 311 312 static struct scmi_raw_buffer *scmi_raw_buffer_dequeue(struct scmi_raw_queue *q) 313 { 314 unsigned long flags; 315 struct scmi_raw_buffer *rb; 316 317 spin_lock_irqsave(&q->msg_q_lock, flags); 318 rb = scmi_raw_buffer_dequeue_unlocked(q); 319 spin_unlock_irqrestore(&q->msg_q_lock, flags); 320 321 return rb; 322 } 323 324 static void scmi_raw_buffer_queue_flush(struct scmi_raw_queue *q) 325 { 326 struct scmi_raw_buffer *rb; 327 328 do { 329 rb = scmi_raw_buffer_dequeue(q); 330 if (rb) 331 scmi_raw_buffer_put(q, rb); 332 } while (rb); 333 } 334 335 static struct scmi_xfer_raw_waiter * 336 scmi_xfer_raw_waiter_get(struct scmi_raw_mode_info *raw, struct scmi_xfer *xfer, 337 struct scmi_chan_info *cinfo, bool async) 338 { 339 struct scmi_xfer_raw_waiter *rw = NULL; 340 341 mutex_lock(&raw->free_mtx); 342 if (!list_empty(&raw->free_waiters)) { 343 rw = list_first_entry(&raw->free_waiters, 344 struct scmi_xfer_raw_waiter, node); 345 list_del_init(&rw->node); 346 347 if (async) { 348 reinit_completion(&rw->async_response); 349 xfer->async_done = &rw->async_response; 350 } 351 352 rw->cinfo = cinfo; 353 rw->xfer = xfer; 354 } 355 mutex_unlock(&raw->free_mtx); 356 357 return rw; 358 } 359 360 static void scmi_xfer_raw_waiter_put(struct scmi_raw_mode_info *raw, 361 struct scmi_xfer_raw_waiter *rw) 362 { 363 if (rw->xfer) { 364 rw->xfer->async_done = NULL; 365 rw->xfer = NULL; 366 } 367 368 mutex_lock(&raw->free_mtx); 369 list_add_tail(&rw->node, &raw->free_waiters); 370 mutex_unlock(&raw->free_mtx); 371 } 372 373 static void scmi_xfer_raw_waiter_enqueue(struct scmi_raw_mode_info *raw, 374 struct scmi_xfer_raw_waiter *rw) 375 { 376 /* A timestamp for the deferred worker to know how much this has aged */ 377 rw->start_jiffies = jiffies; 378 379 trace_scmi_xfer_response_wait(rw->xfer->transfer_id, rw->xfer->hdr.id, 380 rw->xfer->hdr.protocol_id, 381 rw->xfer->hdr.seq, 382 raw->desc->max_rx_timeout_ms, 383 rw->xfer->hdr.poll_completion); 384 385 mutex_lock(&raw->active_mtx); 386 list_add_tail(&rw->node, &raw->active_waiters); 387 mutex_unlock(&raw->active_mtx); 388 389 /* kick waiter work */ 390 queue_work(raw->wait_wq, &raw->waiters_work); 391 } 392 393 static struct scmi_xfer_raw_waiter * 394 scmi_xfer_raw_waiter_dequeue(struct scmi_raw_mode_info *raw) 395 { 396 struct scmi_xfer_raw_waiter *rw = NULL; 397 398 mutex_lock(&raw->active_mtx); 399 if (!list_empty(&raw->active_waiters)) { 400 rw = list_first_entry(&raw->active_waiters, 401 struct scmi_xfer_raw_waiter, node); 402 list_del_init(&rw->node); 403 } 404 mutex_unlock(&raw->active_mtx); 405 406 return rw; 407 } 408 409 /** 410 * scmi_xfer_raw_worker - Work function to wait for Raw xfers completions 411 * 412 * @work: A reference to the work. 413 * 414 * In SCMI Raw mode, once a user-provided injected SCMI message is sent, we 415 * cannot wait to receive its response (if any) in the context of the injection 416 * routines so as not to leave the userspace write syscall, which delivered the 417 * SCMI message to send, pending till eventually a reply is received. 418 * Userspace should and will poll/wait instead on the read syscalls which will 419 * be in charge of reading a received reply (if any). 420 * 421 * Even though reply messages are collected and reported into the SCMI Raw layer 422 * on the RX path, nonetheless we have to properly wait for their completion as 423 * usual (and async_completion too if needed) in order to properly release the 424 * xfer structure at the end: to do this out of the context of the write/send 425 * these waiting jobs are delegated to this deferred worker. 426 * 427 * Any sent xfer, to be waited for, is timestamped and queued for later 428 * consumption by this worker: queue aging is accounted for while choosing a 429 * timeout for the completion, BUT we do not really care here if we end up 430 * accidentally waiting for a bit too long. 431 */ 432 static void scmi_xfer_raw_worker(struct work_struct *work) 433 { 434 struct scmi_raw_mode_info *raw; 435 struct device *dev; 436 unsigned long max_tmo; 437 438 raw = container_of(work, struct scmi_raw_mode_info, waiters_work); 439 dev = raw->handle->dev; 440 max_tmo = msecs_to_jiffies(raw->desc->max_rx_timeout_ms); 441 442 do { 443 int ret = 0; 444 unsigned int timeout_ms; 445 unsigned long aging; 446 struct scmi_xfer *xfer; 447 struct scmi_xfer_raw_waiter *rw; 448 struct scmi_chan_info *cinfo; 449 450 rw = scmi_xfer_raw_waiter_dequeue(raw); 451 if (!rw) 452 return; 453 454 cinfo = rw->cinfo; 455 xfer = rw->xfer; 456 /* 457 * Waiters are queued by wait-deadline at the end, so some of 458 * them could have been already expired when processed, BUT we 459 * have to check the completion status anyway just in case a 460 * virtually expired (aged) transaction was indeed completed 461 * fine and we'll have to wait for the asynchronous part (if 462 * any): for this reason a 1 ms timeout is used for already 463 * expired/aged xfers. 464 */ 465 aging = jiffies - rw->start_jiffies; 466 timeout_ms = max_tmo > aging ? 467 jiffies_to_msecs(max_tmo - aging) : 1; 468 469 ret = scmi_xfer_raw_wait_for_message_response(cinfo, xfer, 470 timeout_ms); 471 if (!ret && xfer->hdr.status) 472 ret = scmi_to_linux_errno(xfer->hdr.status); 473 474 if (raw->desc->ops->mark_txdone) 475 raw->desc->ops->mark_txdone(rw->cinfo, ret, xfer); 476 477 trace_scmi_xfer_end(xfer->transfer_id, xfer->hdr.id, 478 xfer->hdr.protocol_id, xfer->hdr.seq, ret); 479 480 /* Wait also for an async delayed response if needed */ 481 if (!ret && xfer->async_done) { 482 unsigned long tmo = msecs_to_jiffies(SCMI_MAX_RESPONSE_TIMEOUT); 483 484 if (!wait_for_completion_timeout(xfer->async_done, tmo)) 485 dev_err(dev, 486 "timed out in RAW delayed resp - HDR:%08X\n", 487 pack_scmi_header(&xfer->hdr)); 488 } 489 490 /* Release waiter and xfer */ 491 scmi_xfer_raw_put(raw->handle, xfer); 492 scmi_xfer_raw_waiter_put(raw, rw); 493 } while (1); 494 } 495 496 static void scmi_xfer_raw_reset(struct scmi_raw_mode_info *raw) 497 { 498 int i; 499 500 dev_info(raw->handle->dev, "Resetting SCMI Raw stack.\n"); 501 502 for (i = 0; i < SCMI_RAW_MAX_QUEUE; i++) 503 scmi_raw_buffer_queue_flush(raw->q[i]); 504 } 505 506 /** 507 * scmi_xfer_raw_get_init - An helper to build a valid xfer from the provided 508 * bare SCMI message. 509 * 510 * @raw: A reference to the Raw instance. 511 * @buf: A buffer containing the whole SCMI message to send (including the 512 * header) in little-endian binary formmat. 513 * @len: Length of the message in @buf. 514 * @p: A pointer to return the initialized Raw xfer. 515 * 516 * After an xfer is picked from the TX pool and filled in with the message 517 * content, the xfer is registered as pending with the core in the usual way 518 * using the original sequence number provided by the user with the message. 519 * 520 * Note that, in case the testing user application is NOT using distinct 521 * sequence-numbers between successive SCMI messages such registration could 522 * fail temporarily if the previous message, using the same sequence number, 523 * had still not released; in such a case we just wait and retry. 524 * 525 * Return: 0 on Success 526 */ 527 static int scmi_xfer_raw_get_init(struct scmi_raw_mode_info *raw, void *buf, 528 size_t len, struct scmi_xfer **p) 529 { 530 u32 msg_hdr; 531 size_t tx_size; 532 struct scmi_xfer *xfer; 533 int ret, retry = SCMI_XFER_RAW_MAX_RETRIES; 534 struct device *dev = raw->handle->dev; 535 536 if (!buf || len < sizeof(u32)) 537 return -EINVAL; 538 539 tx_size = len - sizeof(u32); 540 /* Ensure we have sane transfer sizes */ 541 if (tx_size > raw->desc->max_msg_size) 542 return -ERANGE; 543 544 xfer = scmi_xfer_raw_get(raw->handle); 545 if (IS_ERR(xfer)) { 546 dev_warn(dev, "RAW - Cannot get a free RAW xfer !\n"); 547 return PTR_ERR(xfer); 548 } 549 550 /* Build xfer from the provided SCMI bare LE message */ 551 msg_hdr = le32_to_cpu(*((__le32 *)buf)); 552 unpack_scmi_header(msg_hdr, &xfer->hdr); 553 xfer->hdr.seq = (u16)MSG_XTRACT_TOKEN(msg_hdr); 554 /* Polling not supported */ 555 xfer->hdr.poll_completion = false; 556 xfer->hdr.status = SCMI_SUCCESS; 557 xfer->tx.len = tx_size; 558 xfer->rx.len = raw->desc->max_msg_size; 559 /* Clear the whole TX buffer */ 560 memset(xfer->tx.buf, 0x00, raw->desc->max_msg_size); 561 if (xfer->tx.len) 562 memcpy(xfer->tx.buf, (u8 *)buf + sizeof(msg_hdr), xfer->tx.len); 563 *p = xfer; 564 565 /* 566 * In flight registration can temporarily fail in case of Raw messages 567 * if the user injects messages without using monotonically increasing 568 * sequence numbers since, in Raw mode, the xfer (and the token) is 569 * finally released later by a deferred worker. Just retry for a while. 570 */ 571 do { 572 ret = scmi_xfer_raw_inflight_register(raw->handle, xfer); 573 if (ret) { 574 dev_dbg(dev, 575 "...retrying[%d] inflight registration\n", 576 retry); 577 msleep(raw->desc->max_rx_timeout_ms / 578 SCMI_XFER_RAW_MAX_RETRIES); 579 } 580 } while (ret && --retry); 581 582 if (ret) { 583 dev_warn(dev, 584 "RAW - Could NOT register xfer %d in-flight HDR:0x%08X\n", 585 xfer->hdr.seq, msg_hdr); 586 scmi_xfer_raw_put(raw->handle, xfer); 587 } 588 589 return ret; 590 } 591 592 /** 593 * scmi_do_xfer_raw_start - An helper to send a valid raw xfer 594 * 595 * @raw: A reference to the Raw instance. 596 * @xfer: The xfer to send 597 * @chan_id: The channel ID to use, if zero the channels is automatically 598 * selected based on the protocol used. 599 * @async: A flag stating if an asynchronous command is required. 600 * 601 * This function send a previously built raw xfer using an appropriate channel 602 * and queues the related waiting work. 603 * 604 * Note that we need to know explicitly if the required command is meant to be 605 * asynchronous in kind since we have to properly setup the waiter. 606 * (and deducing this from the payload is weak and do not scale given there is 607 * NOT a common header-flag stating if the command is asynchronous or not) 608 * 609 * Return: 0 on Success 610 */ 611 static int scmi_do_xfer_raw_start(struct scmi_raw_mode_info *raw, 612 struct scmi_xfer *xfer, u8 chan_id, 613 bool async) 614 { 615 int ret; 616 struct scmi_chan_info *cinfo; 617 struct scmi_xfer_raw_waiter *rw; 618 struct device *dev = raw->handle->dev; 619 620 if (!chan_id) 621 chan_id = xfer->hdr.protocol_id; 622 else 623 xfer->flags |= SCMI_XFER_FLAG_CHAN_SET; 624 625 cinfo = scmi_xfer_raw_channel_get(raw->handle, chan_id); 626 if (IS_ERR(cinfo)) 627 return PTR_ERR(cinfo); 628 629 rw = scmi_xfer_raw_waiter_get(raw, xfer, cinfo, async); 630 if (!rw) { 631 dev_warn(dev, "RAW - Cannot get a free waiter !\n"); 632 return -ENOMEM; 633 } 634 635 /* True ONLY if also supported by transport. */ 636 if (is_polling_enabled(cinfo, raw->desc)) 637 xfer->hdr.poll_completion = true; 638 639 reinit_completion(&xfer->done); 640 /* Make sure xfer state update is visible before sending */ 641 smp_store_mb(xfer->state, SCMI_XFER_SENT_OK); 642 643 trace_scmi_xfer_begin(xfer->transfer_id, xfer->hdr.id, 644 xfer->hdr.protocol_id, xfer->hdr.seq, 645 xfer->hdr.poll_completion); 646 647 ret = raw->desc->ops->send_message(rw->cinfo, xfer); 648 if (ret) { 649 dev_err(dev, "Failed to send RAW message %d\n", ret); 650 scmi_xfer_raw_waiter_put(raw, rw); 651 return ret; 652 } 653 654 trace_scmi_msg_dump(raw->id, cinfo->id, xfer->hdr.protocol_id, 655 xfer->hdr.id, "cmnd", xfer->hdr.seq, 656 xfer->hdr.status, 657 xfer->tx.buf, xfer->tx.len); 658 659 scmi_xfer_raw_waiter_enqueue(raw, rw); 660 661 return ret; 662 } 663 664 /** 665 * scmi_raw_message_send - An helper to build and send an SCMI command using 666 * the provided SCMI bare message buffer 667 * 668 * @raw: A reference to the Raw instance. 669 * @buf: A buffer containing the whole SCMI message to send (including the 670 * header) in little-endian binary format. 671 * @len: Length of the message in @buf. 672 * @chan_id: The channel ID to use. 673 * @async: A flag stating if an asynchronous command is required. 674 * @poll: A flag stating if a polling transmission is required. 675 * 676 * Return: 0 on Success 677 */ 678 static int scmi_raw_message_send(struct scmi_raw_mode_info *raw, 679 void *buf, size_t len, u8 chan_id, 680 bool async, bool poll) 681 { 682 int ret; 683 struct scmi_xfer *xfer; 684 685 ret = scmi_xfer_raw_get_init(raw, buf, len, &xfer); 686 if (ret) 687 return ret; 688 689 if (poll) { 690 if (is_transport_polling_capable(raw->desc)) { 691 xfer->hdr.poll_completion = true; 692 } else { 693 dev_err(raw->handle->dev, 694 "Failed to send RAW message - Polling NOT supported\n"); 695 return -EINVAL; 696 } 697 } 698 699 ret = scmi_do_xfer_raw_start(raw, xfer, chan_id, async); 700 if (ret) 701 scmi_xfer_raw_put(raw->handle, xfer); 702 703 return ret; 704 } 705 706 static struct scmi_raw_buffer * 707 scmi_raw_message_dequeue(struct scmi_raw_queue *q, bool o_nonblock) 708 { 709 unsigned long flags; 710 struct scmi_raw_buffer *rb; 711 712 spin_lock_irqsave(&q->msg_q_lock, flags); 713 while (list_empty(&q->msg_q)) { 714 spin_unlock_irqrestore(&q->msg_q_lock, flags); 715 716 if (o_nonblock) 717 return ERR_PTR(-EAGAIN); 718 719 if (wait_event_interruptible(q->wq, !list_empty(&q->msg_q))) 720 return ERR_PTR(-ERESTARTSYS); 721 722 spin_lock_irqsave(&q->msg_q_lock, flags); 723 } 724 725 rb = scmi_raw_buffer_dequeue_unlocked(q); 726 727 spin_unlock_irqrestore(&q->msg_q_lock, flags); 728 729 return rb; 730 } 731 732 /** 733 * scmi_raw_message_receive - An helper to dequeue and report the next 734 * available enqueued raw message payload that has been collected. 735 * 736 * @raw: A reference to the Raw instance. 737 * @buf: A buffer to get hold of the whole SCMI message received and represented 738 * in little-endian binary format. 739 * @len: Length of @buf. 740 * @size: The effective size of the message copied into @buf 741 * @idx: The index of the queue to pick the next queued message from. 742 * @chan_id: The channel ID to use. 743 * @o_nonblock: A flag to request a non-blocking message dequeue. 744 * 745 * Return: 0 on Success 746 */ 747 static int scmi_raw_message_receive(struct scmi_raw_mode_info *raw, 748 void *buf, size_t len, size_t *size, 749 unsigned int idx, unsigned int chan_id, 750 bool o_nonblock) 751 { 752 int ret = 0; 753 struct scmi_raw_buffer *rb; 754 struct scmi_raw_queue *q; 755 756 q = scmi_raw_queue_select(raw, idx, chan_id); 757 if (!q) 758 return -ENODEV; 759 760 rb = scmi_raw_message_dequeue(q, o_nonblock); 761 if (IS_ERR(rb)) { 762 dev_dbg(raw->handle->dev, "RAW - No message available!\n"); 763 return PTR_ERR(rb); 764 } 765 766 if (rb->msg.len <= len) { 767 memcpy(buf, rb->msg.buf, rb->msg.len); 768 *size = rb->msg.len; 769 } else { 770 ret = -ENOSPC; 771 } 772 773 scmi_raw_buffer_put(q, rb); 774 775 return ret; 776 } 777 778 /* SCMI Raw debugfs helpers */ 779 780 static ssize_t scmi_dbg_raw_mode_common_read(struct file *filp, 781 char __user *buf, 782 size_t count, loff_t *ppos, 783 unsigned int idx) 784 { 785 ssize_t cnt; 786 struct scmi_dbg_raw_data *rd = filp->private_data; 787 788 if (!rd->rx_size) { 789 int ret; 790 791 ret = scmi_raw_message_receive(rd->raw, rd->rx.buf, rd->rx.len, 792 &rd->rx_size, idx, rd->chan_id, 793 filp->f_flags & O_NONBLOCK); 794 if (ret) { 795 rd->rx_size = 0; 796 return ret; 797 } 798 799 /* Reset any previous filepos change, including writes */ 800 *ppos = 0; 801 } else if (*ppos == rd->rx_size) { 802 /* Return EOF once all the message has been read-out */ 803 rd->rx_size = 0; 804 return 0; 805 } 806 807 cnt = simple_read_from_buffer(buf, count, ppos, 808 rd->rx.buf, rd->rx_size); 809 810 return cnt; 811 } 812 813 static ssize_t scmi_dbg_raw_mode_common_write(struct file *filp, 814 const char __user *buf, 815 size_t count, loff_t *ppos, 816 bool async, bool poll) 817 { 818 int ret; 819 struct scmi_dbg_raw_data *rd = filp->private_data; 820 821 if (count > rd->tx.len - rd->tx_size) 822 return -ENOSPC; 823 824 /* On first write attempt @count carries the total full message size. */ 825 if (!rd->tx_size) 826 rd->tx_req_size = count; 827 828 /* 829 * Gather a full message, possibly across multiple interrupted wrrtes, 830 * before sending it with a single RAW xfer. 831 */ 832 if (rd->tx_size < rd->tx_req_size) { 833 ssize_t cnt; 834 835 cnt = simple_write_to_buffer(rd->tx.buf, rd->tx.len, ppos, 836 buf, count); 837 if (cnt < 0) 838 return cnt; 839 840 rd->tx_size += cnt; 841 if (cnt < count) 842 return cnt; 843 } 844 845 ret = scmi_raw_message_send(rd->raw, rd->tx.buf, rd->tx_size, 846 rd->chan_id, async, poll); 847 848 /* Reset ppos for next message ... */ 849 rd->tx_size = 0; 850 *ppos = 0; 851 852 return ret ?: count; 853 } 854 855 static __poll_t scmi_test_dbg_raw_common_poll(struct file *filp, 856 struct poll_table_struct *wait, 857 unsigned int idx) 858 { 859 unsigned long flags; 860 struct scmi_dbg_raw_data *rd = filp->private_data; 861 struct scmi_raw_queue *q; 862 __poll_t mask = 0; 863 864 q = scmi_raw_queue_select(rd->raw, idx, rd->chan_id); 865 if (!q) 866 return mask; 867 868 poll_wait(filp, &q->wq, wait); 869 870 spin_lock_irqsave(&q->msg_q_lock, flags); 871 if (!list_empty(&q->msg_q)) 872 mask = EPOLLIN | EPOLLRDNORM; 873 spin_unlock_irqrestore(&q->msg_q_lock, flags); 874 875 return mask; 876 } 877 878 static ssize_t scmi_dbg_raw_mode_message_read(struct file *filp, 879 char __user *buf, 880 size_t count, loff_t *ppos) 881 { 882 return scmi_dbg_raw_mode_common_read(filp, buf, count, ppos, 883 SCMI_RAW_REPLY_QUEUE); 884 } 885 886 static ssize_t scmi_dbg_raw_mode_message_write(struct file *filp, 887 const char __user *buf, 888 size_t count, loff_t *ppos) 889 { 890 return scmi_dbg_raw_mode_common_write(filp, buf, count, ppos, 891 false, false); 892 } 893 894 static __poll_t scmi_dbg_raw_mode_message_poll(struct file *filp, 895 struct poll_table_struct *wait) 896 { 897 return scmi_test_dbg_raw_common_poll(filp, wait, SCMI_RAW_REPLY_QUEUE); 898 } 899 900 static int scmi_dbg_raw_mode_open(struct inode *inode, struct file *filp) 901 { 902 struct scmi_raw_mode_info *raw; 903 struct scmi_dbg_raw_data *rd; 904 905 if (!inode->i_private) 906 return -ENODEV; 907 908 raw = inode->i_private; 909 rd = kzalloc(sizeof(*rd), GFP_KERNEL); 910 if (!rd) 911 return -ENOMEM; 912 913 rd->rx.len = raw->desc->max_msg_size + sizeof(u32); 914 rd->rx.buf = kzalloc(rd->rx.len, GFP_KERNEL); 915 if (!rd->rx.buf) { 916 kfree(rd); 917 return -ENOMEM; 918 } 919 920 rd->tx.len = raw->desc->max_msg_size + sizeof(u32); 921 rd->tx.buf = kzalloc(rd->tx.len, GFP_KERNEL); 922 if (!rd->tx.buf) { 923 kfree(rd->rx.buf); 924 kfree(rd); 925 return -ENOMEM; 926 } 927 928 /* Grab channel ID from debugfs entry naming if any */ 929 /* not set - reassing 0 we already had after kzalloc() */ 930 rd->chan_id = debugfs_get_aux_num(filp); 931 932 rd->raw = raw; 933 filp->private_data = rd; 934 935 return nonseekable_open(inode, filp); 936 } 937 938 static int scmi_dbg_raw_mode_release(struct inode *inode, struct file *filp) 939 { 940 struct scmi_dbg_raw_data *rd = filp->private_data; 941 942 kfree(rd->rx.buf); 943 kfree(rd->tx.buf); 944 kfree(rd); 945 946 return 0; 947 } 948 949 static ssize_t scmi_dbg_raw_mode_reset_write(struct file *filp, 950 const char __user *buf, 951 size_t count, loff_t *ppos) 952 { 953 struct scmi_dbg_raw_data *rd = filp->private_data; 954 955 scmi_xfer_raw_reset(rd->raw); 956 957 return count; 958 } 959 960 static const struct file_operations scmi_dbg_raw_mode_reset_fops = { 961 .open = scmi_dbg_raw_mode_open, 962 .release = scmi_dbg_raw_mode_release, 963 .write = scmi_dbg_raw_mode_reset_write, 964 .owner = THIS_MODULE, 965 }; 966 967 static const struct file_operations scmi_dbg_raw_mode_message_fops = { 968 .open = scmi_dbg_raw_mode_open, 969 .release = scmi_dbg_raw_mode_release, 970 .read = scmi_dbg_raw_mode_message_read, 971 .write = scmi_dbg_raw_mode_message_write, 972 .poll = scmi_dbg_raw_mode_message_poll, 973 .owner = THIS_MODULE, 974 }; 975 976 static ssize_t scmi_dbg_raw_mode_message_async_write(struct file *filp, 977 const char __user *buf, 978 size_t count, loff_t *ppos) 979 { 980 return scmi_dbg_raw_mode_common_write(filp, buf, count, ppos, 981 true, false); 982 } 983 984 static const struct file_operations scmi_dbg_raw_mode_message_async_fops = { 985 .open = scmi_dbg_raw_mode_open, 986 .release = scmi_dbg_raw_mode_release, 987 .read = scmi_dbg_raw_mode_message_read, 988 .write = scmi_dbg_raw_mode_message_async_write, 989 .poll = scmi_dbg_raw_mode_message_poll, 990 .owner = THIS_MODULE, 991 }; 992 993 static ssize_t scmi_dbg_raw_mode_message_poll_write(struct file *filp, 994 const char __user *buf, 995 size_t count, loff_t *ppos) 996 { 997 return scmi_dbg_raw_mode_common_write(filp, buf, count, ppos, 998 false, true); 999 } 1000 1001 static const struct file_operations scmi_dbg_raw_mode_message_poll_fops = { 1002 .open = scmi_dbg_raw_mode_open, 1003 .release = scmi_dbg_raw_mode_release, 1004 .read = scmi_dbg_raw_mode_message_read, 1005 .write = scmi_dbg_raw_mode_message_poll_write, 1006 .poll = scmi_dbg_raw_mode_message_poll, 1007 .owner = THIS_MODULE, 1008 }; 1009 1010 static ssize_t scmi_dbg_raw_mode_message_poll_async_write(struct file *filp, 1011 const char __user *buf, 1012 size_t count, loff_t *ppos) 1013 { 1014 return scmi_dbg_raw_mode_common_write(filp, buf, count, ppos, 1015 true, true); 1016 } 1017 1018 static const struct file_operations scmi_dbg_raw_mode_message_poll_async_fops = { 1019 .open = scmi_dbg_raw_mode_open, 1020 .release = scmi_dbg_raw_mode_release, 1021 .read = scmi_dbg_raw_mode_message_read, 1022 .write = scmi_dbg_raw_mode_message_poll_async_write, 1023 .poll = scmi_dbg_raw_mode_message_poll, 1024 .owner = THIS_MODULE, 1025 }; 1026 1027 static ssize_t scmi_test_dbg_raw_mode_notif_read(struct file *filp, 1028 char __user *buf, 1029 size_t count, loff_t *ppos) 1030 { 1031 return scmi_dbg_raw_mode_common_read(filp, buf, count, ppos, 1032 SCMI_RAW_NOTIF_QUEUE); 1033 } 1034 1035 static __poll_t 1036 scmi_test_dbg_raw_mode_notif_poll(struct file *filp, 1037 struct poll_table_struct *wait) 1038 { 1039 return scmi_test_dbg_raw_common_poll(filp, wait, SCMI_RAW_NOTIF_QUEUE); 1040 } 1041 1042 static const struct file_operations scmi_dbg_raw_mode_notification_fops = { 1043 .open = scmi_dbg_raw_mode_open, 1044 .release = scmi_dbg_raw_mode_release, 1045 .read = scmi_test_dbg_raw_mode_notif_read, 1046 .poll = scmi_test_dbg_raw_mode_notif_poll, 1047 .owner = THIS_MODULE, 1048 }; 1049 1050 static ssize_t scmi_test_dbg_raw_mode_errors_read(struct file *filp, 1051 char __user *buf, 1052 size_t count, loff_t *ppos) 1053 { 1054 return scmi_dbg_raw_mode_common_read(filp, buf, count, ppos, 1055 SCMI_RAW_ERRS_QUEUE); 1056 } 1057 1058 static __poll_t 1059 scmi_test_dbg_raw_mode_errors_poll(struct file *filp, 1060 struct poll_table_struct *wait) 1061 { 1062 return scmi_test_dbg_raw_common_poll(filp, wait, SCMI_RAW_ERRS_QUEUE); 1063 } 1064 1065 static const struct file_operations scmi_dbg_raw_mode_errors_fops = { 1066 .open = scmi_dbg_raw_mode_open, 1067 .release = scmi_dbg_raw_mode_release, 1068 .read = scmi_test_dbg_raw_mode_errors_read, 1069 .poll = scmi_test_dbg_raw_mode_errors_poll, 1070 .owner = THIS_MODULE, 1071 }; 1072 1073 static struct scmi_raw_queue * 1074 scmi_raw_queue_init(struct scmi_raw_mode_info *raw) 1075 { 1076 int i; 1077 struct scmi_raw_buffer *rb; 1078 struct device *dev = raw->handle->dev; 1079 struct scmi_raw_queue *q; 1080 1081 q = devm_kzalloc(dev, sizeof(*q), GFP_KERNEL); 1082 if (!q) 1083 return ERR_PTR(-ENOMEM); 1084 1085 rb = devm_kcalloc(dev, raw->tx_max_msg, sizeof(*rb), GFP_KERNEL); 1086 if (!rb) 1087 return ERR_PTR(-ENOMEM); 1088 1089 spin_lock_init(&q->free_bufs_lock); 1090 INIT_LIST_HEAD(&q->free_bufs); 1091 for (i = 0; i < raw->tx_max_msg; i++, rb++) { 1092 rb->max_len = raw->desc->max_msg_size + sizeof(u32); 1093 rb->msg.buf = devm_kzalloc(dev, rb->max_len, GFP_KERNEL); 1094 if (!rb->msg.buf) 1095 return ERR_PTR(-ENOMEM); 1096 scmi_raw_buffer_put(q, rb); 1097 } 1098 1099 spin_lock_init(&q->msg_q_lock); 1100 INIT_LIST_HEAD(&q->msg_q); 1101 init_waitqueue_head(&q->wq); 1102 1103 return q; 1104 } 1105 1106 static int scmi_xfer_raw_worker_init(struct scmi_raw_mode_info *raw) 1107 { 1108 int i; 1109 struct scmi_xfer_raw_waiter *rw; 1110 struct device *dev = raw->handle->dev; 1111 1112 rw = devm_kcalloc(dev, raw->tx_max_msg, sizeof(*rw), GFP_KERNEL); 1113 if (!rw) 1114 return -ENOMEM; 1115 1116 raw->wait_wq = alloc_workqueue("scmi-raw-wait-wq-%d", 1117 WQ_UNBOUND | WQ_FREEZABLE | 1118 WQ_HIGHPRI | WQ_SYSFS, 0, raw->id); 1119 if (!raw->wait_wq) 1120 return -ENOMEM; 1121 1122 mutex_init(&raw->free_mtx); 1123 INIT_LIST_HEAD(&raw->free_waiters); 1124 mutex_init(&raw->active_mtx); 1125 INIT_LIST_HEAD(&raw->active_waiters); 1126 1127 for (i = 0; i < raw->tx_max_msg; i++, rw++) { 1128 init_completion(&rw->async_response); 1129 scmi_xfer_raw_waiter_put(raw, rw); 1130 } 1131 INIT_WORK(&raw->waiters_work, scmi_xfer_raw_worker); 1132 1133 return 0; 1134 } 1135 1136 static int scmi_raw_mode_setup(struct scmi_raw_mode_info *raw, 1137 u8 *channels, int num_chans) 1138 { 1139 int ret, idx; 1140 void *gid; 1141 struct device *dev = raw->handle->dev; 1142 1143 gid = devres_open_group(dev, NULL, GFP_KERNEL); 1144 if (!gid) 1145 return -ENOMEM; 1146 1147 for (idx = 0; idx < SCMI_RAW_MAX_QUEUE; idx++) { 1148 raw->q[idx] = scmi_raw_queue_init(raw); 1149 if (IS_ERR(raw->q[idx])) { 1150 ret = PTR_ERR(raw->q[idx]); 1151 goto err; 1152 } 1153 } 1154 1155 xa_init(&raw->chans_q); 1156 if (num_chans > 1) { 1157 int i; 1158 1159 for (i = 0; i < num_chans; i++) { 1160 struct scmi_raw_queue *q; 1161 1162 q = scmi_raw_queue_init(raw); 1163 if (IS_ERR(q)) { 1164 ret = PTR_ERR(q); 1165 goto err_xa; 1166 } 1167 1168 ret = xa_insert(&raw->chans_q, channels[i], q, 1169 GFP_KERNEL); 1170 if (ret) { 1171 dev_err(dev, 1172 "Fail to allocate Raw queue 0x%02X\n", 1173 channels[i]); 1174 goto err_xa; 1175 } 1176 } 1177 } 1178 1179 ret = scmi_xfer_raw_worker_init(raw); 1180 if (ret) 1181 goto err_xa; 1182 1183 devres_close_group(dev, gid); 1184 raw->gid = gid; 1185 1186 return 0; 1187 1188 err_xa: 1189 xa_destroy(&raw->chans_q); 1190 err: 1191 devres_release_group(dev, gid); 1192 return ret; 1193 } 1194 1195 /** 1196 * scmi_raw_mode_init - Function to initialize the SCMI Raw stack 1197 * 1198 * @handle: Pointer to SCMI entity handle 1199 * @top_dentry: A reference to the top Raw debugfs dentry 1200 * @instance_id: The ID of the underlying SCMI platform instance represented by 1201 * this Raw instance 1202 * @channels: The list of the existing channels 1203 * @num_chans: The number of entries in @channels 1204 * @desc: Reference to the transport operations 1205 * @tx_max_msg: Max number of in-flight messages allowed by the transport 1206 * 1207 * This function prepare the SCMI Raw stack and creates the debugfs API. 1208 * 1209 * Return: An opaque handle to the Raw instance on Success, an ERR_PTR otherwise 1210 */ 1211 void *scmi_raw_mode_init(const struct scmi_handle *handle, 1212 struct dentry *top_dentry, int instance_id, 1213 u8 *channels, int num_chans, 1214 const struct scmi_desc *desc, int tx_max_msg) 1215 { 1216 int ret; 1217 struct scmi_raw_mode_info *raw; 1218 struct device *dev; 1219 1220 if (!handle || !desc) 1221 return ERR_PTR(-EINVAL); 1222 1223 dev = handle->dev; 1224 raw = devm_kzalloc(dev, sizeof(*raw), GFP_KERNEL); 1225 if (!raw) 1226 return ERR_PTR(-ENOMEM); 1227 1228 raw->handle = handle; 1229 raw->desc = desc; 1230 raw->tx_max_msg = tx_max_msg; 1231 raw->id = instance_id; 1232 1233 ret = scmi_raw_mode_setup(raw, channels, num_chans); 1234 if (ret) { 1235 devm_kfree(dev, raw); 1236 return ERR_PTR(ret); 1237 } 1238 1239 raw->dentry = debugfs_create_dir("raw", top_dentry); 1240 1241 debugfs_create_file("reset", 0200, raw->dentry, raw, 1242 &scmi_dbg_raw_mode_reset_fops); 1243 1244 debugfs_create_file("message", 0600, raw->dentry, raw, 1245 &scmi_dbg_raw_mode_message_fops); 1246 1247 debugfs_create_file("message_async", 0600, raw->dentry, raw, 1248 &scmi_dbg_raw_mode_message_async_fops); 1249 1250 debugfs_create_file("message_poll", 0600, raw->dentry, raw, 1251 &scmi_dbg_raw_mode_message_poll_fops); 1252 1253 debugfs_create_file("message_poll_async", 0600, raw->dentry, raw, 1254 &scmi_dbg_raw_mode_message_poll_async_fops); 1255 1256 debugfs_create_file("notification", 0400, raw->dentry, raw, 1257 &scmi_dbg_raw_mode_notification_fops); 1258 1259 debugfs_create_file("errors", 0400, raw->dentry, raw, 1260 &scmi_dbg_raw_mode_errors_fops); 1261 1262 /* 1263 * Expose per-channel entries if multiple channels available. 1264 * Just ignore errors while setting up these interfaces since we 1265 * have anyway already a working core Raw support. 1266 */ 1267 if (num_chans > 1) { 1268 int i; 1269 struct dentry *top_chans; 1270 1271 top_chans = debugfs_create_dir("channels", raw->dentry); 1272 1273 for (i = 0; i < num_chans; i++) { 1274 char cdir[8]; 1275 struct dentry *chd; 1276 1277 snprintf(cdir, 8, "0x%02X", channels[i]); 1278 chd = debugfs_create_dir(cdir, top_chans); 1279 1280 debugfs_create_file_aux_num("message", 0600, chd, 1281 raw, channels[i], 1282 &scmi_dbg_raw_mode_message_fops); 1283 1284 debugfs_create_file_aux_num("message_async", 0600, chd, 1285 raw, channels[i], 1286 &scmi_dbg_raw_mode_message_async_fops); 1287 1288 debugfs_create_file_aux_num("message_poll", 0600, chd, 1289 raw, channels[i], 1290 &scmi_dbg_raw_mode_message_poll_fops); 1291 1292 debugfs_create_file_aux_num("message_poll_async", 0600, 1293 chd, raw, channels[i], 1294 &scmi_dbg_raw_mode_message_poll_async_fops); 1295 } 1296 } 1297 1298 dev_info(dev, "SCMI RAW Mode initialized for instance %d\n", raw->id); 1299 1300 return raw; 1301 } 1302 1303 /** 1304 * scmi_raw_mode_cleanup - Function to cleanup the SCMI Raw stack 1305 * 1306 * @r: An opaque handle to an initialized SCMI Raw instance 1307 */ 1308 void scmi_raw_mode_cleanup(void *r) 1309 { 1310 struct scmi_raw_mode_info *raw = r; 1311 1312 if (!raw) 1313 return; 1314 1315 debugfs_remove_recursive(raw->dentry); 1316 1317 cancel_work_sync(&raw->waiters_work); 1318 destroy_workqueue(raw->wait_wq); 1319 xa_destroy(&raw->chans_q); 1320 } 1321 1322 static int scmi_xfer_raw_collect(void *msg, size_t *msg_len, 1323 struct scmi_xfer *xfer) 1324 { 1325 __le32 *m; 1326 size_t msg_size; 1327 1328 if (!xfer || !msg || !msg_len) 1329 return -EINVAL; 1330 1331 /* Account for hdr ...*/ 1332 msg_size = xfer->rx.len + sizeof(u32); 1333 /* ... and status if needed */ 1334 if (xfer->hdr.type != MSG_TYPE_NOTIFICATION) 1335 msg_size += sizeof(u32); 1336 1337 if (msg_size > *msg_len) 1338 return -ENOSPC; 1339 1340 m = msg; 1341 *m = cpu_to_le32(pack_scmi_header(&xfer->hdr)); 1342 if (xfer->hdr.type != MSG_TYPE_NOTIFICATION) 1343 *++m = cpu_to_le32(xfer->hdr.status); 1344 1345 memcpy(++m, xfer->rx.buf, xfer->rx.len); 1346 1347 *msg_len = msg_size; 1348 1349 return 0; 1350 } 1351 1352 /** 1353 * scmi_raw_message_report - Helper to report back valid reponses/notifications 1354 * to raw message requests. 1355 * 1356 * @r: An opaque reference to the raw instance configuration 1357 * @xfer: The xfer containing the message to be reported 1358 * @idx: The index of the queue. 1359 * @chan_id: The channel ID to use. 1360 * 1361 * If Raw mode is enabled, this is called from the SCMI core on the regular RX 1362 * path to save and enqueue the response/notification payload carried by this 1363 * xfer into a dedicated scmi_raw_buffer for later consumption by the user. 1364 * 1365 * This way the caller can free the related xfer immediately afterwards and the 1366 * user can read back the raw message payload at its own pace (if ever) without 1367 * holding an xfer for too long. 1368 */ 1369 void scmi_raw_message_report(void *r, struct scmi_xfer *xfer, 1370 unsigned int idx, unsigned int chan_id) 1371 { 1372 int ret; 1373 unsigned long flags; 1374 struct scmi_raw_buffer *rb; 1375 struct device *dev; 1376 struct scmi_raw_queue *q; 1377 struct scmi_raw_mode_info *raw = r; 1378 1379 if (!raw || (idx == SCMI_RAW_REPLY_QUEUE && !SCMI_XFER_IS_RAW(xfer))) 1380 return; 1381 1382 dev = raw->handle->dev; 1383 q = scmi_raw_queue_select(raw, idx, 1384 SCMI_XFER_IS_CHAN_SET(xfer) ? chan_id : 0); 1385 if (!q) { 1386 dev_warn(dev, 1387 "RAW[%d] - NO queue for chan 0x%X. Dropping report.\n", 1388 idx, chan_id); 1389 return; 1390 } 1391 1392 /* 1393 * Grab the msg_q_lock upfront to avoid a possible race between 1394 * realizing the free list was empty and effectively picking the next 1395 * buffer to use from the oldest one enqueued and still unread on this 1396 * msg_q. 1397 * 1398 * Note that nowhere else these locks are taken together, so no risk of 1399 * deadlocks du eto inversion. 1400 */ 1401 spin_lock_irqsave(&q->msg_q_lock, flags); 1402 rb = scmi_raw_buffer_get(q); 1403 if (!rb) { 1404 /* 1405 * Immediate and delayed replies to previously injected Raw 1406 * commands MUST be read back from userspace to free the buffers: 1407 * if this is not happening something is seriously broken and 1408 * must be fixed at the application level: complain loudly. 1409 */ 1410 if (idx == SCMI_RAW_REPLY_QUEUE) { 1411 spin_unlock_irqrestore(&q->msg_q_lock, flags); 1412 dev_warn(dev, 1413 "RAW[%d] - Buffers exhausted. Dropping report.\n", 1414 idx); 1415 return; 1416 } 1417 1418 /* 1419 * Notifications and errors queues are instead handled in a 1420 * circular manner: unread old buffers are just overwritten by 1421 * newer ones. 1422 * 1423 * The main reason for this is that notifications originated 1424 * by Raw requests cannot be distinguished from normal ones, so 1425 * your Raw buffers queues risk to be flooded and depleted by 1426 * notifications if you left it mistakenly enabled or when in 1427 * coexistence mode. 1428 */ 1429 rb = scmi_raw_buffer_dequeue_unlocked(q); 1430 if (WARN_ON(!rb)) { 1431 spin_unlock_irqrestore(&q->msg_q_lock, flags); 1432 return; 1433 } 1434 1435 /* Reset to full buffer length */ 1436 rb->msg.len = rb->max_len; 1437 1438 dev_warn_once(dev, 1439 "RAW[%d] - Buffers exhausted. Re-using oldest.\n", 1440 idx); 1441 } 1442 spin_unlock_irqrestore(&q->msg_q_lock, flags); 1443 1444 ret = scmi_xfer_raw_collect(rb->msg.buf, &rb->msg.len, xfer); 1445 if (ret) { 1446 dev_warn(dev, "RAW - Cannot collect xfer into buffer !\n"); 1447 scmi_raw_buffer_put(q, rb); 1448 return; 1449 } 1450 1451 scmi_raw_buffer_enqueue(q, rb); 1452 } 1453 1454 static void scmi_xfer_raw_fill(struct scmi_raw_mode_info *raw, 1455 struct scmi_chan_info *cinfo, 1456 struct scmi_xfer *xfer, u32 msg_hdr) 1457 { 1458 /* Unpack received HDR as it is */ 1459 unpack_scmi_header(msg_hdr, &xfer->hdr); 1460 xfer->hdr.seq = MSG_XTRACT_TOKEN(msg_hdr); 1461 1462 memset(xfer->rx.buf, 0x00, xfer->rx.len); 1463 1464 raw->desc->ops->fetch_response(cinfo, xfer); 1465 } 1466 1467 /** 1468 * scmi_raw_error_report - Helper to report back timed-out or generally 1469 * unexpected replies. 1470 * 1471 * @r: An opaque reference to the raw instance configuration 1472 * @cinfo: A reference to the channel to use to retrieve the broken xfer 1473 * @msg_hdr: The SCMI message header of the message to fetch and report 1474 * @priv: Any private data related to the xfer. 1475 * 1476 * If Raw mode is enabled, this is called from the SCMI core on the RX path in 1477 * case of errors to save and enqueue the bad message payload carried by the 1478 * message that has just been received. 1479 * 1480 * Note that we have to manually fetch any available payload into a temporary 1481 * xfer to be able to save and enqueue the message, since the regular RX error 1482 * path which had called this would have not fetched the message payload having 1483 * classified it as an error. 1484 */ 1485 void scmi_raw_error_report(void *r, struct scmi_chan_info *cinfo, 1486 u32 msg_hdr, void *priv) 1487 { 1488 struct scmi_xfer xfer; 1489 struct scmi_raw_mode_info *raw = r; 1490 1491 if (!raw) 1492 return; 1493 1494 xfer.rx.len = raw->desc->max_msg_size; 1495 xfer.rx.buf = kzalloc(xfer.rx.len, GFP_ATOMIC); 1496 if (!xfer.rx.buf) { 1497 dev_info(raw->handle->dev, 1498 "Cannot report Raw error for HDR:0x%X - ENOMEM\n", 1499 msg_hdr); 1500 return; 1501 } 1502 1503 /* Any transport-provided priv must be passed back down to transport */ 1504 if (priv) 1505 /* Ensure priv is visible */ 1506 smp_store_mb(xfer.priv, priv); 1507 1508 scmi_xfer_raw_fill(raw, cinfo, &xfer, msg_hdr); 1509 scmi_raw_message_report(raw, &xfer, SCMI_RAW_ERRS_QUEUE, 0); 1510 1511 kfree(xfer.rx.buf); 1512 } 1513