1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ 2 /* Copyright (c) 2010-2012 Broadcom. All rights reserved. */ 3 4 #ifndef VCHIQ_CORE_H 5 #define VCHIQ_CORE_H 6 7 #include <linux/mutex.h> 8 #include <linux/completion.h> 9 #include <linux/debugfs.h> 10 #include <linux/dev_printk.h> 11 #include <linux/kthread.h> 12 #include <linux/kref.h> 13 #include <linux/rcupdate.h> 14 #include <linux/wait.h> 15 16 #include "../../include/linux/raspberrypi/vchiq.h" 17 #include "vchiq_cfg.h" 18 19 /* Do this so that we can test-build the code on non-rpi systems */ 20 #if IS_ENABLED(CONFIG_RASPBERRYPI_FIRMWARE) 21 22 #else 23 24 #ifndef dsb 25 #define dsb(a) 26 #endif 27 28 #endif /* IS_ENABLED(CONFIG_RASPBERRYPI_FIRMWARE) */ 29 30 #define VCHIQ_SERVICE_HANDLE_INVALID 0 31 32 #define VCHIQ_SLOT_SIZE 4096 33 #define VCHIQ_MAX_MSG_SIZE (VCHIQ_SLOT_SIZE - sizeof(struct vchiq_header)) 34 35 #define VCHIQ_SLOT_MASK (VCHIQ_SLOT_SIZE - 1) 36 #define VCHIQ_SLOT_QUEUE_MASK (VCHIQ_MAX_SLOTS_PER_SIDE - 1) 37 #define VCHIQ_SLOT_ZERO_SLOTS DIV_ROUND_UP(sizeof(struct vchiq_slot_zero), \ 38 VCHIQ_SLOT_SIZE) 39 40 #define BITSET_SIZE(b) ((b + 31) >> 5) 41 #define BITSET_WORD(b) (b >> 5) 42 #define BITSET_BIT(b) (1 << (b & 31)) 43 #define BITSET_IS_SET(bs, b) (bs[BITSET_WORD(b)] & BITSET_BIT(b)) 44 #define BITSET_SET(bs, b) (bs[BITSET_WORD(b)] |= BITSET_BIT(b)) 45 46 enum { 47 DEBUG_ENTRIES, 48 #if VCHIQ_ENABLE_DEBUG 49 DEBUG_SLOT_HANDLER_COUNT, 50 DEBUG_SLOT_HANDLER_LINE, 51 DEBUG_PARSE_LINE, 52 DEBUG_PARSE_HEADER, 53 DEBUG_PARSE_MSGID, 54 DEBUG_AWAIT_COMPLETION_LINE, 55 DEBUG_DEQUEUE_MESSAGE_LINE, 56 DEBUG_SERVICE_CALLBACK_LINE, 57 DEBUG_MSG_QUEUE_FULL_COUNT, 58 DEBUG_COMPLETION_QUEUE_FULL_COUNT, 59 #endif 60 DEBUG_MAX 61 }; 62 63 #if VCHIQ_ENABLE_DEBUG 64 65 #define DEBUG_INITIALISE(local) int *debug_ptr = (local)->debug 66 #define DEBUG_TRACE(d) \ 67 do { debug_ptr[DEBUG_ ## d] = __LINE__; dsb(sy); } while (0) 68 #define DEBUG_VALUE(d, v) \ 69 do { debug_ptr[DEBUG_ ## d] = (v); dsb(sy); } while (0) 70 #define DEBUG_COUNT(d) \ 71 do { debug_ptr[DEBUG_ ## d]++; dsb(sy); } while (0) 72 73 #else /* VCHIQ_ENABLE_DEBUG */ 74 75 #define DEBUG_INITIALISE(local) 76 #define DEBUG_TRACE(d) 77 #define DEBUG_VALUE(d, v) 78 #define DEBUG_COUNT(d) 79 80 #endif /* VCHIQ_ENABLE_DEBUG */ 81 82 enum vchiq_connstate { 83 VCHIQ_CONNSTATE_DISCONNECTED, 84 VCHIQ_CONNSTATE_CONNECTING, 85 VCHIQ_CONNSTATE_CONNECTED, 86 VCHIQ_CONNSTATE_PAUSING, 87 VCHIQ_CONNSTATE_PAUSE_SENT, 88 VCHIQ_CONNSTATE_PAUSED, 89 VCHIQ_CONNSTATE_RESUMING, 90 VCHIQ_CONNSTATE_PAUSE_TIMEOUT, 91 VCHIQ_CONNSTATE_RESUME_TIMEOUT 92 }; 93 94 enum { 95 VCHIQ_SRVSTATE_FREE, 96 VCHIQ_SRVSTATE_HIDDEN, 97 VCHIQ_SRVSTATE_LISTENING, 98 VCHIQ_SRVSTATE_OPENING, 99 VCHIQ_SRVSTATE_OPEN, 100 VCHIQ_SRVSTATE_OPENSYNC, 101 VCHIQ_SRVSTATE_CLOSESENT, 102 VCHIQ_SRVSTATE_CLOSERECVD, 103 VCHIQ_SRVSTATE_CLOSEWAIT, 104 VCHIQ_SRVSTATE_CLOSED 105 }; 106 107 enum vchiq_bulk_dir { 108 VCHIQ_BULK_TRANSMIT, 109 VCHIQ_BULK_RECEIVE 110 }; 111 112 struct vchiq_bulk { 113 short mode; 114 short dir; 115 void *userdata; 116 dma_addr_t data; 117 int size; 118 void *remote_data; 119 int remote_size; 120 int actual; 121 }; 122 123 struct vchiq_bulk_queue { 124 int local_insert; /* Where to insert the next local bulk */ 125 int remote_insert; /* Where to insert the next remote bulk (master) */ 126 int process; /* Bulk to transfer next */ 127 int remote_notify; /* Bulk to notify the remote client of next (mstr) */ 128 int remove; /* Bulk to notify the local client of, and remove, next */ 129 struct vchiq_bulk bulks[VCHIQ_NUM_SERVICE_BULKS]; 130 }; 131 132 /* 133 * Remote events provide a way of presenting several virtual doorbells to a 134 * peer (ARM host to VPU) using only one physical doorbell. They can be thought 135 * of as a way for the peer to signal a semaphore, in this case implemented as 136 * a workqueue. 137 * 138 * Remote events remain signalled until acknowledged by the receiver, and they 139 * are non-counting. They are designed in such a way as to minimise the number 140 * of interrupts and avoid unnecessary waiting. 141 * 142 * A remote_event is as small data structures that live in shared memory. It 143 * comprises two booleans - armed and fired: 144 * 145 * The sender sets fired when they signal the receiver. 146 * If fired is set, the receiver has been signalled and need not wait. 147 * The receiver sets the armed field before they begin to wait. 148 * If armed is set, the receiver is waiting and wishes to be woken by interrupt. 149 */ 150 struct remote_event { 151 int armed; 152 int fired; 153 u32 __unused; 154 }; 155 156 struct opaque_platform_state; 157 158 struct vchiq_slot { 159 char data[VCHIQ_SLOT_SIZE]; 160 }; 161 162 struct vchiq_slot_info { 163 /* Use two counters rather than one to avoid the need for a mutex. */ 164 short use_count; 165 short release_count; 166 }; 167 168 struct vchiq_service { 169 struct vchiq_service_base base; 170 unsigned int handle; 171 struct kref ref_count; 172 struct rcu_head rcu; 173 int srvstate; 174 void (*userdata_term)(void *userdata); 175 unsigned int localport; 176 unsigned int remoteport; 177 int public_fourcc; 178 int client_id; 179 char auto_close; 180 char sync; 181 char closing; 182 char trace; 183 atomic_t poll_flags; 184 short version; 185 short version_min; 186 short peer_version; 187 188 struct vchiq_state *state; 189 struct vchiq_instance *instance; 190 191 int service_use_count; 192 193 struct vchiq_bulk_queue bulk_tx; 194 struct vchiq_bulk_queue bulk_rx; 195 196 struct completion remove_event; 197 struct completion bulk_remove_event; 198 struct mutex bulk_mutex; 199 200 struct service_stats_struct { 201 int quota_stalls; 202 int slot_stalls; 203 int bulk_stalls; 204 int error_count; 205 int ctrl_tx_count; 206 int ctrl_rx_count; 207 int bulk_tx_count; 208 int bulk_rx_count; 209 int bulk_aborted_count; 210 u64 ctrl_tx_bytes; 211 u64 ctrl_rx_bytes; 212 u64 bulk_tx_bytes; 213 u64 bulk_rx_bytes; 214 } stats; 215 216 int msg_queue_read; 217 int msg_queue_write; 218 struct completion msg_queue_pop; 219 struct completion msg_queue_push; 220 struct vchiq_header *msg_queue[VCHIQ_MAX_SLOTS]; 221 }; 222 223 /* 224 * The quota information is outside struct vchiq_service so that it can 225 * be statically allocated, since for accounting reasons a service's slot 226 * usage is carried over between users of the same port number. 227 */ 228 struct vchiq_service_quota { 229 unsigned short slot_quota; 230 unsigned short slot_use_count; 231 unsigned short message_quota; 232 unsigned short message_use_count; 233 struct completion quota_event; 234 int previous_tx_index; 235 }; 236 237 struct vchiq_shared_state { 238 /* A non-zero value here indicates that the content is valid. */ 239 int initialised; 240 241 /* The first and last (inclusive) slots allocated to the owner. */ 242 int slot_first; 243 int slot_last; 244 245 /* The slot allocated to synchronous messages from the owner. */ 246 int slot_sync; 247 248 /* 249 * Signalling this event indicates that owner's slot handler thread 250 * should run. 251 */ 252 struct remote_event trigger; 253 254 /* 255 * Indicates the byte position within the stream where the next message 256 * will be written. The least significant bits are an index into the 257 * slot. The next bits are the index of the slot in slot_queue. 258 */ 259 int tx_pos; 260 261 /* This event should be signalled when a slot is recycled. */ 262 struct remote_event recycle; 263 264 /* The slot_queue index where the next recycled slot will be written. */ 265 int slot_queue_recycle; 266 267 /* This event should be signalled when a synchronous message is sent. */ 268 struct remote_event sync_trigger; 269 270 /* 271 * This event should be signalled when a synchronous message has been 272 * released. 273 */ 274 struct remote_event sync_release; 275 276 /* A circular buffer of slot indexes. */ 277 int slot_queue[VCHIQ_MAX_SLOTS_PER_SIDE]; 278 279 /* Debugging state */ 280 int debug[DEBUG_MAX]; 281 }; 282 283 struct vchiq_slot_zero { 284 int magic; 285 short version; 286 short version_min; 287 int slot_zero_size; 288 int slot_size; 289 int max_slots; 290 int max_slots_per_side; 291 int platform_data[2]; 292 struct vchiq_shared_state master; 293 struct vchiq_shared_state slave; 294 struct vchiq_slot_info slots[VCHIQ_MAX_SLOTS]; 295 }; 296 297 struct vchiq_state { 298 struct device *dev; 299 int id; 300 int initialised; 301 enum vchiq_connstate conn_state; 302 short version_common; 303 304 struct vchiq_shared_state *local; 305 struct vchiq_shared_state *remote; 306 struct vchiq_slot *slot_data; 307 308 unsigned short default_slot_quota; 309 unsigned short default_message_quota; 310 311 /* Event indicating connect message received */ 312 struct completion connect; 313 314 /* Mutex protecting services */ 315 struct mutex mutex; 316 struct vchiq_instance **instance; 317 318 /* Processes incoming messages */ 319 struct task_struct *slot_handler_thread; 320 321 /* Processes recycled slots */ 322 struct task_struct *recycle_thread; 323 324 /* Processes synchronous messages */ 325 struct task_struct *sync_thread; 326 327 /* Local implementation of the trigger remote event */ 328 wait_queue_head_t trigger_event; 329 330 /* Local implementation of the recycle remote event */ 331 wait_queue_head_t recycle_event; 332 333 /* Local implementation of the sync trigger remote event */ 334 wait_queue_head_t sync_trigger_event; 335 336 /* Local implementation of the sync release remote event */ 337 wait_queue_head_t sync_release_event; 338 339 char *tx_data; 340 char *rx_data; 341 struct vchiq_slot_info *rx_info; 342 343 struct mutex slot_mutex; 344 345 struct mutex recycle_mutex; 346 347 struct mutex sync_mutex; 348 349 struct mutex bulk_transfer_mutex; 350 351 /* 352 * Indicates the byte position within the stream from where the next 353 * message will be read. The least significant bits are an index into 354 * the slot.The next bits are the index of the slot in 355 * remote->slot_queue. 356 */ 357 int rx_pos; 358 359 /* 360 * A cached copy of local->tx_pos. Only write to local->tx_pos, and read 361 * from remote->tx_pos. 362 */ 363 int local_tx_pos; 364 365 /* The slot_queue index of the slot to become available next. */ 366 int slot_queue_available; 367 368 /* A flag to indicate if any poll has been requested */ 369 int poll_needed; 370 371 /* Ths index of the previous slot used for data messages. */ 372 int previous_data_index; 373 374 /* The number of slots occupied by data messages. */ 375 unsigned short data_use_count; 376 377 /* The maximum number of slots to be occupied by data messages. */ 378 unsigned short data_quota; 379 380 /* An array of bit sets indicating which services must be polled. */ 381 atomic_t poll_services[BITSET_SIZE(VCHIQ_MAX_SERVICES)]; 382 383 /* The number of the first unused service */ 384 int unused_service; 385 386 /* Signalled when a free slot becomes available. */ 387 struct completion slot_available_event; 388 389 struct completion slot_remove_event; 390 391 /* Signalled when a free data slot becomes available. */ 392 struct completion data_quota_event; 393 394 struct state_stats_struct { 395 int slot_stalls; 396 int data_stalls; 397 int ctrl_tx_count; 398 int ctrl_rx_count; 399 int error_count; 400 } stats; 401 402 struct vchiq_service __rcu *services[VCHIQ_MAX_SERVICES]; 403 struct vchiq_service_quota service_quotas[VCHIQ_MAX_SERVICES]; 404 struct vchiq_slot_info slot_info[VCHIQ_MAX_SLOTS]; 405 406 struct opaque_platform_state *platform_state; 407 }; 408 409 struct bulk_waiter { 410 struct vchiq_bulk *bulk; 411 struct completion event; 412 int actual; 413 }; 414 415 struct vchiq_config { 416 unsigned int max_msg_size; 417 unsigned int bulk_threshold; /* The message size above which it 418 * is better to use a bulk transfer 419 * (<= max_msg_size) 420 */ 421 unsigned int max_outstanding_bulks; 422 unsigned int max_services; 423 short version; /* The version of VCHIQ */ 424 short version_min; /* The minimum compatible version of VCHIQ */ 425 }; 426 427 extern spinlock_t bulk_waiter_spinlock; 428 429 extern const char * 430 get_conn_state_name(enum vchiq_connstate conn_state); 431 432 extern struct vchiq_slot_zero * 433 vchiq_init_slots(struct device *dev, void *mem_base, int mem_size); 434 435 extern int 436 vchiq_init_state(struct vchiq_state *state, struct vchiq_slot_zero *slot_zero, struct device *dev); 437 438 extern int 439 vchiq_connect_internal(struct vchiq_state *state, struct vchiq_instance *instance); 440 441 struct vchiq_service * 442 vchiq_add_service_internal(struct vchiq_state *state, 443 const struct vchiq_service_params_kernel *params, 444 int srvstate, struct vchiq_instance *instance, 445 void (*userdata_term)(void *userdata)); 446 447 extern int 448 vchiq_open_service_internal(struct vchiq_service *service, int client_id); 449 450 extern int 451 vchiq_close_service_internal(struct vchiq_service *service, int close_recvd); 452 453 extern void 454 vchiq_terminate_service_internal(struct vchiq_service *service); 455 456 extern void 457 vchiq_free_service_internal(struct vchiq_service *service); 458 459 extern void 460 vchiq_shutdown_internal(struct vchiq_state *state, struct vchiq_instance *instance); 461 462 extern void 463 remote_event_pollall(struct vchiq_state *state); 464 465 extern int 466 vchiq_bulk_transfer(struct vchiq_instance *instance, unsigned int handle, void *offset, 467 void __user *uoffset, int size, void *userdata, enum vchiq_bulk_mode mode, 468 enum vchiq_bulk_dir dir); 469 470 extern void 471 vchiq_dump_state(struct seq_file *f, struct vchiq_state *state); 472 473 extern void 474 vchiq_loud_error_header(void); 475 476 extern void 477 vchiq_loud_error_footer(void); 478 479 extern void 480 request_poll(struct vchiq_state *state, struct vchiq_service *service, 481 int poll_type); 482 483 struct vchiq_service *handle_to_service(struct vchiq_instance *instance, unsigned int handle); 484 485 extern struct vchiq_service * 486 find_service_by_handle(struct vchiq_instance *instance, unsigned int handle); 487 488 extern struct vchiq_service * 489 find_service_by_port(struct vchiq_state *state, unsigned int localport); 490 491 extern struct vchiq_service * 492 find_service_for_instance(struct vchiq_instance *instance, unsigned int handle); 493 494 extern struct vchiq_service * 495 find_closed_service_for_instance(struct vchiq_instance *instance, unsigned int handle); 496 497 extern struct vchiq_service * 498 __next_service_by_instance(struct vchiq_state *state, 499 struct vchiq_instance *instance, 500 int *pidx); 501 502 extern struct vchiq_service * 503 next_service_by_instance(struct vchiq_state *state, 504 struct vchiq_instance *instance, 505 int *pidx); 506 507 extern void 508 vchiq_service_get(struct vchiq_service *service); 509 510 extern void 511 vchiq_service_put(struct vchiq_service *service); 512 513 extern int 514 vchiq_queue_message(struct vchiq_instance *instance, unsigned int handle, 515 ssize_t (*copy_callback)(void *context, void *dest, 516 size_t offset, size_t maxsize), 517 void *context, 518 size_t size); 519 520 int vchiq_prepare_bulk_data(struct vchiq_instance *instance, struct vchiq_bulk *bulk, void *offset, 521 void __user *uoffset, int size, int dir); 522 523 void vchiq_complete_bulk(struct vchiq_instance *instance, struct vchiq_bulk *bulk); 524 525 void remote_event_signal(struct remote_event *event); 526 527 void vchiq_dump_platform_state(struct seq_file *f); 528 529 void vchiq_dump_platform_instances(struct seq_file *f); 530 531 void vchiq_dump_platform_service_state(struct seq_file *f, struct vchiq_service *service); 532 533 int vchiq_use_service_internal(struct vchiq_service *service); 534 535 int vchiq_release_service_internal(struct vchiq_service *service); 536 537 void vchiq_on_remote_use(struct vchiq_state *state); 538 539 void vchiq_on_remote_release(struct vchiq_state *state); 540 541 int vchiq_platform_init_state(struct vchiq_state *state); 542 543 int vchiq_check_service(struct vchiq_service *service); 544 545 void vchiq_on_remote_use_active(struct vchiq_state *state); 546 547 int vchiq_send_remote_use(struct vchiq_state *state); 548 549 int vchiq_send_remote_use_active(struct vchiq_state *state); 550 551 void vchiq_platform_conn_state_changed(struct vchiq_state *state, 552 enum vchiq_connstate oldstate, 553 enum vchiq_connstate newstate); 554 555 void vchiq_set_conn_state(struct vchiq_state *state, enum vchiq_connstate newstate); 556 557 void vchiq_log_dump_mem(struct device *dev, const char *label, u32 addr, 558 const void *void_mem, size_t num_bytes); 559 560 int vchiq_remove_service(struct vchiq_instance *instance, unsigned int service); 561 562 int vchiq_get_client_id(struct vchiq_instance *instance, unsigned int service); 563 564 void vchiq_get_config(struct vchiq_config *config); 565 566 int vchiq_set_service_option(struct vchiq_instance *instance, unsigned int service, 567 enum vchiq_service_option option, int value); 568 569 #endif 570