1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Internal Thunderbolt Connection Manager. This is a firmware running on 4 * the Thunderbolt host controller performing most of the low-level 5 * handling. 6 * 7 * Copyright (C) 2017, Intel Corporation 8 * Authors: Michael Jamet <michael.jamet@intel.com> 9 * Mika Westerberg <mika.westerberg@linux.intel.com> 10 */ 11 12 #include <linux/delay.h> 13 #include <linux/mutex.h> 14 #include <linux/moduleparam.h> 15 #include <linux/pci.h> 16 #include <linux/pm_runtime.h> 17 #include <linux/platform_data/x86/apple.h> 18 #include <linux/sizes.h> 19 #include <linux/slab.h> 20 #include <linux/workqueue.h> 21 22 #include "ctl.h" 23 #include "nhi_regs.h" 24 #include "tb.h" 25 #include "tunnel.h" 26 27 #define PCIE2CIO_CMD 0x30 28 #define PCIE2CIO_CMD_TIMEOUT BIT(31) 29 #define PCIE2CIO_CMD_START BIT(30) 30 #define PCIE2CIO_CMD_WRITE BIT(21) 31 #define PCIE2CIO_CMD_CS_MASK GENMASK(20, 19) 32 #define PCIE2CIO_CMD_CS_SHIFT 19 33 #define PCIE2CIO_CMD_PORT_MASK GENMASK(18, 13) 34 #define PCIE2CIO_CMD_PORT_SHIFT 13 35 36 #define PCIE2CIO_WRDATA 0x34 37 #define PCIE2CIO_RDDATA 0x38 38 39 #define PHY_PORT_CS1 0x37 40 #define PHY_PORT_CS1_LINK_DISABLE BIT(14) 41 #define PHY_PORT_CS1_LINK_STATE_MASK GENMASK(29, 26) 42 #define PHY_PORT_CS1_LINK_STATE_SHIFT 26 43 44 #define ICM_TIMEOUT 5000 /* ms */ 45 #define ICM_RETRIES 3 46 #define ICM_APPROVE_TIMEOUT 10000 /* ms */ 47 #define ICM_MAX_LINK 4 48 49 static bool start_icm; 50 module_param(start_icm, bool, 0444); 51 MODULE_PARM_DESC(start_icm, "start ICM firmware if it is not running (default: false)"); 52 53 /** 54 * struct usb4_switch_nvm_auth - Holds USB4 NVM_AUTH status 55 * @reply: Reply from ICM firmware is placed here 56 * @request: Request that is sent to ICM firmware 57 * @icm: Pointer to ICM private data 58 */ 59 struct usb4_switch_nvm_auth { 60 struct icm_usb4_switch_op_response reply; 61 struct icm_usb4_switch_op request; 62 struct icm *icm; 63 }; 64 65 /** 66 * struct icm - Internal connection manager private data 67 * @request_lock: Makes sure only one message is send to ICM at time 68 * @rescan_work: Work used to rescan the surviving switches after resume 69 * @upstream_port: Pointer to the PCIe upstream port this host 70 * controller is connected. This is only set for systems 71 * where ICM needs to be started manually 72 * @vnd_cap: Vendor defined capability where PCIe2CIO mailbox resides 73 * (only set when @upstream_port is not %NULL) 74 * @safe_mode: ICM is in safe mode 75 * @max_boot_acl: Maximum number of preboot ACL entries (%0 if not supported) 76 * @rpm: Does the controller support runtime PM (RTD3) 77 * @can_upgrade_nvm: Can the NVM firmware be upgrade on this controller 78 * @proto_version: Firmware protocol version 79 * @last_nvm_auth: Last USB4 router NVM_AUTH result (or %NULL if not set) 80 * @veto: Is RTD3 veto in effect 81 * @is_supported: Checks if we can support ICM on this controller 82 * @cio_reset: Trigger CIO reset 83 * @get_mode: Read and return the ICM firmware mode (optional) 84 * @get_route: Find a route string for given switch 85 * @save_devices: Ask ICM to save devices to ACL when suspending (optional) 86 * @driver_ready: Send driver ready message to ICM 87 * @set_uuid: Set UUID for the root switch (optional) 88 * @device_connected: Handle device connected ICM message 89 * @device_disconnected: Handle device disconnected ICM message 90 * @xdomain_connected: Handle XDomain connected ICM message 91 * @xdomain_disconnected: Handle XDomain disconnected ICM message 92 * @rtd3_veto: Handle RTD3 veto notification ICM message 93 */ 94 struct icm { 95 struct mutex request_lock; 96 struct delayed_work rescan_work; 97 struct pci_dev *upstream_port; 98 int vnd_cap; 99 bool safe_mode; 100 size_t max_boot_acl; 101 bool rpm; 102 bool can_upgrade_nvm; 103 u8 proto_version; 104 struct usb4_switch_nvm_auth *last_nvm_auth; 105 bool veto; 106 bool (*is_supported)(struct tb *tb); 107 int (*cio_reset)(struct tb *tb); 108 int (*get_mode)(struct tb *tb); 109 int (*get_route)(struct tb *tb, u8 link, u8 depth, u64 *route); 110 void (*save_devices)(struct tb *tb); 111 int (*driver_ready)(struct tb *tb, 112 enum tb_security_level *security_level, 113 u8 *proto_version, size_t *nboot_acl, bool *rpm); 114 void (*set_uuid)(struct tb *tb); 115 void (*device_connected)(struct tb *tb, 116 const struct icm_pkg_header *hdr); 117 void (*device_disconnected)(struct tb *tb, 118 const struct icm_pkg_header *hdr); 119 void (*xdomain_connected)(struct tb *tb, 120 const struct icm_pkg_header *hdr); 121 void (*xdomain_disconnected)(struct tb *tb, 122 const struct icm_pkg_header *hdr); 123 void (*rtd3_veto)(struct tb *tb, const struct icm_pkg_header *hdr); 124 }; 125 126 struct icm_notification { 127 struct work_struct work; 128 struct icm_pkg_header *pkg; 129 struct tb *tb; 130 }; 131 132 struct ep_name_entry { 133 u8 len; 134 u8 type; 135 u8 data[]; 136 }; 137 138 #define EP_NAME_INTEL_VSS 0x10 139 140 /* Intel Vendor specific structure */ 141 struct intel_vss { 142 u16 vendor; 143 u16 model; 144 u8 mc; 145 u8 flags; 146 u16 pci_devid; 147 u32 nvm_version; 148 }; 149 150 #define INTEL_VSS_FLAGS_RTD3 BIT(0) 151 152 static const struct intel_vss *parse_intel_vss(const void *ep_name, size_t size) 153 { 154 const void *end = ep_name + size; 155 156 while (ep_name < end) { 157 const struct ep_name_entry *ep = ep_name; 158 159 if (!ep->len) 160 break; 161 if (ep_name + ep->len > end) 162 break; 163 164 if (ep->type == EP_NAME_INTEL_VSS) 165 return (const struct intel_vss *)ep->data; 166 167 ep_name += ep->len; 168 } 169 170 return NULL; 171 } 172 173 static bool intel_vss_is_rtd3(const void *ep_name, size_t size) 174 { 175 const struct intel_vss *vss; 176 177 vss = parse_intel_vss(ep_name, size); 178 if (vss) 179 return !!(vss->flags & INTEL_VSS_FLAGS_RTD3); 180 181 return false; 182 } 183 184 static inline struct tb *icm_to_tb(struct icm *icm) 185 { 186 return ((void *)icm - sizeof(struct tb)); 187 } 188 189 static inline u8 phy_port_from_route(u64 route, u8 depth) 190 { 191 u8 link; 192 193 link = depth ? route >> ((depth - 1) * 8) : route; 194 return tb_phy_port_from_link(link); 195 } 196 197 static inline u8 dual_link_from_link(u8 link) 198 { 199 return link ? ((link - 1) ^ 0x01) + 1 : 0; 200 } 201 202 static inline u64 get_route(u32 route_hi, u32 route_lo) 203 { 204 return (u64)route_hi << 32 | route_lo; 205 } 206 207 static inline u64 get_parent_route(u64 route) 208 { 209 int depth = tb_route_length(route); 210 return depth ? route & ~(0xffULL << (depth - 1) * TB_ROUTE_SHIFT) : 0; 211 } 212 213 static int pci2cio_wait_completion(struct icm *icm, unsigned long timeout_msec) 214 { 215 unsigned long end = jiffies + msecs_to_jiffies(timeout_msec); 216 u32 cmd; 217 218 do { 219 pci_read_config_dword(icm->upstream_port, 220 icm->vnd_cap + PCIE2CIO_CMD, &cmd); 221 if (!(cmd & PCIE2CIO_CMD_START)) { 222 if (cmd & PCIE2CIO_CMD_TIMEOUT) 223 break; 224 return 0; 225 } 226 227 msleep(50); 228 } while (time_before(jiffies, end)); 229 230 return -ETIMEDOUT; 231 } 232 233 static int pcie2cio_read(struct icm *icm, enum tb_cfg_space cs, 234 unsigned int port, unsigned int index, u32 *data) 235 { 236 struct pci_dev *pdev = icm->upstream_port; 237 int ret, vnd_cap = icm->vnd_cap; 238 u32 cmd; 239 240 cmd = index; 241 cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK; 242 cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK; 243 cmd |= PCIE2CIO_CMD_START; 244 pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd); 245 246 ret = pci2cio_wait_completion(icm, 5000); 247 if (ret) 248 return ret; 249 250 pci_read_config_dword(pdev, vnd_cap + PCIE2CIO_RDDATA, data); 251 return 0; 252 } 253 254 static int pcie2cio_write(struct icm *icm, enum tb_cfg_space cs, 255 unsigned int port, unsigned int index, u32 data) 256 { 257 struct pci_dev *pdev = icm->upstream_port; 258 int vnd_cap = icm->vnd_cap; 259 u32 cmd; 260 261 pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_WRDATA, data); 262 263 cmd = index; 264 cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK; 265 cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK; 266 cmd |= PCIE2CIO_CMD_WRITE | PCIE2CIO_CMD_START; 267 pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd); 268 269 return pci2cio_wait_completion(icm, 5000); 270 } 271 272 static bool icm_match(const struct tb_cfg_request *req, 273 const struct ctl_pkg *pkg) 274 { 275 const struct icm_pkg_header *res_hdr = pkg->buffer; 276 const struct icm_pkg_header *req_hdr = req->request; 277 278 if (pkg->frame.eof != req->response_type) 279 return false; 280 if (res_hdr->code != req_hdr->code) 281 return false; 282 283 return true; 284 } 285 286 static bool icm_copy(struct tb_cfg_request *req, const struct ctl_pkg *pkg) 287 { 288 const struct icm_pkg_header *hdr = pkg->buffer; 289 290 if (hdr->packet_id < req->npackets) { 291 size_t offset = hdr->packet_id * req->response_size; 292 293 memcpy(req->response + offset, pkg->buffer, req->response_size); 294 } 295 296 return hdr->packet_id == hdr->total_packets - 1; 297 } 298 299 static int icm_request(struct tb *tb, const void *request, size_t request_size, 300 void *response, size_t response_size, size_t npackets, 301 int retries, unsigned int timeout_msec) 302 { 303 struct icm *icm = tb_priv(tb); 304 305 do { 306 struct tb_cfg_request *req; 307 struct tb_cfg_result res; 308 309 req = tb_cfg_request_alloc(); 310 if (!req) 311 return -ENOMEM; 312 313 req->match = icm_match; 314 req->copy = icm_copy; 315 req->request = request; 316 req->request_size = request_size; 317 req->request_type = TB_CFG_PKG_ICM_CMD; 318 req->response = response; 319 req->npackets = npackets; 320 req->response_size = response_size; 321 req->response_type = TB_CFG_PKG_ICM_RESP; 322 323 mutex_lock(&icm->request_lock); 324 res = tb_cfg_request_sync(tb->ctl, req, timeout_msec); 325 mutex_unlock(&icm->request_lock); 326 327 tb_cfg_request_put(req); 328 329 if (res.err != -ETIMEDOUT) 330 return res.err == 1 ? -EIO : res.err; 331 332 usleep_range(20, 50); 333 } while (retries--); 334 335 return -ETIMEDOUT; 336 } 337 338 /* 339 * If rescan is queued to run (we are resuming), postpone it to give the 340 * firmware some more time to send device connected notifications for next 341 * devices in the chain. 342 */ 343 static void icm_postpone_rescan(struct tb *tb) 344 { 345 struct icm *icm = tb_priv(tb); 346 347 if (delayed_work_pending(&icm->rescan_work)) 348 mod_delayed_work(tb->wq, &icm->rescan_work, 349 msecs_to_jiffies(500)); 350 } 351 352 static void icm_veto_begin(struct tb *tb) 353 { 354 struct icm *icm = tb_priv(tb); 355 356 if (!icm->veto) { 357 icm->veto = true; 358 /* Keep the domain powered while veto is in effect */ 359 pm_runtime_get(&tb->dev); 360 } 361 } 362 363 static void icm_veto_end(struct tb *tb) 364 { 365 struct icm *icm = tb_priv(tb); 366 367 if (icm->veto) { 368 icm->veto = false; 369 /* Allow the domain suspend now */ 370 pm_runtime_mark_last_busy(&tb->dev); 371 pm_runtime_put_autosuspend(&tb->dev); 372 } 373 } 374 375 static bool icm_firmware_running(const struct tb_nhi *nhi) 376 { 377 u32 val; 378 379 val = ioread32(nhi->iobase + REG_FW_STS); 380 return !!(val & REG_FW_STS_ICM_EN); 381 } 382 383 static void icm_xdomain_activated(struct tb_xdomain *xd, bool activated) 384 { 385 struct tb_port *nhi_port, *dst_port; 386 struct tb *tb = xd->tb; 387 388 nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI); 389 dst_port = tb_xdomain_downstream_port(xd); 390 391 if (activated) 392 tb_tunnel_event(tb, TB_TUNNEL_ACTIVATED, TB_TUNNEL_DMA, 393 nhi_port, dst_port); 394 else 395 tb_tunnel_event(tb, TB_TUNNEL_DEACTIVATED, TB_TUNNEL_DMA, 396 nhi_port, dst_port); 397 } 398 399 static void icm_dp_event(struct tb *tb) 400 { 401 tb_tunnel_event(tb, TB_TUNNEL_CHANGED, TB_TUNNEL_DP, NULL, NULL); 402 } 403 404 static bool icm_fr_is_supported(struct tb *tb) 405 { 406 return !x86_apple_machine; 407 } 408 409 static inline int icm_fr_get_switch_index(u32 port) 410 { 411 int index; 412 413 if ((port & ICM_PORT_TYPE_MASK) != TB_TYPE_PORT) 414 return 0; 415 416 index = port >> ICM_PORT_INDEX_SHIFT; 417 return index != 0xff ? index : 0; 418 } 419 420 static int icm_fr_get_route(struct tb *tb, u8 link, u8 depth, u64 *route) 421 { 422 struct icm_fr_pkg_get_topology_response *switches, *sw; 423 struct icm_fr_pkg_get_topology request = { 424 .hdr = { .code = ICM_GET_TOPOLOGY }, 425 }; 426 size_t npackets = ICM_GET_TOPOLOGY_PACKETS; 427 int ret, index; 428 u8 i; 429 430 switches = kcalloc(npackets, sizeof(*switches), GFP_KERNEL); 431 if (!switches) 432 return -ENOMEM; 433 434 ret = icm_request(tb, &request, sizeof(request), switches, 435 sizeof(*switches), npackets, ICM_RETRIES, ICM_TIMEOUT); 436 if (ret) 437 goto err_free; 438 439 sw = &switches[0]; 440 index = icm_fr_get_switch_index(sw->ports[link]); 441 if (!index) { 442 ret = -ENODEV; 443 goto err_free; 444 } 445 446 sw = &switches[index]; 447 for (i = 1; i < depth; i++) { 448 unsigned int j; 449 450 if (!(sw->first_data & ICM_SWITCH_USED)) { 451 ret = -ENODEV; 452 goto err_free; 453 } 454 455 for (j = 0; j < ARRAY_SIZE(sw->ports); j++) { 456 index = icm_fr_get_switch_index(sw->ports[j]); 457 if (index > sw->switch_index) { 458 sw = &switches[index]; 459 break; 460 } 461 } 462 } 463 464 *route = get_route(sw->route_hi, sw->route_lo); 465 466 err_free: 467 kfree(switches); 468 return ret; 469 } 470 471 static void icm_fr_save_devices(struct tb *tb) 472 { 473 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_SAVE_DEVS, 0); 474 } 475 476 static int 477 icm_fr_driver_ready(struct tb *tb, enum tb_security_level *security_level, 478 u8 *proto_version, size_t *nboot_acl, bool *rpm) 479 { 480 struct icm_fr_pkg_driver_ready_response reply; 481 struct icm_pkg_driver_ready request = { 482 .hdr.code = ICM_DRIVER_READY, 483 }; 484 int ret; 485 486 memset(&reply, 0, sizeof(reply)); 487 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply), 488 1, ICM_RETRIES, ICM_TIMEOUT); 489 if (ret) 490 return ret; 491 492 if (security_level) 493 *security_level = reply.security_level & ICM_FR_SLEVEL_MASK; 494 495 return 0; 496 } 497 498 static int icm_fr_approve_switch(struct tb *tb, struct tb_switch *sw) 499 { 500 struct icm_fr_pkg_approve_device request; 501 struct icm_fr_pkg_approve_device reply; 502 int ret; 503 504 memset(&request, 0, sizeof(request)); 505 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid)); 506 request.hdr.code = ICM_APPROVE_DEVICE; 507 request.connection_id = sw->connection_id; 508 request.connection_key = sw->connection_key; 509 510 memset(&reply, 0, sizeof(reply)); 511 /* Use larger timeout as establishing tunnels can take some time */ 512 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply), 513 1, ICM_RETRIES, ICM_APPROVE_TIMEOUT); 514 if (ret) 515 return ret; 516 517 if (reply.hdr.flags & ICM_FLAGS_ERROR) { 518 tb_warn(tb, "PCIe tunnel creation failed\n"); 519 return -EIO; 520 } 521 522 return 0; 523 } 524 525 static int icm_fr_add_switch_key(struct tb *tb, struct tb_switch *sw) 526 { 527 struct icm_fr_pkg_add_device_key request; 528 struct icm_fr_pkg_add_device_key_response reply; 529 int ret; 530 531 memset(&request, 0, sizeof(request)); 532 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid)); 533 request.hdr.code = ICM_ADD_DEVICE_KEY; 534 request.connection_id = sw->connection_id; 535 request.connection_key = sw->connection_key; 536 memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE); 537 538 memset(&reply, 0, sizeof(reply)); 539 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply), 540 1, ICM_RETRIES, ICM_TIMEOUT); 541 if (ret) 542 return ret; 543 544 if (reply.hdr.flags & ICM_FLAGS_ERROR) { 545 tb_warn(tb, "Adding key to switch failed\n"); 546 return -EIO; 547 } 548 549 return 0; 550 } 551 552 static int icm_fr_challenge_switch_key(struct tb *tb, struct tb_switch *sw, 553 const u8 *challenge, u8 *response) 554 { 555 struct icm_fr_pkg_challenge_device request; 556 struct icm_fr_pkg_challenge_device_response reply; 557 int ret; 558 559 memset(&request, 0, sizeof(request)); 560 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid)); 561 request.hdr.code = ICM_CHALLENGE_DEVICE; 562 request.connection_id = sw->connection_id; 563 request.connection_key = sw->connection_key; 564 memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE); 565 566 memset(&reply, 0, sizeof(reply)); 567 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply), 568 1, ICM_RETRIES, ICM_TIMEOUT); 569 if (ret) 570 return ret; 571 572 if (reply.hdr.flags & ICM_FLAGS_ERROR) 573 return -EKEYREJECTED; 574 if (reply.hdr.flags & ICM_FLAGS_NO_KEY) 575 return -ENOKEY; 576 577 memcpy(response, reply.response, TB_SWITCH_KEY_SIZE); 578 579 return 0; 580 } 581 582 static int icm_fr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd, 583 int transmit_path, int transmit_ring, 584 int receive_path, int receive_ring) 585 { 586 struct icm_fr_pkg_approve_xdomain_response reply; 587 struct icm_fr_pkg_approve_xdomain request; 588 int ret; 589 590 memset(&request, 0, sizeof(request)); 591 request.hdr.code = ICM_APPROVE_XDOMAIN; 592 request.link_info = xd->depth << ICM_LINK_INFO_DEPTH_SHIFT | xd->link; 593 memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid)); 594 595 request.transmit_path = transmit_path; 596 request.transmit_ring = transmit_ring; 597 request.receive_path = receive_path; 598 request.receive_ring = receive_ring; 599 600 memset(&reply, 0, sizeof(reply)); 601 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply), 602 1, ICM_RETRIES, ICM_TIMEOUT); 603 if (ret) 604 return ret; 605 606 if (reply.hdr.flags & ICM_FLAGS_ERROR) 607 return -EIO; 608 609 icm_xdomain_activated(xd, true); 610 return 0; 611 } 612 613 static int icm_fr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd, 614 int transmit_path, int transmit_ring, 615 int receive_path, int receive_ring) 616 { 617 u8 phy_port; 618 u8 cmd; 619 620 phy_port = tb_phy_port_from_link(xd->link); 621 if (phy_port == 0) 622 cmd = NHI_MAILBOX_DISCONNECT_PA; 623 else 624 cmd = NHI_MAILBOX_DISCONNECT_PB; 625 626 nhi_mailbox_cmd(tb->nhi, cmd, 1); 627 usleep_range(10, 50); 628 nhi_mailbox_cmd(tb->nhi, cmd, 2); 629 630 icm_xdomain_activated(xd, false); 631 return 0; 632 } 633 634 static struct tb_switch *alloc_switch(struct tb_switch *parent_sw, u64 route, 635 const uuid_t *uuid) 636 { 637 struct tb *tb = parent_sw->tb; 638 struct tb_switch *sw; 639 640 sw = tb_switch_alloc(tb, &parent_sw->dev, route); 641 if (IS_ERR(sw)) { 642 tb_warn(tb, "failed to allocate switch at %llx\n", route); 643 return sw; 644 } 645 646 sw->uuid = kmemdup(uuid, sizeof(*uuid), GFP_KERNEL); 647 if (!sw->uuid) { 648 tb_switch_put(sw); 649 return ERR_PTR(-ENOMEM); 650 } 651 652 init_completion(&sw->rpm_complete); 653 return sw; 654 } 655 656 static int add_switch(struct tb_switch *parent_sw, struct tb_switch *sw) 657 { 658 u64 route = tb_route(sw); 659 int ret; 660 661 /* Link the two switches now */ 662 tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw); 663 tb_upstream_port(sw)->remote = tb_port_at(route, parent_sw); 664 665 ret = tb_switch_add(sw); 666 if (ret) 667 tb_port_at(tb_route(sw), parent_sw)->remote = NULL; 668 669 return ret; 670 } 671 672 static void update_switch(struct tb_switch *sw, u64 route, u8 connection_id, 673 u8 connection_key, u8 link, u8 depth, bool boot) 674 { 675 struct tb_switch *parent_sw = tb_switch_parent(sw); 676 677 /* Disconnect from parent */ 678 tb_switch_downstream_port(sw)->remote = NULL; 679 /* Re-connect via updated port */ 680 tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw); 681 682 /* Update with the new addressing information */ 683 sw->config.route_hi = upper_32_bits(route); 684 sw->config.route_lo = lower_32_bits(route); 685 sw->connection_id = connection_id; 686 sw->connection_key = connection_key; 687 sw->link = link; 688 sw->depth = depth; 689 sw->boot = boot; 690 691 /* This switch still exists */ 692 sw->is_unplugged = false; 693 694 /* Runtime resume is now complete */ 695 complete(&sw->rpm_complete); 696 } 697 698 static void remove_switch(struct tb_switch *sw) 699 { 700 tb_switch_downstream_port(sw)->remote = NULL; 701 tb_switch_remove(sw); 702 } 703 704 static void add_xdomain(struct tb_switch *sw, u64 route, 705 const uuid_t *local_uuid, const uuid_t *remote_uuid, 706 u8 link, u8 depth) 707 { 708 struct tb_xdomain *xd; 709 710 pm_runtime_get_sync(&sw->dev); 711 712 xd = tb_xdomain_alloc(sw->tb, &sw->dev, route, local_uuid, remote_uuid); 713 if (!xd) 714 goto out; 715 716 xd->link = link; 717 xd->depth = depth; 718 719 tb_port_at(route, sw)->xdomain = xd; 720 721 tb_xdomain_add(xd); 722 723 out: 724 pm_runtime_mark_last_busy(&sw->dev); 725 pm_runtime_put_autosuspend(&sw->dev); 726 } 727 728 static void update_xdomain(struct tb_xdomain *xd, u64 route, u8 link) 729 { 730 xd->link = link; 731 xd->route = route; 732 xd->is_unplugged = false; 733 } 734 735 static void remove_xdomain(struct tb_xdomain *xd) 736 { 737 struct tb_switch *sw; 738 739 sw = tb_to_switch(xd->dev.parent); 740 tb_port_at(xd->route, sw)->xdomain = NULL; 741 tb_xdomain_remove(xd); 742 } 743 744 static void 745 icm_fr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr) 746 { 747 const struct icm_fr_event_device_connected *pkg = 748 (const struct icm_fr_event_device_connected *)hdr; 749 enum tb_security_level security_level; 750 struct tb_switch *sw, *parent_sw; 751 bool boot, dual_lane, speed_gen3; 752 struct icm *icm = tb_priv(tb); 753 bool authorized = false; 754 struct tb_xdomain *xd; 755 u8 link, depth; 756 u64 route; 757 int ret; 758 759 icm_postpone_rescan(tb); 760 761 link = pkg->link_info & ICM_LINK_INFO_LINK_MASK; 762 depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >> 763 ICM_LINK_INFO_DEPTH_SHIFT; 764 authorized = pkg->link_info & ICM_LINK_INFO_APPROVED; 765 security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >> 766 ICM_FLAGS_SLEVEL_SHIFT; 767 boot = pkg->link_info & ICM_LINK_INFO_BOOT; 768 dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE; 769 speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3; 770 771 if (pkg->link_info & ICM_LINK_INFO_REJECTED) { 772 tb_info(tb, "switch at %u.%u was rejected by ICM firmware because topology limit exceeded\n", 773 link, depth); 774 return; 775 } 776 777 sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid); 778 if (sw) { 779 u8 phy_port, sw_phy_port; 780 781 sw_phy_port = tb_phy_port_from_link(sw->link); 782 phy_port = tb_phy_port_from_link(link); 783 784 /* 785 * On resume ICM will send us connected events for the 786 * devices that still are present. However, that 787 * information might have changed for example by the 788 * fact that a switch on a dual-link connection might 789 * have been enumerated using the other link now. Make 790 * sure our book keeping matches that. 791 */ 792 if (sw->depth == depth && sw_phy_port == phy_port && 793 !!sw->authorized == authorized) { 794 /* 795 * It was enumerated through another link so update 796 * route string accordingly. 797 */ 798 if (sw->link != link) { 799 ret = icm->get_route(tb, link, depth, &route); 800 if (ret) { 801 tb_err(tb, "failed to update route string for switch at %u.%u\n", 802 link, depth); 803 tb_switch_put(sw); 804 return; 805 } 806 } else { 807 route = tb_route(sw); 808 } 809 810 update_switch(sw, route, pkg->connection_id, 811 pkg->connection_key, link, depth, boot); 812 tb_switch_put(sw); 813 return; 814 } 815 816 /* 817 * User connected the same switch to another physical 818 * port or to another part of the topology. Remove the 819 * existing switch now before adding the new one. 820 */ 821 remove_switch(sw); 822 tb_switch_put(sw); 823 } 824 825 /* 826 * If the switch was not found by UUID, look for a switch on 827 * same physical port (taking possible link aggregation into 828 * account) and depth. If we found one it is definitely a stale 829 * one so remove it first. 830 */ 831 sw = tb_switch_find_by_link_depth(tb, link, depth); 832 if (!sw) { 833 u8 dual_link; 834 835 dual_link = dual_link_from_link(link); 836 if (dual_link) 837 sw = tb_switch_find_by_link_depth(tb, dual_link, depth); 838 } 839 if (sw) { 840 remove_switch(sw); 841 tb_switch_put(sw); 842 } 843 844 /* Remove existing XDomain connection if found */ 845 xd = tb_xdomain_find_by_link_depth(tb, link, depth); 846 if (xd) { 847 remove_xdomain(xd); 848 tb_xdomain_put(xd); 849 } 850 851 parent_sw = tb_switch_find_by_link_depth(tb, link, depth - 1); 852 if (!parent_sw) { 853 tb_err(tb, "failed to find parent switch for %u.%u\n", 854 link, depth); 855 return; 856 } 857 858 ret = icm->get_route(tb, link, depth, &route); 859 if (ret) { 860 tb_err(tb, "failed to find route string for switch at %u.%u\n", 861 link, depth); 862 tb_switch_put(parent_sw); 863 return; 864 } 865 866 pm_runtime_get_sync(&parent_sw->dev); 867 868 sw = alloc_switch(parent_sw, route, &pkg->ep_uuid); 869 if (!IS_ERR(sw)) { 870 sw->connection_id = pkg->connection_id; 871 sw->connection_key = pkg->connection_key; 872 sw->link = link; 873 sw->depth = depth; 874 sw->authorized = authorized; 875 sw->security_level = security_level; 876 sw->boot = boot; 877 sw->link_speed = speed_gen3 ? 20 : 10; 878 sw->link_width = dual_lane ? TB_LINK_WIDTH_DUAL : 879 TB_LINK_WIDTH_SINGLE; 880 sw->rpm = intel_vss_is_rtd3(pkg->ep_name, sizeof(pkg->ep_name)); 881 882 if (add_switch(parent_sw, sw)) 883 tb_switch_put(sw); 884 } 885 886 pm_runtime_mark_last_busy(&parent_sw->dev); 887 pm_runtime_put_autosuspend(&parent_sw->dev); 888 889 tb_switch_put(parent_sw); 890 } 891 892 static void 893 icm_fr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr) 894 { 895 const struct icm_fr_event_device_disconnected *pkg = 896 (const struct icm_fr_event_device_disconnected *)hdr; 897 struct tb_switch *sw; 898 u8 link, depth; 899 900 link = pkg->link_info & ICM_LINK_INFO_LINK_MASK; 901 depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >> 902 ICM_LINK_INFO_DEPTH_SHIFT; 903 904 if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) { 905 tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth); 906 return; 907 } 908 909 sw = tb_switch_find_by_link_depth(tb, link, depth); 910 if (!sw) { 911 tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link, 912 depth); 913 return; 914 } 915 916 pm_runtime_get_sync(sw->dev.parent); 917 918 remove_switch(sw); 919 920 pm_runtime_mark_last_busy(sw->dev.parent); 921 pm_runtime_put_autosuspend(sw->dev.parent); 922 923 tb_switch_put(sw); 924 } 925 926 static void 927 icm_fr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr) 928 { 929 const struct icm_fr_event_xdomain_connected *pkg = 930 (const struct icm_fr_event_xdomain_connected *)hdr; 931 struct tb_xdomain *xd; 932 struct tb_switch *sw; 933 u8 link, depth; 934 u64 route; 935 936 link = pkg->link_info & ICM_LINK_INFO_LINK_MASK; 937 depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >> 938 ICM_LINK_INFO_DEPTH_SHIFT; 939 940 if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) { 941 tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth); 942 return; 943 } 944 945 route = get_route(pkg->local_route_hi, pkg->local_route_lo); 946 947 xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid); 948 if (xd) { 949 u8 xd_phy_port, phy_port; 950 951 xd_phy_port = phy_port_from_route(xd->route, xd->depth); 952 phy_port = phy_port_from_route(route, depth); 953 954 if (xd->depth == depth && xd_phy_port == phy_port) { 955 update_xdomain(xd, route, link); 956 tb_xdomain_put(xd); 957 return; 958 } 959 960 /* 961 * If we find an existing XDomain connection remove it 962 * now. We need to go through login handshake and 963 * everything anyway to be able to re-establish the 964 * connection. 965 */ 966 remove_xdomain(xd); 967 tb_xdomain_put(xd); 968 } 969 970 /* 971 * Look if there already exists an XDomain in the same place 972 * than the new one and in that case remove it because it is 973 * most likely another host that got disconnected. 974 */ 975 xd = tb_xdomain_find_by_link_depth(tb, link, depth); 976 if (!xd) { 977 u8 dual_link; 978 979 dual_link = dual_link_from_link(link); 980 if (dual_link) 981 xd = tb_xdomain_find_by_link_depth(tb, dual_link, 982 depth); 983 } 984 if (xd) { 985 remove_xdomain(xd); 986 tb_xdomain_put(xd); 987 } 988 989 /* 990 * If the user disconnected a switch during suspend and 991 * connected another host to the same port, remove the switch 992 * first. 993 */ 994 sw = tb_switch_find_by_route(tb, route); 995 if (sw) { 996 remove_switch(sw); 997 tb_switch_put(sw); 998 } 999 1000 sw = tb_switch_find_by_link_depth(tb, link, depth); 1001 if (!sw) { 1002 tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link, 1003 depth); 1004 return; 1005 } 1006 1007 add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, link, 1008 depth); 1009 tb_switch_put(sw); 1010 } 1011 1012 static void 1013 icm_fr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr) 1014 { 1015 const struct icm_fr_event_xdomain_disconnected *pkg = 1016 (const struct icm_fr_event_xdomain_disconnected *)hdr; 1017 struct tb_xdomain *xd; 1018 1019 /* 1020 * If the connection is through one or multiple devices, the 1021 * XDomain device is removed along with them so it is fine if we 1022 * cannot find it here. 1023 */ 1024 xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid); 1025 if (xd) { 1026 remove_xdomain(xd); 1027 tb_xdomain_put(xd); 1028 } 1029 } 1030 1031 static int icm_tr_cio_reset(struct tb *tb) 1032 { 1033 return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x777, BIT(1)); 1034 } 1035 1036 static int 1037 icm_tr_driver_ready(struct tb *tb, enum tb_security_level *security_level, 1038 u8 *proto_version, size_t *nboot_acl, bool *rpm) 1039 { 1040 struct icm_tr_pkg_driver_ready_response reply; 1041 struct icm_pkg_driver_ready request = { 1042 .hdr.code = ICM_DRIVER_READY, 1043 }; 1044 int ret; 1045 1046 memset(&reply, 0, sizeof(reply)); 1047 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply), 1048 1, 10, 250); 1049 if (ret) 1050 return ret; 1051 1052 if (security_level) 1053 *security_level = reply.info & ICM_TR_INFO_SLEVEL_MASK; 1054 if (proto_version) 1055 *proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >> 1056 ICM_TR_INFO_PROTO_VERSION_SHIFT; 1057 if (nboot_acl) 1058 *nboot_acl = (reply.info & ICM_TR_INFO_BOOT_ACL_MASK) >> 1059 ICM_TR_INFO_BOOT_ACL_SHIFT; 1060 if (rpm) 1061 *rpm = !!(reply.hdr.flags & ICM_TR_FLAGS_RTD3); 1062 1063 return 0; 1064 } 1065 1066 static int icm_tr_approve_switch(struct tb *tb, struct tb_switch *sw) 1067 { 1068 struct icm_tr_pkg_approve_device request; 1069 struct icm_tr_pkg_approve_device reply; 1070 int ret; 1071 1072 memset(&request, 0, sizeof(request)); 1073 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid)); 1074 request.hdr.code = ICM_APPROVE_DEVICE; 1075 request.route_lo = sw->config.route_lo; 1076 request.route_hi = sw->config.route_hi; 1077 request.connection_id = sw->connection_id; 1078 1079 memset(&reply, 0, sizeof(reply)); 1080 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply), 1081 1, ICM_RETRIES, ICM_APPROVE_TIMEOUT); 1082 if (ret) 1083 return ret; 1084 1085 if (reply.hdr.flags & ICM_FLAGS_ERROR) { 1086 tb_warn(tb, "PCIe tunnel creation failed\n"); 1087 return -EIO; 1088 } 1089 1090 return 0; 1091 } 1092 1093 static int icm_tr_add_switch_key(struct tb *tb, struct tb_switch *sw) 1094 { 1095 struct icm_tr_pkg_add_device_key_response reply; 1096 struct icm_tr_pkg_add_device_key request; 1097 int ret; 1098 1099 memset(&request, 0, sizeof(request)); 1100 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid)); 1101 request.hdr.code = ICM_ADD_DEVICE_KEY; 1102 request.route_lo = sw->config.route_lo; 1103 request.route_hi = sw->config.route_hi; 1104 request.connection_id = sw->connection_id; 1105 memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE); 1106 1107 memset(&reply, 0, sizeof(reply)); 1108 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply), 1109 1, ICM_RETRIES, ICM_TIMEOUT); 1110 if (ret) 1111 return ret; 1112 1113 if (reply.hdr.flags & ICM_FLAGS_ERROR) { 1114 tb_warn(tb, "Adding key to switch failed\n"); 1115 return -EIO; 1116 } 1117 1118 return 0; 1119 } 1120 1121 static int icm_tr_challenge_switch_key(struct tb *tb, struct tb_switch *sw, 1122 const u8 *challenge, u8 *response) 1123 { 1124 struct icm_tr_pkg_challenge_device_response reply; 1125 struct icm_tr_pkg_challenge_device request; 1126 int ret; 1127 1128 memset(&request, 0, sizeof(request)); 1129 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid)); 1130 request.hdr.code = ICM_CHALLENGE_DEVICE; 1131 request.route_lo = sw->config.route_lo; 1132 request.route_hi = sw->config.route_hi; 1133 request.connection_id = sw->connection_id; 1134 memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE); 1135 1136 memset(&reply, 0, sizeof(reply)); 1137 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply), 1138 1, ICM_RETRIES, ICM_TIMEOUT); 1139 if (ret) 1140 return ret; 1141 1142 if (reply.hdr.flags & ICM_FLAGS_ERROR) 1143 return -EKEYREJECTED; 1144 if (reply.hdr.flags & ICM_FLAGS_NO_KEY) 1145 return -ENOKEY; 1146 1147 memcpy(response, reply.response, TB_SWITCH_KEY_SIZE); 1148 1149 return 0; 1150 } 1151 1152 static int icm_tr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd, 1153 int transmit_path, int transmit_ring, 1154 int receive_path, int receive_ring) 1155 { 1156 struct icm_tr_pkg_approve_xdomain_response reply; 1157 struct icm_tr_pkg_approve_xdomain request; 1158 int ret; 1159 1160 memset(&request, 0, sizeof(request)); 1161 request.hdr.code = ICM_APPROVE_XDOMAIN; 1162 request.route_hi = upper_32_bits(xd->route); 1163 request.route_lo = lower_32_bits(xd->route); 1164 request.transmit_path = transmit_path; 1165 request.transmit_ring = transmit_ring; 1166 request.receive_path = receive_path; 1167 request.receive_ring = receive_ring; 1168 memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid)); 1169 1170 memset(&reply, 0, sizeof(reply)); 1171 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply), 1172 1, ICM_RETRIES, ICM_TIMEOUT); 1173 if (ret) 1174 return ret; 1175 1176 if (reply.hdr.flags & ICM_FLAGS_ERROR) 1177 return -EIO; 1178 1179 icm_xdomain_activated(xd, true); 1180 return 0; 1181 } 1182 1183 static int icm_tr_xdomain_tear_down(struct tb *tb, struct tb_xdomain *xd, 1184 int stage) 1185 { 1186 struct icm_tr_pkg_disconnect_xdomain_response reply; 1187 struct icm_tr_pkg_disconnect_xdomain request; 1188 int ret; 1189 1190 memset(&request, 0, sizeof(request)); 1191 request.hdr.code = ICM_DISCONNECT_XDOMAIN; 1192 request.stage = stage; 1193 request.route_hi = upper_32_bits(xd->route); 1194 request.route_lo = lower_32_bits(xd->route); 1195 memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid)); 1196 1197 memset(&reply, 0, sizeof(reply)); 1198 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply), 1199 1, ICM_RETRIES, ICM_TIMEOUT); 1200 if (ret) 1201 return ret; 1202 1203 if (reply.hdr.flags & ICM_FLAGS_ERROR) 1204 return -EIO; 1205 1206 return 0; 1207 } 1208 1209 static int icm_tr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd, 1210 int transmit_path, int transmit_ring, 1211 int receive_path, int receive_ring) 1212 { 1213 int ret; 1214 1215 ret = icm_tr_xdomain_tear_down(tb, xd, 1); 1216 if (ret) 1217 return ret; 1218 1219 usleep_range(10, 50); 1220 ret = icm_tr_xdomain_tear_down(tb, xd, 2); 1221 if (ret) 1222 return ret; 1223 1224 icm_xdomain_activated(xd, false); 1225 return 0; 1226 } 1227 1228 static void 1229 __icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr, 1230 bool force_rtd3) 1231 { 1232 const struct icm_tr_event_device_connected *pkg = 1233 (const struct icm_tr_event_device_connected *)hdr; 1234 bool authorized, boot, dual_lane, speed_gen3; 1235 enum tb_security_level security_level; 1236 struct tb_switch *sw, *parent_sw; 1237 struct tb_xdomain *xd; 1238 u64 route; 1239 1240 icm_postpone_rescan(tb); 1241 1242 /* 1243 * Currently we don't use the QoS information coming with the 1244 * device connected message so simply just ignore that extra 1245 * packet for now. 1246 */ 1247 if (pkg->hdr.packet_id) 1248 return; 1249 1250 route = get_route(pkg->route_hi, pkg->route_lo); 1251 authorized = pkg->link_info & ICM_LINK_INFO_APPROVED; 1252 security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >> 1253 ICM_FLAGS_SLEVEL_SHIFT; 1254 boot = pkg->link_info & ICM_LINK_INFO_BOOT; 1255 dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE; 1256 speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3; 1257 1258 if (pkg->link_info & ICM_LINK_INFO_REJECTED) { 1259 tb_info(tb, "switch at %llx was rejected by ICM firmware because topology limit exceeded\n", 1260 route); 1261 return; 1262 } 1263 1264 sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid); 1265 if (sw) { 1266 /* Update the switch if it is still in the same place */ 1267 if (tb_route(sw) == route && !!sw->authorized == authorized) { 1268 update_switch(sw, route, pkg->connection_id, 0, 0, 0, 1269 boot); 1270 tb_switch_put(sw); 1271 return; 1272 } 1273 1274 remove_switch(sw); 1275 tb_switch_put(sw); 1276 } 1277 1278 /* Another switch with the same address */ 1279 sw = tb_switch_find_by_route(tb, route); 1280 if (sw) { 1281 remove_switch(sw); 1282 tb_switch_put(sw); 1283 } 1284 1285 /* XDomain connection with the same address */ 1286 xd = tb_xdomain_find_by_route(tb, route); 1287 if (xd) { 1288 remove_xdomain(xd); 1289 tb_xdomain_put(xd); 1290 } 1291 1292 parent_sw = tb_switch_find_by_route(tb, get_parent_route(route)); 1293 if (!parent_sw) { 1294 tb_err(tb, "failed to find parent switch for %llx\n", route); 1295 return; 1296 } 1297 1298 pm_runtime_get_sync(&parent_sw->dev); 1299 1300 sw = alloc_switch(parent_sw, route, &pkg->ep_uuid); 1301 if (!IS_ERR(sw)) { 1302 sw->connection_id = pkg->connection_id; 1303 sw->authorized = authorized; 1304 sw->security_level = security_level; 1305 sw->boot = boot; 1306 sw->link_speed = speed_gen3 ? 20 : 10; 1307 sw->link_width = dual_lane ? TB_LINK_WIDTH_DUAL : 1308 TB_LINK_WIDTH_SINGLE; 1309 sw->rpm = force_rtd3; 1310 if (!sw->rpm) 1311 sw->rpm = intel_vss_is_rtd3(pkg->ep_name, 1312 sizeof(pkg->ep_name)); 1313 1314 if (add_switch(parent_sw, sw)) 1315 tb_switch_put(sw); 1316 } 1317 1318 pm_runtime_mark_last_busy(&parent_sw->dev); 1319 pm_runtime_put_autosuspend(&parent_sw->dev); 1320 1321 tb_switch_put(parent_sw); 1322 } 1323 1324 static void 1325 icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr) 1326 { 1327 __icm_tr_device_connected(tb, hdr, false); 1328 } 1329 1330 static void 1331 icm_tr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr) 1332 { 1333 const struct icm_tr_event_device_disconnected *pkg = 1334 (const struct icm_tr_event_device_disconnected *)hdr; 1335 struct tb_switch *sw; 1336 u64 route; 1337 1338 route = get_route(pkg->route_hi, pkg->route_lo); 1339 1340 sw = tb_switch_find_by_route(tb, route); 1341 if (!sw) { 1342 tb_warn(tb, "no switch exists at %llx, ignoring\n", route); 1343 return; 1344 } 1345 pm_runtime_get_sync(sw->dev.parent); 1346 1347 remove_switch(sw); 1348 1349 pm_runtime_mark_last_busy(sw->dev.parent); 1350 pm_runtime_put_autosuspend(sw->dev.parent); 1351 1352 tb_switch_put(sw); 1353 } 1354 1355 static void 1356 icm_tr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr) 1357 { 1358 const struct icm_tr_event_xdomain_connected *pkg = 1359 (const struct icm_tr_event_xdomain_connected *)hdr; 1360 struct tb_xdomain *xd; 1361 struct tb_switch *sw; 1362 u64 route; 1363 1364 if (!tb->root_switch) 1365 return; 1366 1367 route = get_route(pkg->local_route_hi, pkg->local_route_lo); 1368 1369 xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid); 1370 if (xd) { 1371 if (xd->route == route) { 1372 update_xdomain(xd, route, 0); 1373 tb_xdomain_put(xd); 1374 return; 1375 } 1376 1377 remove_xdomain(xd); 1378 tb_xdomain_put(xd); 1379 } 1380 1381 /* An existing xdomain with the same address */ 1382 xd = tb_xdomain_find_by_route(tb, route); 1383 if (xd) { 1384 remove_xdomain(xd); 1385 tb_xdomain_put(xd); 1386 } 1387 1388 /* 1389 * If the user disconnected a switch during suspend and 1390 * connected another host to the same port, remove the switch 1391 * first. 1392 */ 1393 sw = tb_switch_find_by_route(tb, route); 1394 if (sw) { 1395 remove_switch(sw); 1396 tb_switch_put(sw); 1397 } 1398 1399 sw = tb_switch_find_by_route(tb, get_parent_route(route)); 1400 if (!sw) { 1401 tb_warn(tb, "no switch exists at %llx, ignoring\n", route); 1402 return; 1403 } 1404 1405 add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, 0, 0); 1406 tb_switch_put(sw); 1407 } 1408 1409 static void 1410 icm_tr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr) 1411 { 1412 const struct icm_tr_event_xdomain_disconnected *pkg = 1413 (const struct icm_tr_event_xdomain_disconnected *)hdr; 1414 struct tb_xdomain *xd; 1415 u64 route; 1416 1417 route = get_route(pkg->route_hi, pkg->route_lo); 1418 1419 xd = tb_xdomain_find_by_route(tb, route); 1420 if (xd) { 1421 remove_xdomain(xd); 1422 tb_xdomain_put(xd); 1423 } 1424 } 1425 1426 static struct pci_dev *get_upstream_port(struct pci_dev *pdev) 1427 { 1428 struct pci_dev *parent; 1429 1430 parent = pci_upstream_bridge(pdev); 1431 while (parent) { 1432 if (!pci_is_pcie(parent)) 1433 return NULL; 1434 if (pci_pcie_type(parent) == PCI_EXP_TYPE_UPSTREAM) 1435 break; 1436 parent = pci_upstream_bridge(parent); 1437 } 1438 1439 if (!parent) 1440 return NULL; 1441 1442 switch (parent->device) { 1443 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_BRIDGE: 1444 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_BRIDGE: 1445 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_BRIDGE: 1446 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_BRIDGE: 1447 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_BRIDGE: 1448 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_BRIDGE: 1449 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_BRIDGE: 1450 return parent; 1451 } 1452 1453 return NULL; 1454 } 1455 1456 static bool icm_ar_is_supported(struct tb *tb) 1457 { 1458 struct pci_dev *upstream_port; 1459 struct icm *icm = tb_priv(tb); 1460 1461 /* 1462 * Starting from Alpine Ridge we can use ICM on Apple machines 1463 * as well. We just need to reset and re-enable it first. 1464 * However, only start it if explicitly asked by the user. 1465 */ 1466 if (icm_firmware_running(tb->nhi)) 1467 return true; 1468 if (!start_icm) 1469 return false; 1470 1471 /* 1472 * Find the upstream PCIe port in case we need to do reset 1473 * through its vendor specific registers. 1474 */ 1475 upstream_port = get_upstream_port(tb->nhi->pdev); 1476 if (upstream_port) { 1477 int cap; 1478 1479 cap = pci_find_ext_capability(upstream_port, 1480 PCI_EXT_CAP_ID_VNDR); 1481 if (cap > 0) { 1482 icm->upstream_port = upstream_port; 1483 icm->vnd_cap = cap; 1484 1485 return true; 1486 } 1487 } 1488 1489 return false; 1490 } 1491 1492 static int icm_ar_cio_reset(struct tb *tb) 1493 { 1494 return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x50, BIT(9)); 1495 } 1496 1497 static int icm_ar_get_mode(struct tb *tb) 1498 { 1499 struct tb_nhi *nhi = tb->nhi; 1500 int retries = 60; 1501 u32 val; 1502 1503 do { 1504 val = ioread32(nhi->iobase + REG_FW_STS); 1505 if (val & REG_FW_STS_NVM_AUTH_DONE) 1506 break; 1507 msleep(50); 1508 } while (--retries); 1509 1510 if (!retries) { 1511 dev_err(&nhi->pdev->dev, "ICM firmware not authenticated\n"); 1512 return -ENODEV; 1513 } 1514 1515 return nhi_mailbox_mode(nhi); 1516 } 1517 1518 static int 1519 icm_ar_driver_ready(struct tb *tb, enum tb_security_level *security_level, 1520 u8 *proto_version, size_t *nboot_acl, bool *rpm) 1521 { 1522 struct icm_ar_pkg_driver_ready_response reply; 1523 struct icm_pkg_driver_ready request = { 1524 .hdr.code = ICM_DRIVER_READY, 1525 }; 1526 int ret; 1527 1528 memset(&reply, 0, sizeof(reply)); 1529 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply), 1530 1, ICM_RETRIES, ICM_TIMEOUT); 1531 if (ret) 1532 return ret; 1533 1534 if (security_level) 1535 *security_level = reply.info & ICM_AR_INFO_SLEVEL_MASK; 1536 if (nboot_acl && (reply.info & ICM_AR_INFO_BOOT_ACL_SUPPORTED)) 1537 *nboot_acl = (reply.info & ICM_AR_INFO_BOOT_ACL_MASK) >> 1538 ICM_AR_INFO_BOOT_ACL_SHIFT; 1539 if (rpm) 1540 *rpm = !!(reply.hdr.flags & ICM_AR_FLAGS_RTD3); 1541 1542 return 0; 1543 } 1544 1545 static int icm_ar_get_route(struct tb *tb, u8 link, u8 depth, u64 *route) 1546 { 1547 struct icm_ar_pkg_get_route_response reply; 1548 struct icm_ar_pkg_get_route request = { 1549 .hdr = { .code = ICM_GET_ROUTE }, 1550 .link_info = depth << ICM_LINK_INFO_DEPTH_SHIFT | link, 1551 }; 1552 int ret; 1553 1554 memset(&reply, 0, sizeof(reply)); 1555 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply), 1556 1, ICM_RETRIES, ICM_TIMEOUT); 1557 if (ret) 1558 return ret; 1559 1560 if (reply.hdr.flags & ICM_FLAGS_ERROR) 1561 return -EIO; 1562 1563 *route = get_route(reply.route_hi, reply.route_lo); 1564 return 0; 1565 } 1566 1567 static int icm_ar_get_boot_acl(struct tb *tb, uuid_t *uuids, size_t nuuids) 1568 { 1569 struct icm_ar_pkg_preboot_acl_response reply; 1570 struct icm_ar_pkg_preboot_acl request = { 1571 .hdr = { .code = ICM_PREBOOT_ACL }, 1572 }; 1573 int ret, i; 1574 1575 memset(&reply, 0, sizeof(reply)); 1576 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply), 1577 1, ICM_RETRIES, ICM_TIMEOUT); 1578 if (ret) 1579 return ret; 1580 1581 if (reply.hdr.flags & ICM_FLAGS_ERROR) 1582 return -EIO; 1583 1584 for (i = 0; i < nuuids; i++) { 1585 u32 *uuid = (u32 *)&uuids[i]; 1586 1587 uuid[0] = reply.acl[i].uuid_lo; 1588 uuid[1] = reply.acl[i].uuid_hi; 1589 1590 if (uuid[0] == 0xffffffff && uuid[1] == 0xffffffff) { 1591 /* Map empty entries to null UUID */ 1592 uuid[0] = 0; 1593 uuid[1] = 0; 1594 } else if (uuid[0] != 0 || uuid[1] != 0) { 1595 /* Upper two DWs are always one's */ 1596 uuid[2] = 0xffffffff; 1597 uuid[3] = 0xffffffff; 1598 } 1599 } 1600 1601 return ret; 1602 } 1603 1604 static int icm_ar_set_boot_acl(struct tb *tb, const uuid_t *uuids, 1605 size_t nuuids) 1606 { 1607 struct icm_ar_pkg_preboot_acl_response reply; 1608 struct icm_ar_pkg_preboot_acl request = { 1609 .hdr = { 1610 .code = ICM_PREBOOT_ACL, 1611 .flags = ICM_FLAGS_WRITE, 1612 }, 1613 }; 1614 int ret, i; 1615 1616 for (i = 0; i < nuuids; i++) { 1617 const u32 *uuid = (const u32 *)&uuids[i]; 1618 1619 if (uuid_is_null(&uuids[i])) { 1620 /* 1621 * Map null UUID to the empty (all one) entries 1622 * for ICM. 1623 */ 1624 request.acl[i].uuid_lo = 0xffffffff; 1625 request.acl[i].uuid_hi = 0xffffffff; 1626 } else { 1627 /* Two high DWs need to be set to all one */ 1628 if (uuid[2] != 0xffffffff || uuid[3] != 0xffffffff) 1629 return -EINVAL; 1630 1631 request.acl[i].uuid_lo = uuid[0]; 1632 request.acl[i].uuid_hi = uuid[1]; 1633 } 1634 } 1635 1636 memset(&reply, 0, sizeof(reply)); 1637 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply), 1638 1, ICM_RETRIES, ICM_TIMEOUT); 1639 if (ret) 1640 return ret; 1641 1642 if (reply.hdr.flags & ICM_FLAGS_ERROR) 1643 return -EIO; 1644 1645 return 0; 1646 } 1647 1648 static int 1649 icm_icl_driver_ready(struct tb *tb, enum tb_security_level *security_level, 1650 u8 *proto_version, size_t *nboot_acl, bool *rpm) 1651 { 1652 struct icm_tr_pkg_driver_ready_response reply; 1653 struct icm_pkg_driver_ready request = { 1654 .hdr.code = ICM_DRIVER_READY, 1655 }; 1656 int ret; 1657 1658 memset(&reply, 0, sizeof(reply)); 1659 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply), 1660 1, ICM_RETRIES, 20000); 1661 if (ret) 1662 return ret; 1663 1664 if (proto_version) 1665 *proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >> 1666 ICM_TR_INFO_PROTO_VERSION_SHIFT; 1667 1668 /* Ice Lake always supports RTD3 */ 1669 if (rpm) 1670 *rpm = true; 1671 1672 return 0; 1673 } 1674 1675 static void icm_icl_set_uuid(struct tb *tb) 1676 { 1677 struct tb_nhi *nhi = tb->nhi; 1678 u32 uuid[4]; 1679 1680 pci_read_config_dword(nhi->pdev, VS_CAP_10, &uuid[0]); 1681 pci_read_config_dword(nhi->pdev, VS_CAP_11, &uuid[1]); 1682 uuid[2] = 0xffffffff; 1683 uuid[3] = 0xffffffff; 1684 1685 tb->root_switch->uuid = kmemdup(uuid, sizeof(uuid), GFP_KERNEL); 1686 } 1687 1688 static void 1689 icm_icl_device_connected(struct tb *tb, const struct icm_pkg_header *hdr) 1690 { 1691 __icm_tr_device_connected(tb, hdr, true); 1692 } 1693 1694 static void icm_icl_rtd3_veto(struct tb *tb, const struct icm_pkg_header *hdr) 1695 { 1696 const struct icm_icl_event_rtd3_veto *pkg = 1697 (const struct icm_icl_event_rtd3_veto *)hdr; 1698 1699 tb_dbg(tb, "ICM rtd3 veto=0x%08x\n", pkg->veto_reason); 1700 1701 if (pkg->veto_reason) 1702 icm_veto_begin(tb); 1703 else 1704 icm_veto_end(tb); 1705 } 1706 1707 static bool icm_tgl_is_supported(struct tb *tb) 1708 { 1709 unsigned long end = jiffies + msecs_to_jiffies(10); 1710 1711 do { 1712 u32 val; 1713 1714 val = ioread32(tb->nhi->iobase + REG_FW_STS); 1715 if (val & REG_FW_STS_NVM_AUTH_DONE) 1716 return true; 1717 usleep_range(100, 500); 1718 } while (time_before(jiffies, end)); 1719 1720 return false; 1721 } 1722 1723 static void icm_handle_notification(struct work_struct *work) 1724 { 1725 struct icm_notification *n = container_of(work, typeof(*n), work); 1726 struct tb *tb = n->tb; 1727 struct icm *icm = tb_priv(tb); 1728 1729 mutex_lock(&tb->lock); 1730 1731 /* 1732 * When the domain is stopped we flush its workqueue but before 1733 * that the root switch is removed. In that case we should treat 1734 * the queued events as being canceled. 1735 */ 1736 if (tb->root_switch) { 1737 switch (n->pkg->code) { 1738 case ICM_EVENT_DEVICE_CONNECTED: 1739 icm->device_connected(tb, n->pkg); 1740 break; 1741 case ICM_EVENT_DEVICE_DISCONNECTED: 1742 icm->device_disconnected(tb, n->pkg); 1743 break; 1744 case ICM_EVENT_XDOMAIN_CONNECTED: 1745 if (tb_is_xdomain_enabled()) 1746 icm->xdomain_connected(tb, n->pkg); 1747 break; 1748 case ICM_EVENT_XDOMAIN_DISCONNECTED: 1749 if (tb_is_xdomain_enabled()) 1750 icm->xdomain_disconnected(tb, n->pkg); 1751 break; 1752 case ICM_EVENT_DP_CONFIG_CHANGED: 1753 icm_dp_event(tb); 1754 break; 1755 case ICM_EVENT_RTD3_VETO: 1756 icm->rtd3_veto(tb, n->pkg); 1757 break; 1758 } 1759 } 1760 1761 mutex_unlock(&tb->lock); 1762 1763 kfree(n->pkg); 1764 kfree(n); 1765 } 1766 1767 static void icm_handle_event(struct tb *tb, enum tb_cfg_pkg_type type, 1768 const void *buf, size_t size) 1769 { 1770 struct icm_notification *n; 1771 1772 n = kmalloc(sizeof(*n), GFP_KERNEL); 1773 if (!n) 1774 return; 1775 1776 n->pkg = kmemdup(buf, size, GFP_KERNEL); 1777 if (!n->pkg) { 1778 kfree(n); 1779 return; 1780 } 1781 1782 INIT_WORK(&n->work, icm_handle_notification); 1783 n->tb = tb; 1784 1785 queue_work(tb->wq, &n->work); 1786 } 1787 1788 static int 1789 __icm_driver_ready(struct tb *tb, enum tb_security_level *security_level, 1790 u8 *proto_version, size_t *nboot_acl, bool *rpm) 1791 { 1792 struct icm *icm = tb_priv(tb); 1793 unsigned int retries = 50; 1794 int ret; 1795 1796 ret = icm->driver_ready(tb, security_level, proto_version, nboot_acl, 1797 rpm); 1798 if (ret) { 1799 tb_err(tb, "failed to send driver ready to ICM\n"); 1800 return ret; 1801 } 1802 1803 /* 1804 * Hold on here until the switch config space is accessible so 1805 * that we can read root switch config successfully. 1806 */ 1807 do { 1808 struct tb_cfg_result res; 1809 u32 tmp; 1810 1811 res = tb_cfg_read_raw(tb->ctl, &tmp, 0, 0, TB_CFG_SWITCH, 1812 0, 1, 100); 1813 if (!res.err) 1814 return 0; 1815 1816 msleep(50); 1817 } while (--retries); 1818 1819 tb_err(tb, "failed to read root switch config space, giving up\n"); 1820 return -ETIMEDOUT; 1821 } 1822 1823 static int icm_firmware_reset(struct tb *tb, struct tb_nhi *nhi) 1824 { 1825 struct icm *icm = tb_priv(tb); 1826 u32 val; 1827 1828 if (!icm->upstream_port) 1829 return -ENODEV; 1830 1831 /* Put ARC to wait for CIO reset event to happen */ 1832 val = ioread32(nhi->iobase + REG_FW_STS); 1833 val |= REG_FW_STS_CIO_RESET_REQ; 1834 iowrite32(val, nhi->iobase + REG_FW_STS); 1835 1836 /* Re-start ARC */ 1837 val = ioread32(nhi->iobase + REG_FW_STS); 1838 val |= REG_FW_STS_ICM_EN_INVERT; 1839 val |= REG_FW_STS_ICM_EN_CPU; 1840 iowrite32(val, nhi->iobase + REG_FW_STS); 1841 1842 /* Trigger CIO reset now */ 1843 return icm->cio_reset(tb); 1844 } 1845 1846 static int icm_firmware_start(struct tb *tb, struct tb_nhi *nhi) 1847 { 1848 unsigned int retries = 10; 1849 int ret; 1850 u32 val; 1851 1852 /* Check if the ICM firmware is already running */ 1853 if (icm_firmware_running(nhi)) 1854 return 0; 1855 1856 dev_dbg(&nhi->pdev->dev, "starting ICM firmware\n"); 1857 1858 ret = icm_firmware_reset(tb, nhi); 1859 if (ret) 1860 return ret; 1861 1862 /* Wait until the ICM firmware tells us it is up and running */ 1863 do { 1864 /* Check that the ICM firmware is running */ 1865 val = ioread32(nhi->iobase + REG_FW_STS); 1866 if (val & REG_FW_STS_NVM_AUTH_DONE) 1867 return 0; 1868 1869 msleep(300); 1870 } while (--retries); 1871 1872 return -ETIMEDOUT; 1873 } 1874 1875 static int icm_reset_phy_port(struct tb *tb, int phy_port) 1876 { 1877 struct icm *icm = tb_priv(tb); 1878 u32 state0, state1; 1879 int port0, port1; 1880 u32 val0, val1; 1881 int ret; 1882 1883 if (!icm->upstream_port) 1884 return 0; 1885 1886 if (phy_port) { 1887 port0 = 3; 1888 port1 = 4; 1889 } else { 1890 port0 = 1; 1891 port1 = 2; 1892 } 1893 1894 /* 1895 * Read link status of both null ports belonging to a single 1896 * physical port. 1897 */ 1898 ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0); 1899 if (ret) 1900 return ret; 1901 ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1); 1902 if (ret) 1903 return ret; 1904 1905 state0 = val0 & PHY_PORT_CS1_LINK_STATE_MASK; 1906 state0 >>= PHY_PORT_CS1_LINK_STATE_SHIFT; 1907 state1 = val1 & PHY_PORT_CS1_LINK_STATE_MASK; 1908 state1 >>= PHY_PORT_CS1_LINK_STATE_SHIFT; 1909 1910 /* If they are both up we need to reset them now */ 1911 if (state0 != TB_PORT_UP || state1 != TB_PORT_UP) 1912 return 0; 1913 1914 val0 |= PHY_PORT_CS1_LINK_DISABLE; 1915 ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0); 1916 if (ret) 1917 return ret; 1918 1919 val1 |= PHY_PORT_CS1_LINK_DISABLE; 1920 ret = pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1); 1921 if (ret) 1922 return ret; 1923 1924 /* Wait a bit and then re-enable both ports */ 1925 usleep_range(10, 100); 1926 1927 ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0); 1928 if (ret) 1929 return ret; 1930 ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1); 1931 if (ret) 1932 return ret; 1933 1934 val0 &= ~PHY_PORT_CS1_LINK_DISABLE; 1935 ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0); 1936 if (ret) 1937 return ret; 1938 1939 val1 &= ~PHY_PORT_CS1_LINK_DISABLE; 1940 return pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1); 1941 } 1942 1943 static int icm_firmware_init(struct tb *tb) 1944 { 1945 struct icm *icm = tb_priv(tb); 1946 struct tb_nhi *nhi = tb->nhi; 1947 int ret; 1948 1949 ret = icm_firmware_start(tb, nhi); 1950 if (ret) { 1951 dev_err(&nhi->pdev->dev, "could not start ICM firmware\n"); 1952 return ret; 1953 } 1954 1955 if (icm->get_mode) { 1956 ret = icm->get_mode(tb); 1957 1958 switch (ret) { 1959 case NHI_FW_SAFE_MODE: 1960 icm->safe_mode = true; 1961 break; 1962 1963 case NHI_FW_CM_MODE: 1964 /* Ask ICM to accept all Thunderbolt devices */ 1965 nhi_mailbox_cmd(nhi, NHI_MAILBOX_ALLOW_ALL_DEVS, 0); 1966 break; 1967 1968 default: 1969 if (ret < 0) 1970 return ret; 1971 1972 tb_err(tb, "ICM firmware is in wrong mode: %u\n", ret); 1973 return -ENODEV; 1974 } 1975 } 1976 1977 /* 1978 * Reset both physical ports if there is anything connected to 1979 * them already. 1980 */ 1981 ret = icm_reset_phy_port(tb, 0); 1982 if (ret) 1983 dev_warn(&nhi->pdev->dev, "failed to reset links on port0\n"); 1984 ret = icm_reset_phy_port(tb, 1); 1985 if (ret) 1986 dev_warn(&nhi->pdev->dev, "failed to reset links on port1\n"); 1987 1988 return 0; 1989 } 1990 1991 static int icm_driver_ready(struct tb *tb) 1992 { 1993 struct icm *icm = tb_priv(tb); 1994 int ret; 1995 1996 ret = icm_firmware_init(tb); 1997 if (ret) 1998 return ret; 1999 2000 if (icm->safe_mode) { 2001 tb_info(tb, "Thunderbolt host controller is in safe mode.\n"); 2002 tb_info(tb, "You need to update NVM firmware of the controller before it can be used.\n"); 2003 tb_info(tb, "For latest updates check https://thunderbolttechnology.net/updates.\n"); 2004 return 0; 2005 } 2006 2007 ret = __icm_driver_ready(tb, &tb->security_level, &icm->proto_version, 2008 &tb->nboot_acl, &icm->rpm); 2009 if (ret) 2010 return ret; 2011 2012 /* 2013 * Make sure the number of supported preboot ACL matches what we 2014 * expect or disable the whole feature. 2015 */ 2016 if (tb->nboot_acl > icm->max_boot_acl) 2017 tb->nboot_acl = 0; 2018 2019 if (icm->proto_version >= 3) 2020 tb_dbg(tb, "USB4 proxy operations supported\n"); 2021 2022 return 0; 2023 } 2024 2025 static int icm_suspend(struct tb *tb) 2026 { 2027 struct icm *icm = tb_priv(tb); 2028 2029 if (icm->save_devices) 2030 icm->save_devices(tb); 2031 2032 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0); 2033 return 0; 2034 } 2035 2036 /* 2037 * Mark all switches (except root switch) below this one unplugged. ICM 2038 * firmware will send us an updated list of switches after we have send 2039 * it driver ready command. If a switch is not in that list it will be 2040 * removed when we perform rescan. 2041 */ 2042 static void icm_unplug_children(struct tb_switch *sw) 2043 { 2044 struct tb_port *port; 2045 2046 if (tb_route(sw)) 2047 sw->is_unplugged = true; 2048 2049 tb_switch_for_each_port(sw, port) { 2050 if (port->xdomain) 2051 port->xdomain->is_unplugged = true; 2052 else if (tb_port_has_remote(port)) 2053 icm_unplug_children(port->remote->sw); 2054 } 2055 } 2056 2057 static int complete_rpm(struct device *dev, void *data) 2058 { 2059 struct tb_switch *sw = tb_to_switch(dev); 2060 2061 if (sw) 2062 complete(&sw->rpm_complete); 2063 return 0; 2064 } 2065 2066 static void remove_unplugged_switch(struct tb_switch *sw) 2067 { 2068 struct device *parent = get_device(sw->dev.parent); 2069 2070 pm_runtime_get_sync(parent); 2071 2072 /* 2073 * Signal this and switches below for rpm_complete because 2074 * tb_switch_remove() calls pm_runtime_get_sync() that then waits 2075 * for it. 2076 */ 2077 complete_rpm(&sw->dev, NULL); 2078 bus_for_each_dev(&tb_bus_type, &sw->dev, NULL, complete_rpm); 2079 tb_switch_remove(sw); 2080 2081 pm_runtime_mark_last_busy(parent); 2082 pm_runtime_put_autosuspend(parent); 2083 2084 put_device(parent); 2085 } 2086 2087 static void icm_free_unplugged_children(struct tb_switch *sw) 2088 { 2089 struct tb_port *port; 2090 2091 tb_switch_for_each_port(sw, port) { 2092 if (port->xdomain && port->xdomain->is_unplugged) { 2093 tb_xdomain_remove(port->xdomain); 2094 port->xdomain = NULL; 2095 } else if (tb_port_has_remote(port)) { 2096 if (port->remote->sw->is_unplugged) { 2097 remove_unplugged_switch(port->remote->sw); 2098 port->remote = NULL; 2099 } else { 2100 icm_free_unplugged_children(port->remote->sw); 2101 } 2102 } 2103 } 2104 } 2105 2106 static void icm_rescan_work(struct work_struct *work) 2107 { 2108 struct icm *icm = container_of(work, struct icm, rescan_work.work); 2109 struct tb *tb = icm_to_tb(icm); 2110 2111 mutex_lock(&tb->lock); 2112 if (tb->root_switch) 2113 icm_free_unplugged_children(tb->root_switch); 2114 mutex_unlock(&tb->lock); 2115 } 2116 2117 static void icm_complete(struct tb *tb) 2118 { 2119 struct icm *icm = tb_priv(tb); 2120 2121 if (tb->nhi->going_away) 2122 return; 2123 2124 /* 2125 * If RTD3 was vetoed before we entered system suspend allow it 2126 * again now before driver ready is sent. Firmware sends a new RTD3 2127 * veto if it is still the case after we have sent it driver ready 2128 * command. 2129 */ 2130 icm_veto_end(tb); 2131 icm_unplug_children(tb->root_switch); 2132 2133 /* 2134 * Now all existing children should be resumed, start events 2135 * from ICM to get updated status. 2136 */ 2137 __icm_driver_ready(tb, NULL, NULL, NULL, NULL); 2138 2139 /* 2140 * We do not get notifications of devices that have been 2141 * unplugged during suspend so schedule rescan to clean them up 2142 * if any. 2143 */ 2144 queue_delayed_work(tb->wq, &icm->rescan_work, msecs_to_jiffies(500)); 2145 } 2146 2147 static int icm_runtime_suspend(struct tb *tb) 2148 { 2149 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0); 2150 return 0; 2151 } 2152 2153 static int icm_runtime_suspend_switch(struct tb_switch *sw) 2154 { 2155 if (tb_route(sw)) 2156 reinit_completion(&sw->rpm_complete); 2157 return 0; 2158 } 2159 2160 static int icm_runtime_resume_switch(struct tb_switch *sw) 2161 { 2162 if (tb_route(sw)) { 2163 if (!wait_for_completion_timeout(&sw->rpm_complete, 2164 msecs_to_jiffies(500))) { 2165 dev_dbg(&sw->dev, "runtime resuming timed out\n"); 2166 } 2167 } 2168 return 0; 2169 } 2170 2171 static int icm_runtime_resume(struct tb *tb) 2172 { 2173 /* 2174 * We can reuse the same resume functionality than with system 2175 * suspend. 2176 */ 2177 icm_complete(tb); 2178 return 0; 2179 } 2180 2181 static int icm_start(struct tb *tb, bool not_used) 2182 { 2183 struct icm *icm = tb_priv(tb); 2184 int ret; 2185 2186 if (icm->safe_mode) 2187 tb->root_switch = tb_switch_alloc_safe_mode(tb, &tb->dev, 0); 2188 else 2189 tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0); 2190 if (IS_ERR(tb->root_switch)) 2191 return PTR_ERR(tb->root_switch); 2192 2193 tb->root_switch->no_nvm_upgrade = !icm->can_upgrade_nvm; 2194 tb->root_switch->rpm = icm->rpm; 2195 2196 if (icm->set_uuid) 2197 icm->set_uuid(tb); 2198 2199 ret = tb_switch_add(tb->root_switch); 2200 if (ret) { 2201 tb_switch_put(tb->root_switch); 2202 tb->root_switch = NULL; 2203 } 2204 2205 return ret; 2206 } 2207 2208 static void icm_stop(struct tb *tb) 2209 { 2210 struct icm *icm = tb_priv(tb); 2211 2212 cancel_delayed_work(&icm->rescan_work); 2213 tb_switch_remove(tb->root_switch); 2214 tb->root_switch = NULL; 2215 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0); 2216 kfree(icm->last_nvm_auth); 2217 icm->last_nvm_auth = NULL; 2218 } 2219 2220 static int icm_disconnect_pcie_paths(struct tb *tb) 2221 { 2222 return nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DISCONNECT_PCIE_PATHS, 0); 2223 } 2224 2225 static void icm_usb4_switch_nvm_auth_complete(void *data) 2226 { 2227 struct usb4_switch_nvm_auth *auth = data; 2228 struct icm *icm = auth->icm; 2229 struct tb *tb = icm_to_tb(icm); 2230 2231 tb_dbg(tb, "NVM_AUTH response for %llx flags %#x status %#x\n", 2232 get_route(auth->reply.route_hi, auth->reply.route_lo), 2233 auth->reply.hdr.flags, auth->reply.status); 2234 2235 mutex_lock(&tb->lock); 2236 if (WARN_ON(icm->last_nvm_auth)) 2237 kfree(icm->last_nvm_auth); 2238 icm->last_nvm_auth = auth; 2239 mutex_unlock(&tb->lock); 2240 } 2241 2242 static int icm_usb4_switch_nvm_authenticate(struct tb *tb, u64 route) 2243 { 2244 struct usb4_switch_nvm_auth *auth; 2245 struct icm *icm = tb_priv(tb); 2246 struct tb_cfg_request *req; 2247 int ret; 2248 2249 auth = kzalloc(sizeof(*auth), GFP_KERNEL); 2250 if (!auth) 2251 return -ENOMEM; 2252 2253 auth->icm = icm; 2254 auth->request.hdr.code = ICM_USB4_SWITCH_OP; 2255 auth->request.route_hi = upper_32_bits(route); 2256 auth->request.route_lo = lower_32_bits(route); 2257 auth->request.opcode = USB4_SWITCH_OP_NVM_AUTH; 2258 2259 req = tb_cfg_request_alloc(); 2260 if (!req) { 2261 ret = -ENOMEM; 2262 goto err_free_auth; 2263 } 2264 2265 req->match = icm_match; 2266 req->copy = icm_copy; 2267 req->request = &auth->request; 2268 req->request_size = sizeof(auth->request); 2269 req->request_type = TB_CFG_PKG_ICM_CMD; 2270 req->response = &auth->reply; 2271 req->npackets = 1; 2272 req->response_size = sizeof(auth->reply); 2273 req->response_type = TB_CFG_PKG_ICM_RESP; 2274 2275 tb_dbg(tb, "NVM_AUTH request for %llx\n", route); 2276 2277 mutex_lock(&icm->request_lock); 2278 ret = tb_cfg_request(tb->ctl, req, icm_usb4_switch_nvm_auth_complete, 2279 auth); 2280 mutex_unlock(&icm->request_lock); 2281 2282 tb_cfg_request_put(req); 2283 if (ret) 2284 goto err_free_auth; 2285 return 0; 2286 2287 err_free_auth: 2288 kfree(auth); 2289 return ret; 2290 } 2291 2292 static int icm_usb4_switch_op(struct tb_switch *sw, u16 opcode, u32 *metadata, 2293 u8 *status, const void *tx_data, size_t tx_data_len, 2294 void *rx_data, size_t rx_data_len) 2295 { 2296 struct icm_usb4_switch_op_response reply; 2297 struct icm_usb4_switch_op request; 2298 struct tb *tb = sw->tb; 2299 struct icm *icm = tb_priv(tb); 2300 u64 route = tb_route(sw); 2301 int ret; 2302 2303 /* 2304 * USB4 router operation proxy is supported in firmware if the 2305 * protocol version is 3 or higher. 2306 */ 2307 if (icm->proto_version < 3) 2308 return -EOPNOTSUPP; 2309 2310 /* 2311 * NVM_AUTH is a special USB4 proxy operation that does not 2312 * return immediately so handle it separately. 2313 */ 2314 if (opcode == USB4_SWITCH_OP_NVM_AUTH) 2315 return icm_usb4_switch_nvm_authenticate(tb, route); 2316 2317 memset(&request, 0, sizeof(request)); 2318 request.hdr.code = ICM_USB4_SWITCH_OP; 2319 request.route_hi = upper_32_bits(route); 2320 request.route_lo = lower_32_bits(route); 2321 request.opcode = opcode; 2322 if (metadata) 2323 request.metadata = *metadata; 2324 2325 if (tx_data_len) { 2326 request.data_len_valid |= ICM_USB4_SWITCH_DATA_VALID; 2327 if (tx_data_len < ARRAY_SIZE(request.data)) 2328 request.data_len_valid = 2329 tx_data_len & ICM_USB4_SWITCH_DATA_LEN_MASK; 2330 memcpy(request.data, tx_data, tx_data_len * sizeof(u32)); 2331 } 2332 2333 memset(&reply, 0, sizeof(reply)); 2334 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply), 2335 1, ICM_RETRIES, ICM_TIMEOUT); 2336 if (ret) 2337 return ret; 2338 2339 if (reply.hdr.flags & ICM_FLAGS_ERROR) 2340 return -EIO; 2341 2342 if (status) 2343 *status = reply.status; 2344 2345 if (metadata) 2346 *metadata = reply.metadata; 2347 2348 if (rx_data_len) 2349 memcpy(rx_data, reply.data, rx_data_len * sizeof(u32)); 2350 2351 return 0; 2352 } 2353 2354 static int icm_usb4_switch_nvm_authenticate_status(struct tb_switch *sw, 2355 u32 *status) 2356 { 2357 struct usb4_switch_nvm_auth *auth; 2358 struct tb *tb = sw->tb; 2359 struct icm *icm = tb_priv(tb); 2360 int ret = 0; 2361 2362 if (icm->proto_version < 3) 2363 return -EOPNOTSUPP; 2364 2365 auth = icm->last_nvm_auth; 2366 icm->last_nvm_auth = NULL; 2367 2368 if (auth && auth->reply.route_hi == sw->config.route_hi && 2369 auth->reply.route_lo == sw->config.route_lo) { 2370 tb_dbg(tb, "NVM_AUTH found for %llx flags %#x status %#x\n", 2371 tb_route(sw), auth->reply.hdr.flags, auth->reply.status); 2372 if (auth->reply.hdr.flags & ICM_FLAGS_ERROR) 2373 ret = -EIO; 2374 else 2375 *status = auth->reply.status; 2376 } else { 2377 *status = 0; 2378 } 2379 2380 kfree(auth); 2381 return ret; 2382 } 2383 2384 /* Falcon Ridge */ 2385 static const struct tb_cm_ops icm_fr_ops = { 2386 .driver_ready = icm_driver_ready, 2387 .start = icm_start, 2388 .stop = icm_stop, 2389 .suspend = icm_suspend, 2390 .complete = icm_complete, 2391 .handle_event = icm_handle_event, 2392 .approve_switch = icm_fr_approve_switch, 2393 .add_switch_key = icm_fr_add_switch_key, 2394 .challenge_switch_key = icm_fr_challenge_switch_key, 2395 .disconnect_pcie_paths = icm_disconnect_pcie_paths, 2396 .approve_xdomain_paths = icm_fr_approve_xdomain_paths, 2397 .disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths, 2398 }; 2399 2400 /* Alpine Ridge */ 2401 static const struct tb_cm_ops icm_ar_ops = { 2402 .driver_ready = icm_driver_ready, 2403 .start = icm_start, 2404 .stop = icm_stop, 2405 .suspend = icm_suspend, 2406 .complete = icm_complete, 2407 .runtime_suspend = icm_runtime_suspend, 2408 .runtime_resume = icm_runtime_resume, 2409 .runtime_suspend_switch = icm_runtime_suspend_switch, 2410 .runtime_resume_switch = icm_runtime_resume_switch, 2411 .handle_event = icm_handle_event, 2412 .get_boot_acl = icm_ar_get_boot_acl, 2413 .set_boot_acl = icm_ar_set_boot_acl, 2414 .approve_switch = icm_fr_approve_switch, 2415 .add_switch_key = icm_fr_add_switch_key, 2416 .challenge_switch_key = icm_fr_challenge_switch_key, 2417 .disconnect_pcie_paths = icm_disconnect_pcie_paths, 2418 .approve_xdomain_paths = icm_fr_approve_xdomain_paths, 2419 .disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths, 2420 }; 2421 2422 /* Titan Ridge */ 2423 static const struct tb_cm_ops icm_tr_ops = { 2424 .driver_ready = icm_driver_ready, 2425 .start = icm_start, 2426 .stop = icm_stop, 2427 .suspend = icm_suspend, 2428 .complete = icm_complete, 2429 .runtime_suspend = icm_runtime_suspend, 2430 .runtime_resume = icm_runtime_resume, 2431 .runtime_suspend_switch = icm_runtime_suspend_switch, 2432 .runtime_resume_switch = icm_runtime_resume_switch, 2433 .handle_event = icm_handle_event, 2434 .get_boot_acl = icm_ar_get_boot_acl, 2435 .set_boot_acl = icm_ar_set_boot_acl, 2436 .approve_switch = icm_tr_approve_switch, 2437 .add_switch_key = icm_tr_add_switch_key, 2438 .challenge_switch_key = icm_tr_challenge_switch_key, 2439 .disconnect_pcie_paths = icm_disconnect_pcie_paths, 2440 .approve_xdomain_paths = icm_tr_approve_xdomain_paths, 2441 .disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths, 2442 .usb4_switch_op = icm_usb4_switch_op, 2443 .usb4_switch_nvm_authenticate_status = 2444 icm_usb4_switch_nvm_authenticate_status, 2445 }; 2446 2447 /* Ice Lake */ 2448 static const struct tb_cm_ops icm_icl_ops = { 2449 .driver_ready = icm_driver_ready, 2450 .start = icm_start, 2451 .stop = icm_stop, 2452 .complete = icm_complete, 2453 .runtime_suspend = icm_runtime_suspend, 2454 .runtime_resume = icm_runtime_resume, 2455 .handle_event = icm_handle_event, 2456 .approve_xdomain_paths = icm_tr_approve_xdomain_paths, 2457 .disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths, 2458 .usb4_switch_op = icm_usb4_switch_op, 2459 .usb4_switch_nvm_authenticate_status = 2460 icm_usb4_switch_nvm_authenticate_status, 2461 }; 2462 2463 struct tb *icm_probe(struct tb_nhi *nhi) 2464 { 2465 struct icm *icm; 2466 struct tb *tb; 2467 2468 tb = tb_domain_alloc(nhi, ICM_TIMEOUT, sizeof(struct icm)); 2469 if (!tb) 2470 return NULL; 2471 2472 icm = tb_priv(tb); 2473 INIT_DELAYED_WORK(&icm->rescan_work, icm_rescan_work); 2474 mutex_init(&icm->request_lock); 2475 2476 switch (nhi->pdev->device) { 2477 case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI: 2478 case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI: 2479 icm->can_upgrade_nvm = true; 2480 icm->is_supported = icm_fr_is_supported; 2481 icm->get_route = icm_fr_get_route; 2482 icm->save_devices = icm_fr_save_devices; 2483 icm->driver_ready = icm_fr_driver_ready; 2484 icm->device_connected = icm_fr_device_connected; 2485 icm->device_disconnected = icm_fr_device_disconnected; 2486 icm->xdomain_connected = icm_fr_xdomain_connected; 2487 icm->xdomain_disconnected = icm_fr_xdomain_disconnected; 2488 tb->cm_ops = &icm_fr_ops; 2489 break; 2490 2491 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_NHI: 2492 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_NHI: 2493 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_NHI: 2494 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_NHI: 2495 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_NHI: 2496 icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES; 2497 /* 2498 * NVM upgrade has not been tested on Apple systems and 2499 * they don't provide images publicly either. To be on 2500 * the safe side prevent root switch NVM upgrade on Macs 2501 * for now. 2502 */ 2503 icm->can_upgrade_nvm = !x86_apple_machine; 2504 icm->is_supported = icm_ar_is_supported; 2505 icm->cio_reset = icm_ar_cio_reset; 2506 icm->get_mode = icm_ar_get_mode; 2507 icm->get_route = icm_ar_get_route; 2508 icm->save_devices = icm_fr_save_devices; 2509 icm->driver_ready = icm_ar_driver_ready; 2510 icm->device_connected = icm_fr_device_connected; 2511 icm->device_disconnected = icm_fr_device_disconnected; 2512 icm->xdomain_connected = icm_fr_xdomain_connected; 2513 icm->xdomain_disconnected = icm_fr_xdomain_disconnected; 2514 tb->cm_ops = &icm_ar_ops; 2515 break; 2516 2517 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_NHI: 2518 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_NHI: 2519 icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES; 2520 icm->can_upgrade_nvm = !x86_apple_machine; 2521 icm->is_supported = icm_ar_is_supported; 2522 icm->cio_reset = icm_tr_cio_reset; 2523 icm->get_mode = icm_ar_get_mode; 2524 icm->driver_ready = icm_tr_driver_ready; 2525 icm->device_connected = icm_tr_device_connected; 2526 icm->device_disconnected = icm_tr_device_disconnected; 2527 icm->xdomain_connected = icm_tr_xdomain_connected; 2528 icm->xdomain_disconnected = icm_tr_xdomain_disconnected; 2529 tb->cm_ops = &icm_tr_ops; 2530 break; 2531 2532 case PCI_DEVICE_ID_INTEL_ICL_NHI0: 2533 case PCI_DEVICE_ID_INTEL_ICL_NHI1: 2534 icm->is_supported = icm_fr_is_supported; 2535 icm->driver_ready = icm_icl_driver_ready; 2536 icm->set_uuid = icm_icl_set_uuid; 2537 icm->device_connected = icm_icl_device_connected; 2538 icm->device_disconnected = icm_tr_device_disconnected; 2539 icm->xdomain_connected = icm_tr_xdomain_connected; 2540 icm->xdomain_disconnected = icm_tr_xdomain_disconnected; 2541 icm->rtd3_veto = icm_icl_rtd3_veto; 2542 tb->cm_ops = &icm_icl_ops; 2543 break; 2544 2545 case PCI_DEVICE_ID_INTEL_TGL_NHI0: 2546 case PCI_DEVICE_ID_INTEL_TGL_NHI1: 2547 case PCI_DEVICE_ID_INTEL_TGL_H_NHI0: 2548 case PCI_DEVICE_ID_INTEL_TGL_H_NHI1: 2549 case PCI_DEVICE_ID_INTEL_ADL_NHI0: 2550 case PCI_DEVICE_ID_INTEL_ADL_NHI1: 2551 case PCI_DEVICE_ID_INTEL_RPL_NHI0: 2552 case PCI_DEVICE_ID_INTEL_RPL_NHI1: 2553 case PCI_DEVICE_ID_INTEL_MTL_M_NHI0: 2554 case PCI_DEVICE_ID_INTEL_MTL_P_NHI0: 2555 case PCI_DEVICE_ID_INTEL_MTL_P_NHI1: 2556 icm->is_supported = icm_tgl_is_supported; 2557 icm->driver_ready = icm_icl_driver_ready; 2558 icm->set_uuid = icm_icl_set_uuid; 2559 icm->device_connected = icm_icl_device_connected; 2560 icm->device_disconnected = icm_tr_device_disconnected; 2561 icm->xdomain_connected = icm_tr_xdomain_connected; 2562 icm->xdomain_disconnected = icm_tr_xdomain_disconnected; 2563 icm->rtd3_veto = icm_icl_rtd3_veto; 2564 tb->cm_ops = &icm_icl_ops; 2565 break; 2566 2567 case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_2C_NHI: 2568 case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_4C_NHI: 2569 icm->can_upgrade_nvm = true; 2570 icm->is_supported = icm_tgl_is_supported; 2571 icm->get_mode = icm_ar_get_mode; 2572 icm->driver_ready = icm_tr_driver_ready; 2573 icm->device_connected = icm_tr_device_connected; 2574 icm->device_disconnected = icm_tr_device_disconnected; 2575 icm->xdomain_connected = icm_tr_xdomain_connected; 2576 icm->xdomain_disconnected = icm_tr_xdomain_disconnected; 2577 tb->cm_ops = &icm_tr_ops; 2578 break; 2579 } 2580 2581 if (!icm->is_supported || !icm->is_supported(tb)) { 2582 dev_dbg(&nhi->pdev->dev, "ICM not supported on this controller\n"); 2583 tb_domain_put(tb); 2584 return NULL; 2585 } 2586 2587 tb_dbg(tb, "using firmware connection manager\n"); 2588 2589 return tb; 2590 } 2591