Lines Matching +full:dma +full:- +full:router
1 // SPDX-License-Identifier: GPL-2.0
3 * Thunderbolt driver - bus logic (NHI independent)
30 * direction. This is 40G - 10% guard band bandwidth.
53 * struct tb_cm - Simple Thunderbolt connection manager
74 return ((void *)tcm - sizeof(struct tb)); in tcm_to_tb()
101 ev->tb = tb; in tb_queue_hotplug()
102 ev->route = route; in tb_queue_hotplug()
103 ev->port = port; in tb_queue_hotplug()
104 ev->unplug = unplug; in tb_queue_hotplug()
105 INIT_DELAYED_WORK(&ev->work, tb_handle_hotplug); in tb_queue_hotplug()
106 queue_delayed_work(tb->wq, &ev->work, 0); in tb_queue_hotplug()
113 struct tb_cm *tcm = tb_priv(sw->tb); in tb_add_dp_resources()
124 * If DP IN on device router exist, position it at the in tb_add_dp_resources()
126 * before DP IN of the host router. This way external GPU(s) in tb_add_dp_resources()
130 list_add(&port->list, &tcm->dp_resources); in tb_add_dp_resources()
132 list_add_tail(&port->list, &tcm->dp_resources); in tb_add_dp_resources()
140 struct tb_cm *tcm = tb_priv(sw->tb); in tb_remove_dp_resources()
146 tb_remove_dp_resources(port->remote->sw); in tb_remove_dp_resources()
149 list_for_each_entry_safe(port, tmp, &tcm->dp_resources, list) { in tb_remove_dp_resources()
150 if (port->sw == sw) { in tb_remove_dp_resources()
152 list_del_init(&port->list); in tb_remove_dp_resources()
162 list_for_each_entry(p, &tcm->dp_resources, list) { in tb_discover_dp_resource()
169 list_add_tail(&port->list, &tcm->dp_resources); in tb_discover_dp_resource()
177 list_for_each_entry(tunnel, &tcm->tunnel_list, list) { in tb_discover_dp_resources()
179 tb_discover_dp_resource(tb, tunnel->dst_port); in tb_discover_dp_resources()
183 /* Enables CL states up to host router */
186 struct tb_cm *tcm = tb_priv(sw->tb); in tb_enable_clx()
208 * If we are re-enabling then check if there is an active DMA in tb_enable_clx()
211 list_for_each_entry(tunnel, &tcm->tunnel_list, list) { in tb_enable_clx()
223 if (ret == -EOPNOTSUPP) in tb_enable_clx()
225 return ret == -EOPNOTSUPP ? 0 : ret; in tb_enable_clx()
229 * tb_disable_clx() - Disable CL states up to host router
230 * @sw: Router to start
232 * Disables CL states from @sw up to the host router. Returns true if
292 * accuracy of first depth child routers (and the host router) in tb_increase_tmu_accuracy()
299 sw = tunnel->tb->root_switch; in tb_increase_tmu_accuracy()
300 device_for_each_child(&sw->dev, NULL, tb_increase_switch_tmu_accuracy); in tb_increase_tmu_accuracy()
317 return device_for_each_child(&tb->dev, NULL, in tb_tmu_hifi_uni_required()
327 * enable the enhanched uni-directional mode. That covers all in tb_enable_tmu()
334 if (ret == -EOPNOTSUPP) { in tb_enable_tmu()
337 * Figure out uni-directional HiFi TMU requirements in tb_enable_tmu()
339 * uni-directional HiFi requirements we can put the TMU in tb_enable_tmu()
342 * Deliberately skip bi-directional HiFi links in tb_enable_tmu()
346 if (tb_tmu_hifi_uni_required(sw->tb)) in tb_enable_tmu()
356 /* If not supported, fallback to bi-directional HiFi */ in tb_enable_tmu()
357 if (ret == -EOPNOTSUPP) in tb_enable_tmu()
382 struct tb *tb = sw->tb; in tb_switch_discover_tunnels()
388 switch (port->config.type) { in tb_switch_discover_tunnels()
407 list_add_tail(&tunnel->list, list); in tb_switch_discover_tunnels()
412 tb_switch_discover_tunnels(port->remote->sw, list, in tb_switch_discover_tunnels()
420 if (tb_switch_is_usb4(port->sw)) in tb_port_configure_xdomain()
427 if (tb_switch_is_usb4(port->sw)) in tb_port_unconfigure_xdomain()
435 struct tb_switch *sw = port->sw; in tb_scan_xdomain()
436 struct tb *tb = sw->tb; in tb_scan_xdomain()
450 xd = tb_xdomain_alloc(tb, &sw->dev, route, tb->root_switch->uuid, in tb_scan_xdomain()
453 tb_port_at(route, sw)->xdomain = xd; in tb_scan_xdomain()
460 * tb_find_unused_port() - return the first inactive port on @sw
472 if (port->config.type != type) in tb_find_unused_port()
474 if (!port->cap_adap) in tb_find_unused_port()
501 list_for_each_entry(tunnel, &tcm->tunnel_list, list) { in tb_find_tunnel()
502 if (tunnel->type == type && in tb_find_tunnel()
503 ((src_port && src_port == tunnel->src_port) || in tb_find_tunnel()
504 (dst_port && dst_port == tunnel->dst_port))) { in tb_find_tunnel()
519 /* Pick the router that is deepest in the topology */ in tb_find_first_usb3_tunnel()
521 sw = dst_port->sw; in tb_find_first_usb3_tunnel()
523 sw = src_port->sw; in tb_find_first_usb3_tunnel()
525 /* Can't be the host router */ in tb_find_first_usb3_tunnel()
526 if (sw == tb->root_switch) in tb_find_first_usb3_tunnel()
529 /* Find the downstream USB4 port that leads to this router */ in tb_find_first_usb3_tunnel()
530 port = tb_port_at(tb_route(sw), tb->root_switch); in tb_find_first_usb3_tunnel()
531 /* Find the corresponding host router USB3 downstream port */ in tb_find_first_usb3_tunnel()
532 usb3_down = usb4_switch_map_usb3_down(tb->root_switch, port); in tb_find_first_usb3_tunnel()
540 * tb_consumed_usb3_pcie_bandwidth() - Consumed USB3/PCIe bandwidth over a single link
589 * tb_consumed_dp_bandwidth() - Consumed DP bandwidth over a single link
624 list_for_each_entry(tunnel, &tcm->tunnel_list, list) { in tb_consumed_dp_bandwidth()
642 group = tunnel->src_port->group; in tb_consumed_dp_bandwidth()
643 if (group && group->reserved && !group_reserved[group->index]) in tb_consumed_dp_bandwidth()
644 group_reserved[group->index] = group->reserved; in tb_consumed_dp_bandwidth()
649 * re-calculating estimated bandwidth. in tb_consumed_dp_bandwidth()
651 if (tunnel->src_port == src_port && in tb_consumed_dp_bandwidth()
652 tunnel->dst_port == dst_port) in tb_consumed_dp_bandwidth()
690 * tb_maximum_bandwidth() - Maximum bandwidth over a single link
720 link_speed = port->sw->link_speed; in tb_maximum_bandwidth()
722 * sw->link_width is from upstream perspective so we use in tb_maximum_bandwidth()
723 * the opposite for downstream of the host router. in tb_maximum_bandwidth()
725 if (port->sw->link_width == TB_LINK_WIDTH_ASYM_TX) { in tb_maximum_bandwidth()
728 } else if (port->sw->link_width == TB_LINK_WIDTH_ASYM_RX) { in tb_maximum_bandwidth()
746 up_bw = link_speed * port->sw->link_width * 1000; in tb_maximum_bandwidth()
785 *max_up = up_bw - up_bw / 10; in tb_maximum_bandwidth()
786 *max_down = down_bw - down_bw / 10; in tb_maximum_bandwidth()
793 * tb_available_bandwidth() - Available bandwidth for tunneling
838 max_up -= consumed_up; in tb_available_bandwidth()
839 max_down -= consumed_down; in tb_available_bandwidth()
845 max_up -= consumed_up; in tb_available_bandwidth()
846 max_down -= consumed_down; in tb_available_bandwidth()
888 ret = tb_available_bandwidth(tb, tunnel->src_port, tunnel->dst_port, in tb_reclaim_usb3_bandwidth()
918 if (!sw->link_usb4) in tb_tunnel_usb3()
958 ret = -ENOMEM; in tb_tunnel_usb3()
965 ret = -EIO; in tb_tunnel_usb3()
969 list_add_tail(&tunnel->list, &tcm->tunnel_list); in tb_tunnel_usb3()
993 ret = tb_tunnel_usb3(sw->tb, sw); in tb_create_usb3_tunnels()
1001 ret = tb_create_usb3_tunnels(port->remote->sw); in tb_create_usb3_tunnels()
1010 * tb_configure_asym() - Transition links to asymmetric if needed
1037 /* Pick up router deepest in the hierarchy */ in tb_configure_asym()
1039 sw = dst_port->sw; in tb_configure_asym()
1041 sw = src_port->sw; in tb_configure_asym()
1044 struct tb_port *down = tb_switch_downstream_port(up->sw); in tb_configure_asym()
1056 * (40G - guard band 10%), and the requested is above in tb_configure_asym()
1060 ret = -ENOBUFS; in tb_configure_asym()
1072 ret = -ENOBUFS; in tb_configure_asym()
1082 if (up->sw->link_width == width_up) in tb_configure_asym()
1099 tb_sw_dbg(up->sw, "configuring asymmetric link\n"); in tb_configure_asym()
1105 ret = tb_switch_set_link_width(up->sw, width_up); in tb_configure_asym()
1107 tb_sw_warn(up->sw, "failed to set link width\n"); in tb_configure_asym()
1112 /* Re-enable CL states if they were previosly enabled */ in tb_configure_asym()
1120 * tb_configure_sym() - Transition links to symmetric if possible
1142 /* Pick up router deepest in the hierarchy */ in tb_configure_sym()
1144 sw = dst_port->sw; in tb_configure_sym()
1146 sw = src_port->sw; in tb_configure_sym()
1152 if (up->sw->link_width <= TB_LINK_WIDTH_DUAL) in tb_configure_sym()
1155 if (up->sw->is_unplugged) in tb_configure_sym()
1177 if (up->sw->link_width == TB_LINK_WIDTH_DUAL) in tb_configure_sym()
1184 * However, if the router prefers asymmetric link we in tb_configure_sym()
1188 up->sw->preferred_link_width > TB_LINK_WIDTH_DUAL) { in tb_configure_sym()
1189 tb_sw_dbg(up->sw, "keeping preferred asymmetric link\n"); in tb_configure_sym()
1199 tb_sw_dbg(up->sw, "configuring symmetric link\n"); in tb_configure_sym()
1201 ret = tb_switch_set_link_width(up->sw, TB_LINK_WIDTH_DUAL); in tb_configure_sym()
1203 tb_sw_warn(up->sw, "failed to set link width\n"); in tb_configure_sym()
1208 /* Re-enable CL states if they were previosly enabled */ in tb_configure_sym()
1218 struct tb *tb = sw->tb; in tb_configure_link()
1221 down->remote = up; in tb_configure_link()
1222 up->remote = down; in tb_configure_link()
1223 if (down->dual_link_port && up->dual_link_port) { in tb_configure_link()
1224 down->dual_link_port->remote = up->dual_link_port; in tb_configure_link()
1225 up->dual_link_port->remote = down->dual_link_port; in tb_configure_link()
1232 if (sw->link_width < TB_LINK_WIDTH_DUAL) in tb_configure_link()
1236 * Device router that comes up as symmetric link is in tb_configure_link()
1242 up->sw->link_width == TB_LINK_WIDTH_DUAL) { in tb_configure_link()
1245 host_port = tb_port_at(tb_route(sw), tb->root_switch); in tb_configure_link()
1254 * tb_scan_switch() - scan for and initialize downstream switches
1260 pm_runtime_get_sync(&sw->dev); in tb_scan_switch()
1265 pm_runtime_mark_last_busy(&sw->dev); in tb_scan_switch()
1266 pm_runtime_put_autosuspend(&sw->dev); in tb_scan_switch()
1270 * tb_scan_port() - check for and initialize switches below port
1274 struct tb_cm *tcm = tb_priv(port->sw->tb); in tb_scan_port()
1285 tb_queue_hotplug(port->sw->tb, tb_route(port->sw), port->port, in tb_scan_port()
1290 if (port->config.type != TB_TYPE_PORT) in tb_scan_port()
1292 if (port->dual_link_port && port->link_nr) in tb_scan_port()
1298 if (port->usb4) in tb_scan_port()
1299 pm_runtime_get_sync(&port->usb4->dev); in tb_scan_port()
1303 if (port->remote) { in tb_scan_port()
1308 sw = tb_switch_alloc(port->sw->tb, &port->sw->dev, in tb_scan_port()
1313 * is no router connected. in tb_scan_port()
1322 if (PTR_ERR(sw) == -EIO || PTR_ERR(sw) == -EADDRNOTAVAIL) in tb_scan_port()
1336 if (port->xdomain) { in tb_scan_port()
1337 tb_xdomain_remove(port->xdomain); in tb_scan_port()
1339 port->xdomain = NULL; in tb_scan_port()
1347 if (!tcm->hotplug_active) { in tb_scan_port()
1348 dev_set_uevent_suppress(&sw->dev, true); in tb_scan_port()
1356 sw->rpm = sw->generation > 1; in tb_scan_port()
1368 * router has been enumerated to avoid issues with certain in tb_scan_port()
1388 * enabled for the upstream port of the router so we do it here. in tb_scan_port()
1401 if (tcm->hotplug_active && tb_tunnel_usb3(sw->tb, sw)) in tb_scan_port()
1408 if (port->usb4) { in tb_scan_port()
1409 pm_runtime_mark_last_busy(&port->usb4->dev); in tb_scan_port()
1410 pm_runtime_put_autosuspend(&port->usb4->dev); in tb_scan_port()
1418 struct tb *tb = group->tb; in tb_recalc_estimated_bandwidth_for_group()
1422 tb_dbg(tb, "re-calculating bandwidth estimation for group %u\n", in tb_recalc_estimated_bandwidth_for_group()
1423 group->index); in tb_recalc_estimated_bandwidth_for_group()
1426 list_for_each_entry(in, &group->ports, group_list) { in tb_recalc_estimated_bandwidth_for_group()
1441 * tunnels under the host router USB4 port, even in tb_recalc_estimated_bandwidth_for_group()
1442 * if they do not begin from the host router, we in tb_recalc_estimated_bandwidth_for_group()
1448 first_tunnel->src_port, first_tunnel->dst_port); in tb_recalc_estimated_bandwidth_for_group()
1456 out = tunnel->dst_port; in tb_recalc_estimated_bandwidth_for_group()
1461 "failed to re-calculate estimated bandwidth\n"); in tb_recalc_estimated_bandwidth_for_group()
1467 * - already allocated bandwidth for the DP tunnel in tb_recalc_estimated_bandwidth_for_group()
1468 * - available bandwidth along the path in tb_recalc_estimated_bandwidth_for_group()
1469 * - bandwidth allocated for USB 3.x but not used. in tb_recalc_estimated_bandwidth_for_group()
1481 "re-calculated estimated bandwidth %u (+ %u reserved) = %u Mb/s\n", in tb_recalc_estimated_bandwidth_for_group()
1482 estimated_bw, group->reserved, in tb_recalc_estimated_bandwidth_for_group()
1483 estimated_bw + group->reserved); in tb_recalc_estimated_bandwidth_for_group()
1486 estimated_bw + group->reserved)) in tb_recalc_estimated_bandwidth_for_group()
1492 tb_reclaim_usb3_bandwidth(tb, first_tunnel->src_port, in tb_recalc_estimated_bandwidth_for_group()
1493 first_tunnel->dst_port); in tb_recalc_estimated_bandwidth_for_group()
1495 tb_dbg(tb, "bandwidth estimation for group %u done\n", group->index); in tb_recalc_estimated_bandwidth_for_group()
1503 tb_dbg(tb, "bandwidth consumption changed, re-calculating estimated bandwidth\n"); in tb_recalc_estimated_bandwidth()
1505 for (i = 0; i < ARRAY_SIZE(tcm->groups); i++) { in tb_recalc_estimated_bandwidth()
1506 struct tb_bandwidth_group *group = &tcm->groups[i]; in tb_recalc_estimated_bandwidth()
1508 if (!list_empty(&group->ports)) in tb_recalc_estimated_bandwidth()
1512 tb_dbg(tb, "bandwidth re-calculation done\n"); in tb_recalc_estimated_bandwidth()
1517 if (group->reserved) { in __release_group_bandwidth()
1518 tb_dbg(group->tb, "group %d released total %d Mb/s\n", group->index, in __release_group_bandwidth()
1519 group->reserved); in __release_group_bandwidth()
1520 group->reserved = 0; in __release_group_bandwidth()
1531 if (list_empty(&group->ports)) in __configure_group_sym()
1544 in = list_first_entry(&group->ports, struct tb_port, group_list); in __configure_group_sym()
1545 tunnel = tb_find_tunnel(group->tb, TB_TUNNEL_DP, in, NULL); in __configure_group_sym()
1547 tb_configure_sym(group->tb, in, tunnel->dst_port, true); in __configure_group_sym()
1554 struct tb *tb = group->tb; in tb_bandwidth_group_release_work()
1556 mutex_lock(&tb->lock); in tb_bandwidth_group_release_work()
1560 mutex_unlock(&tb->lock); in tb_bandwidth_group_release_work()
1567 for (i = 0; i < ARRAY_SIZE(tcm->groups); i++) { in tb_init_bandwidth_groups()
1568 struct tb_bandwidth_group *group = &tcm->groups[i]; in tb_init_bandwidth_groups()
1570 group->tb = tcm_to_tb(tcm); in tb_init_bandwidth_groups()
1571 group->index = i + 1; in tb_init_bandwidth_groups()
1572 INIT_LIST_HEAD(&group->ports); in tb_init_bandwidth_groups()
1573 INIT_DELAYED_WORK(&group->release_work, in tb_init_bandwidth_groups()
1581 if (!group || WARN_ON(in->group)) in tb_bandwidth_group_attach_port()
1584 in->group = group; in tb_bandwidth_group_attach_port()
1585 list_add_tail(&in->group_list, &group->ports); in tb_bandwidth_group_attach_port()
1587 tb_port_dbg(in, "attached to bandwidth group %d\n", group->index); in tb_bandwidth_group_attach_port()
1594 for (i = 0; i < ARRAY_SIZE(tcm->groups); i++) { in tb_find_free_bandwidth_group()
1595 struct tb_bandwidth_group *group = &tcm->groups[i]; in tb_find_free_bandwidth_group()
1597 if (list_empty(&group->ports)) in tb_find_free_bandwidth_group()
1617 list_for_each_entry(tunnel, &tcm->tunnel_list, list) { in tb_attach_bandwidth_group()
1621 if (tunnel->src_port->sw == in->sw && in tb_attach_bandwidth_group()
1622 tunnel->dst_port->sw == out->sw) { in tb_attach_bandwidth_group()
1623 group = tunnel->src_port->group; in tb_attach_bandwidth_group()
1648 for (i = 0; i < ARRAY_SIZE(tcm->groups); i++) { in tb_discover_bandwidth_group()
1649 if (tcm->groups[i].index == index) { in tb_discover_bandwidth_group()
1650 tb_bandwidth_group_attach_port(&tcm->groups[i], in); in tb_discover_bandwidth_group()
1661 struct tb_bandwidth_group *group = in->group; in tb_detach_bandwidth_group()
1664 in->group = NULL; in tb_detach_bandwidth_group()
1665 list_del_init(&in->group_list); in tb_detach_bandwidth_group()
1667 tb_port_dbg(in, "detached from bandwidth group %d\n", group->index); in tb_detach_bandwidth_group()
1670 if (list_empty(&group->ports)) { in tb_detach_bandwidth_group()
1671 cancel_delayed_work(&group->release_work); in tb_detach_bandwidth_group()
1682 tb_switch_discover_tunnels(tb->root_switch, &tcm->tunnel_list, true); in tb_discover_tunnels()
1684 list_for_each_entry(tunnel, &tcm->tunnel_list, list) { in tb_discover_tunnels()
1686 struct tb_switch *parent = tunnel->dst_port->sw; in tb_discover_tunnels()
1688 while (parent != tunnel->src_port->sw) { in tb_discover_tunnels()
1689 parent->boot = true; in tb_discover_tunnels()
1693 struct tb_port *in = tunnel->src_port; in tb_discover_tunnels()
1694 struct tb_port *out = tunnel->dst_port; in tb_discover_tunnels()
1697 pm_runtime_get_sync(&in->sw->dev); in tb_discover_tunnels()
1698 pm_runtime_get_sync(&out->sw->dev); in tb_discover_tunnels()
1714 list_del(&tunnel->list); in tb_deactivate_and_free_tunnel()
1716 tb = tunnel->tb; in tb_deactivate_and_free_tunnel()
1717 src_port = tunnel->src_port; in tb_deactivate_and_free_tunnel()
1718 dst_port = tunnel->dst_port; in tb_deactivate_and_free_tunnel()
1720 switch (tunnel->type) { in tb_deactivate_and_free_tunnel()
1727 tb_switch_dealloc_dp_resource(src_port->sw, src_port); in tb_deactivate_and_free_tunnel()
1734 pm_runtime_mark_last_busy(&dst_port->sw->dev); in tb_deactivate_and_free_tunnel()
1735 pm_runtime_put_autosuspend(&dst_port->sw->dev); in tb_deactivate_and_free_tunnel()
1736 pm_runtime_mark_last_busy(&src_port->sw->dev); in tb_deactivate_and_free_tunnel()
1737 pm_runtime_put_autosuspend(&src_port->sw->dev); in tb_deactivate_and_free_tunnel()
1746 * PCIe and DMA tunnels do not consume guaranteed in tb_deactivate_and_free_tunnel()
1756 * tb_free_invalid_tunnels() - destroy tunnels of devices that have gone away
1764 list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) { in tb_free_invalid_tunnels()
1771 * tb_free_unplugged_children() - traverse hierarchy and free unplugged switches
1781 if (port->remote->sw->is_unplugged) { in tb_free_unplugged_children()
1783 tb_remove_dp_resources(port->remote->sw); in tb_free_unplugged_children()
1784 tb_switch_unconfigure_link(port->remote->sw); in tb_free_unplugged_children()
1785 tb_switch_set_link_width(port->remote->sw, in tb_free_unplugged_children()
1787 tb_switch_remove(port->remote->sw); in tb_free_unplugged_children()
1788 port->remote = NULL; in tb_free_unplugged_children()
1789 if (port->dual_link_port) in tb_free_unplugged_children()
1790 port->dual_link_port->remote = NULL; in tb_free_unplugged_children()
1792 tb_free_unplugged_children(port->remote->sw); in tb_free_unplugged_children()
1809 int phy_port = tb_phy_port_from_link(port->port); in tb_find_pcie_down()
1813 * Hard-coded Thunderbolt port to PCIe down port mapping in tb_find_pcie_down()
1826 /* Validate the hard-coding */ in tb_find_pcie_down()
1827 if (WARN_ON(index > sw->config.max_port_number)) in tb_find_pcie_down()
1830 down = &sw->ports[index]; in tb_find_pcie_down()
1851 host_port = tb_route(in->sw) ? in tb_find_dp_out()
1852 tb_port_at(tb_route(in->sw), tb->root_switch) : NULL; in tb_find_dp_out()
1854 list_for_each_entry(port, &tcm->dp_resources, list) { in tb_find_dp_out()
1864 if (in->sw == port->sw) { in tb_find_dp_out()
1865 tb_port_dbg(port, "skipping DP OUT on same router\n"); in tb_find_dp_out()
1873 * the same host router downstream port. in tb_find_dp_out()
1875 if (host_port && tb_route(port->sw)) { in tb_find_dp_out()
1878 p = tb_port_at(tb_route(port->sw), tb->root_switch); in tb_find_dp_out()
1891 struct tb_port *in = tunnel->src_port; in tb_dp_tunnel_active()
1892 struct tb_port *out = tunnel->dst_port; in tb_dp_tunnel_active()
1895 mutex_lock(&tb->lock); in tb_dp_tunnel_active()
1923 * router's 1st children TMU mode to HiFi for in tb_dp_tunnel_active()
1929 struct tb_port *in = tunnel->src_port; in tb_dp_tunnel_active()
1936 * discrete router. in tb_dp_tunnel_active()
1940 * also tear down the tunnel and try to re-use the in tb_dp_tunnel_active()
1949 mutex_unlock(&tb->lock); in tb_dp_tunnel_active()
1968 list_for_each_entry(tunnel, &tcm->tunnel_list, list) { in tb_tunnel_one_dp()
1983 pm_runtime_get_sync(&in->sw->dev); in tb_tunnel_one_dp()
1984 pm_runtime_get_sync(&out->sw->dev); in tb_tunnel_one_dp()
1986 if (tb_switch_alloc_dp_resource(in->sw, in)) { in tb_tunnel_one_dp()
2017 list_add_tail(&tunnel->list, &tcm->tunnel_list); in tb_tunnel_one_dp()
2020 if (ret && ret != -EINPROGRESS) { in tb_tunnel_one_dp()
2022 list_del(&tunnel->list); in tb_tunnel_one_dp()
2036 tb_switch_dealloc_dp_resource(in->sw, in); in tb_tunnel_one_dp()
2038 pm_runtime_mark_last_busy(&out->sw->dev); in tb_tunnel_one_dp()
2039 pm_runtime_put_autosuspend(&out->sw->dev); in tb_tunnel_one_dp()
2040 pm_runtime_mark_last_busy(&in->sw->dev); in tb_tunnel_one_dp()
2041 pm_runtime_put_autosuspend(&in->sw->dev); in tb_tunnel_one_dp()
2058 tb_dbg(tb, "looking for DP IN <-> DP OUT pairs:\n"); in tb_tunnel_dp()
2062 list_for_each_entry(port, &tcm->dp_resources, list) { in tb_tunnel_dp()
2087 struct tb_switch *sw = port->sw; in tb_enter_redrive()
2089 if (!(sw->quirks & QUIRK_KEEP_POWER_IN_DP_REDRIVE)) in tb_enter_redrive()
2093 * If we get hot-unplug for the DP IN port of the host router in tb_enter_redrive()
2095 * is a monitor connected directly to the Type-C port and we are in tb_enter_redrive()
2104 port->redrive = true; in tb_enter_redrive()
2105 pm_runtime_get(&sw->dev); in tb_enter_redrive()
2112 struct tb_switch *sw = port->sw; in tb_exit_redrive()
2114 if (!(sw->quirks & QUIRK_KEEP_POWER_IN_DP_REDRIVE)) in tb_exit_redrive()
2121 if (port->redrive && tb_switch_query_dp_resource(sw, port)) { in tb_exit_redrive()
2122 port->redrive = false; in tb_exit_redrive()
2123 pm_runtime_put(&sw->dev); in tb_exit_redrive()
2144 if (!(sw->quirks & QUIRK_KEEP_POWER_IN_DP_REDRIVE)) in tb_switch_exit_redrive()
2151 if (port->redrive) { in tb_switch_exit_redrive()
2152 port->redrive = false; in tb_switch_exit_redrive()
2153 pm_runtime_put(&sw->dev); in tb_switch_exit_redrive()
2180 list_del_init(&port->list); in tb_dp_resource_unavailable()
2198 list_for_each_entry(p, &tcm->dp_resources, list) { in tb_dp_resource_available()
2205 list_add_tail(&port->list, &tcm->dp_resources); in tb_dp_resource_available()
2208 /* Look for suitable DP IN <-> DP OUT pairs now */ in tb_dp_resource_available()
2219 * will be re-established after resume based on plug events. in tb_disconnect_and_release_dp()
2221 list_for_each_entry_safe_reverse(tunnel, n, &tcm->tunnel_list, list) { in tb_disconnect_and_release_dp()
2226 while (!list_empty(&tcm->dp_resources)) { in tb_disconnect_and_release_dp()
2229 port = list_first_entry(&tcm->dp_resources, in tb_disconnect_and_release_dp()
2231 list_del_init(&port->list); in tb_disconnect_and_release_dp()
2242 return -ENODEV; in tb_disconnect_pci()
2246 return -ENODEV; in tb_disconnect_pci()
2251 list_del(&tunnel->list); in tb_disconnect_pci()
2277 return -ENOMEM; in tb_tunnel_pci()
2283 return -EIO; in tb_tunnel_pci()
2296 list_add_tail(&tunnel->list, &tcm->tunnel_list); in tb_tunnel_pci()
2310 sw = tb_to_switch(xd->dev.parent); in tb_approve_xdomain_paths()
2311 dst_port = tb_port_at(xd->route, sw); in tb_approve_xdomain_paths()
2312 nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI); in tb_approve_xdomain_paths()
2314 mutex_lock(&tb->lock); in tb_approve_xdomain_paths()
2317 * When tunneling DMA paths the link should not enter CL states in tb_approve_xdomain_paths()
2325 ret = -ENOMEM; in tb_approve_xdomain_paths()
2331 "DMA tunnel activation failed, aborting\n"); in tb_approve_xdomain_paths()
2332 ret = -EIO; in tb_approve_xdomain_paths()
2336 list_add_tail(&tunnel->list, &tcm->tunnel_list); in tb_approve_xdomain_paths()
2337 mutex_unlock(&tb->lock); in tb_approve_xdomain_paths()
2344 mutex_unlock(&tb->lock); in tb_approve_xdomain_paths()
2358 sw = tb_to_switch(xd->dev.parent); in __tb_disconnect_xdomain_paths()
2359 dst_port = tb_port_at(xd->route, sw); in __tb_disconnect_xdomain_paths()
2360 nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI); in __tb_disconnect_xdomain_paths()
2362 list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) { in __tb_disconnect_xdomain_paths()
2365 if (tunnel->src_port != nhi_port || tunnel->dst_port != dst_port) in __tb_disconnect_xdomain_paths()
2374 * Try to re-enable CL states now, it is OK if this fails in __tb_disconnect_xdomain_paths()
2375 * because we may still have another DMA tunnel active through in __tb_disconnect_xdomain_paths()
2376 * the same host router USB4 downstream port. in __tb_disconnect_xdomain_paths()
2385 if (!xd->is_unplugged) { in tb_disconnect_xdomain_paths()
2386 mutex_lock(&tb->lock); in tb_disconnect_xdomain_paths()
2390 mutex_unlock(&tb->lock); in tb_disconnect_xdomain_paths()
2398 * tb_handle_hotplug() - handle hotplug event
2400 * Executes on tb->wq.
2405 struct tb *tb = ev->tb; in tb_handle_hotplug()
2411 pm_runtime_get_sync(&tb->dev); in tb_handle_hotplug()
2413 mutex_lock(&tb->lock); in tb_handle_hotplug()
2414 if (!tcm->hotplug_active) in tb_handle_hotplug()
2417 sw = tb_switch_find_by_route(tb, ev->route); in tb_handle_hotplug()
2421 ev->route, ev->port, ev->unplug); in tb_handle_hotplug()
2424 if (ev->port > sw->config.max_port_number) { in tb_handle_hotplug()
2427 ev->route, ev->port, ev->unplug); in tb_handle_hotplug()
2430 port = &sw->ports[ev->port]; in tb_handle_hotplug()
2433 ev->route, ev->port, ev->unplug); in tb_handle_hotplug()
2437 pm_runtime_get_sync(&sw->dev); in tb_handle_hotplug()
2439 if (ev->unplug) { in tb_handle_hotplug()
2444 tb_sw_set_unplugged(port->remote->sw); in tb_handle_hotplug()
2446 tb_remove_dp_resources(port->remote->sw); in tb_handle_hotplug()
2447 tb_switch_tmu_disable(port->remote->sw); in tb_handle_hotplug()
2448 tb_switch_unconfigure_link(port->remote->sw); in tb_handle_hotplug()
2449 tb_switch_set_link_width(port->remote->sw, in tb_handle_hotplug()
2451 tb_switch_remove(port->remote->sw); in tb_handle_hotplug()
2452 port->remote = NULL; in tb_handle_hotplug()
2453 if (port->dual_link_port) in tb_handle_hotplug()
2454 port->dual_link_port->remote = NULL; in tb_handle_hotplug()
2458 } else if (port->xdomain) { in tb_handle_hotplug()
2459 struct tb_xdomain *xd = tb_xdomain_get(port->xdomain); in tb_handle_hotplug()
2469 xd->is_unplugged = true; in tb_handle_hotplug()
2471 port->xdomain = NULL; in tb_handle_hotplug()
2472 __tb_disconnect_xdomain_paths(tb, xd, -1, -1, -1, -1); in tb_handle_hotplug()
2477 } else if (!port->port) { in tb_handle_hotplug()
2484 } else if (port->remote) { in tb_handle_hotplug()
2486 } else if (!port->port && sw->authorized) { in tb_handle_hotplug()
2493 if (!port->remote) in tb_handle_hotplug()
2500 pm_runtime_mark_last_busy(&sw->dev); in tb_handle_hotplug()
2501 pm_runtime_put_autosuspend(&sw->dev); in tb_handle_hotplug()
2506 mutex_unlock(&tb->lock); in tb_handle_hotplug()
2508 pm_runtime_mark_last_busy(&tb->dev); in tb_handle_hotplug()
2509 pm_runtime_put_autosuspend(&tb->dev); in tb_handle_hotplug()
2521 struct tb *tb = tunnel->tb; in tb_alloc_dp_bandwidth()
2529 in = tunnel->src_port; in tb_alloc_dp_bandwidth()
2530 out = tunnel->dst_port; in tb_alloc_dp_bandwidth()
2583 ret = -ENOBUFS; in tb_alloc_dp_bandwidth()
2588 group = in->group; in tb_alloc_dp_bandwidth()
2592 if (tunnel->bw_mode) { in tb_alloc_dp_bandwidth()
2603 reserved = allocated_down - *requested_down; in tb_alloc_dp_bandwidth()
2605 reserved = allocated_up - *requested_up; in tb_alloc_dp_bandwidth()
2608 group->reserved += reserved; in tb_alloc_dp_bandwidth()
2610 group->index, reserved, group->reserved); in tb_alloc_dp_bandwidth()
2620 mod_delayed_work(system_wq, &group->release_work, in tb_alloc_dp_bandwidth()
2648 available_up, available_down, group->reserved); in tb_alloc_dp_bandwidth()
2651 available_up + group->reserved >= requested_up_corrected) || in tb_alloc_dp_bandwidth()
2653 available_down + group->reserved >= requested_down_corrected)) { in tb_alloc_dp_bandwidth()
2676 released = *requested_down - available_down; in tb_alloc_dp_bandwidth()
2679 released = *requested_up - available_up; in tb_alloc_dp_bandwidth()
2682 group->reserved -= released; in tb_alloc_dp_bandwidth()
2684 group->index, released, group->reserved); in tb_alloc_dp_bandwidth()
2687 ret = -ENOBUFS; in tb_alloc_dp_bandwidth()
2693 if (ret && ret != -ENODEV) { in tb_alloc_dp_bandwidth()
2712 struct tb *tb = ev->tb; in tb_handle_dp_bandwidth_request()
2717 pm_runtime_get_sync(&tb->dev); in tb_handle_dp_bandwidth_request()
2719 mutex_lock(&tb->lock); in tb_handle_dp_bandwidth_request()
2720 if (!tcm->hotplug_active) in tb_handle_dp_bandwidth_request()
2723 sw = tb_switch_find_by_route(tb, ev->route); in tb_handle_dp_bandwidth_request()
2725 tb_warn(tb, "bandwidth request from non-existent router %llx\n", in tb_handle_dp_bandwidth_request()
2726 ev->route); in tb_handle_dp_bandwidth_request()
2730 in = &sw->ports[ev->port]; in tb_handle_dp_bandwidth_request()
2732 tb_port_warn(in, "bandwidth request to non-DP IN adapter\n"); in tb_handle_dp_bandwidth_request()
2736 tb_port_dbg(in, "handling bandwidth allocation request, retry %d\n", ev->retry); in tb_handle_dp_bandwidth_request()
2745 if (tunnel->bw_mode) { in tb_handle_dp_bandwidth_request()
2750 tunnel->bw_mode = false; in tb_handle_dp_bandwidth_request()
2760 if (ret == -ENODATA) { in tb_handle_dp_bandwidth_request()
2780 requested_up = -1; in tb_handle_dp_bandwidth_request()
2784 requested_down = -1; in tb_handle_dp_bandwidth_request()
2789 if (ret == -ENOBUFS) { in tb_handle_dp_bandwidth_request()
2792 } else if (ret == -ENOTCONN) { in tb_handle_dp_bandwidth_request()
2801 if (ev->retry < TB_BW_ALLOC_RETRIES) { in tb_handle_dp_bandwidth_request()
2804 tb_queue_dp_bandwidth_request(tb, ev->route, in tb_handle_dp_bandwidth_request()
2805 ev->port, in tb_handle_dp_bandwidth_request()
2806 ev->retry + 1, in tb_handle_dp_bandwidth_request()
2828 mutex_unlock(&tb->lock); in tb_handle_dp_bandwidth_request()
2830 pm_runtime_mark_last_busy(&tb->dev); in tb_handle_dp_bandwidth_request()
2831 pm_runtime_put_autosuspend(&tb->dev); in tb_handle_dp_bandwidth_request()
2845 ev->tb = tb; in tb_queue_dp_bandwidth_request()
2846 ev->route = route; in tb_queue_dp_bandwidth_request()
2847 ev->port = port; in tb_queue_dp_bandwidth_request()
2848 ev->retry = retry; in tb_queue_dp_bandwidth_request()
2849 INIT_DELAYED_WORK(&ev->work, tb_handle_dp_bandwidth_request); in tb_queue_dp_bandwidth_request()
2850 queue_delayed_work(tb->wq, &ev->work, delay); in tb_queue_dp_bandwidth_request()
2857 switch (error->error) { in tb_handle_notification()
2861 if (tb_cfg_ack_notification(tb->ctl, route, error)) in tb_handle_notification()
2867 if (tb_cfg_ack_notification(tb->ctl, route, error)) in tb_handle_notification()
2870 tb_queue_dp_bandwidth_request(tb, route, error->port, 0, 0); in tb_handle_notification()
2880 * tb_schedule_hotplug_handler() - callback function for the control channel
2888 u64 route = tb_cfg_get_route(&pkg->header); in tb_handle_event()
2901 if (tb_cfg_ack_plug(tb->ctl, route, pkg->port, pkg->unplug)) { in tb_handle_event()
2903 pkg->port); in tb_handle_event()
2906 tb_queue_hotplug(tb, route, pkg->port, pkg->unplug); in tb_handle_event()
2915 cancel_delayed_work(&tcm->remove_work); in tb_stop()
2917 list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) { in tb_stop()
2919 * DMA tunnels require the driver to be functional so we in tb_stop()
2927 tb_switch_remove(tb->root_switch); in tb_stop()
2928 tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */ in tb_stop()
2937 for (i = 0; i < ARRAY_SIZE(tcm->groups); i++) in tb_deinit()
2938 cancel_delayed_work_sync(&tcm->groups[i].release_work); in tb_deinit()
2951 if (sw->boot) in tb_scan_finalize_switch()
2952 sw->authorized = 1; in tb_scan_finalize_switch()
2955 kobject_uevent(&dev->kobj, KOBJ_ADD); in tb_scan_finalize_switch()
2968 tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0); in tb_start()
2969 if (IS_ERR(tb->root_switch)) in tb_start()
2970 return PTR_ERR(tb->root_switch); in tb_start()
2978 * implement the necessary router operations. in tb_start()
2980 tb->root_switch->no_nvm_upgrade = !tb_switch_is_usb4(tb->root_switch); in tb_start()
2982 tb->root_switch->rpm = tb_switch_is_usb4(tb->root_switch); in tb_start()
2984 ret = tb_switch_configure(tb->root_switch); in tb_start()
2986 tb_switch_put(tb->root_switch); in tb_start()
2991 ret = tb_switch_add(tb->root_switch); in tb_start()
2993 tb_switch_put(tb->root_switch); in tb_start()
2998 * To support highest CLx state, we set host router's TMU to in tb_start()
3001 tb_switch_tmu_configure(tb->root_switch, TB_SWITCH_TMU_MODE_LOWRES); in tb_start()
3003 tb_switch_tmu_enable(tb->root_switch); in tb_start()
3011 if (reset && tb_switch_is_usb4(tb->root_switch)) { in tb_start()
3013 if (usb4_switch_version(tb->root_switch) == 1) in tb_start()
3014 tb_switch_reset(tb->root_switch); in tb_start()
3019 tb_scan_switch(tb->root_switch); in tb_start()
3030 tb_create_usb3_tunnels(tb->root_switch); in tb_start()
3032 tb_add_dp_resources(tb->root_switch); in tb_start()
3033 tb_switch_enter_redrive(tb->root_switch); in tb_start()
3035 device_for_each_child(&tb->root_switch->dev, NULL, in tb_start()
3039 tcm->hotplug_active = true; in tb_start()
3049 tb_switch_exit_redrive(tb->root_switch); in tb_suspend_noirq()
3050 tb_switch_suspend(tb->root_switch, false); in tb_suspend_noirq()
3051 tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */ in tb_suspend_noirq()
3061 /* No need to restore if the router is already unplugged */ in tb_restore_children()
3062 if (sw->is_unplugged) in tb_restore_children()
3066 tb_sw_warn(sw, "failed to re-enable CL states\n"); in tb_restore_children()
3074 if (!tb_port_has_remote(port) && !port->xdomain) in tb_restore_children()
3077 if (port->remote) { in tb_restore_children()
3078 tb_switch_set_link_width(port->remote->sw, in tb_restore_children()
3079 port->remote->sw->link_width); in tb_restore_children()
3080 tb_switch_configure_link(port->remote->sw); in tb_restore_children()
3082 tb_restore_children(port->remote->sw); in tb_restore_children()
3083 } else if (port->xdomain) { in tb_restore_children()
3084 tb_port_configure_xdomain(port, port->xdomain); in tb_restore_children()
3099 * For non-USB4 hosts (Apple systems) remove any PCIe devices in tb_resume_noirq()
3102 if (!tb_switch_is_usb4(tb->root_switch)) in tb_resume_noirq()
3103 tb_switch_reset(tb->root_switch); in tb_resume_noirq()
3105 tb_switch_resume(tb->root_switch, false); in tb_resume_noirq()
3107 tb_free_unplugged_children(tb->root_switch); in tb_resume_noirq()
3108 tb_restore_children(tb->root_switch); in tb_resume_noirq()
3116 tb_switch_discover_tunnels(tb->root_switch, &tunnels, false); in tb_resume_noirq()
3124 /* Re-create our tunnels now */ in tb_resume_noirq()
3125 list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) { in tb_resume_noirq()
3126 /* USB3 requires delay before it can be re-activated */ in tb_resume_noirq()
3134 if (!list_empty(&tcm->tunnel_list)) { in tb_resume_noirq()
3142 tb_switch_enter_redrive(tb->root_switch); in tb_resume_noirq()
3144 tcm->hotplug_active = true; in tb_resume_noirq()
3158 if (port->xdomain && port->xdomain->is_unplugged) { in tb_free_unplugged_xdomains()
3160 tb_xdomain_remove(port->xdomain); in tb_free_unplugged_xdomains()
3162 port->xdomain = NULL; in tb_free_unplugged_xdomains()
3164 } else if (port->remote) { in tb_free_unplugged_xdomains()
3165 ret += tb_free_unplugged_xdomains(port->remote->sw); in tb_free_unplugged_xdomains()
3176 tcm->hotplug_active = false; in tb_freeze_noirq()
3184 tcm->hotplug_active = true; in tb_thaw_noirq()
3195 mutex_lock(&tb->lock); in tb_complete()
3196 if (tb_free_unplugged_xdomains(tb->root_switch)) in tb_complete()
3197 tb_scan_switch(tb->root_switch); in tb_complete()
3198 mutex_unlock(&tb->lock); in tb_complete()
3205 mutex_lock(&tb->lock); in tb_runtime_suspend()
3208 * re-entering redrive mode. in tb_runtime_suspend()
3211 tb_switch_exit_redrive(tb->root_switch); in tb_runtime_suspend()
3212 tb_switch_suspend(tb->root_switch, true); in tb_runtime_suspend()
3213 tcm->hotplug_active = false; in tb_runtime_suspend()
3214 mutex_unlock(&tb->lock); in tb_runtime_suspend()
3224 mutex_lock(&tb->lock); in tb_remove_work()
3225 if (tb->root_switch) { in tb_remove_work()
3226 tb_free_unplugged_children(tb->root_switch); in tb_remove_work()
3227 tb_free_unplugged_xdomains(tb->root_switch); in tb_remove_work()
3229 mutex_unlock(&tb->lock); in tb_remove_work()
3237 mutex_lock(&tb->lock); in tb_runtime_resume()
3238 tb_switch_resume(tb->root_switch, true); in tb_runtime_resume()
3240 tb_restore_children(tb->root_switch); in tb_runtime_resume()
3241 list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) in tb_runtime_resume()
3243 tb_switch_enter_redrive(tb->root_switch); in tb_runtime_resume()
3244 tcm->hotplug_active = true; in tb_runtime_resume()
3245 mutex_unlock(&tb->lock); in tb_runtime_resume()
3252 queue_delayed_work(tb->wq, &tcm->remove_work, msecs_to_jiffies(50)); in tb_runtime_resume()
3289 switch (nhi->pdev->device) { in tb_apple_add_links()
3299 upstream = pci_upstream_bridge(nhi->pdev); in tb_apple_add_links()
3313 * back to NHI so that PCIe tunnels can be re-established after in tb_apple_add_links()
3317 for_each_pci_bridge(pdev, upstream->subordinate) { in tb_apple_add_links()
3323 !pdev->is_hotplug_bridge) in tb_apple_add_links()
3326 link = device_link_add(&pdev->dev, &nhi->pdev->dev, in tb_apple_add_links()
3330 dev_dbg(&nhi->pdev->dev, "created link from %s\n", in tb_apple_add_links()
3331 dev_name(&pdev->dev)); in tb_apple_add_links()
3334 dev_warn(&nhi->pdev->dev, "device link creation from %s failed\n", in tb_apple_add_links()
3335 dev_name(&pdev->dev)); in tb_apple_add_links()
3352 tb->security_level = TB_SECURITY_USER; in tb_probe()
3354 tb->security_level = TB_SECURITY_NOPCIE; in tb_probe()
3356 tb->cm_ops = &tb_cm_ops; in tb_probe()
3359 INIT_LIST_HEAD(&tcm->tunnel_list); in tb_probe()
3360 INIT_LIST_HEAD(&tcm->dp_resources); in tb_probe()
3361 INIT_DELAYED_WORK(&tcm->remove_work, tb_remove_work); in tb_probe()