3  * Enable PCIe link L0s/L1 state and Clock Power Management
49 	struct pci_dev *pdev;		/* Upstream component of the Link */
51 	struct pcie_link_state *root;	/* pointer to the root port link */
52 	struct pcie_link_state *parent;	/* pointer to the parent Link state */
110 static int policy_to_aspm_state(struct pcie_link_state *link)  in policy_to_aspm_state()  argument
123 		return link->aspm_default;  in policy_to_aspm_state()
128 static int policy_to_clkpm_state(struct pcie_link_state *link)  in policy_to_clkpm_state()  argument
139 		return link->clkpm_default;  in policy_to_clkpm_state()
144 static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)  in pcie_set_clkpm_nocheck()  argument
147 	struct pci_bus *linkbus = link->pdev->subordinate;  in pcie_set_clkpm_nocheck()
154 	link->clkpm_enabled = !!enable;  in pcie_set_clkpm_nocheck()
157 static void pcie_set_clkpm(struct pcie_link_state *link, int enable)  in pcie_set_clkpm()  argument
160 	 * Don't enable Clock PM if the link is not Clock PM capable  in pcie_set_clkpm()
163 	if (!link->clkpm_capable || link->clkpm_disable)  in pcie_set_clkpm()
166 	if (link->clkpm_enabled == enable)  in pcie_set_clkpm()
168 	pcie_set_clkpm_nocheck(link, enable);  in pcie_set_clkpm()
171 static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)  in pcie_clkpm_cap_init()  argument
177 	struct pci_bus *linkbus = link->pdev->subordinate;  in pcie_clkpm_cap_init()
191 	link->clkpm_enabled = enabled;  in pcie_clkpm_cap_init()
192 	link->clkpm_default = enabled;  in pcie_clkpm_cap_init()
193 	link->clkpm_capable = capable;  in pcie_clkpm_cap_init()
194 	link->clkpm_disable = blacklist ? 1 : 0;  in pcie_clkpm_cap_init()
198  * pcie_aspm_configure_common_clock: check if the 2 ends of a link
202 static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)  in pcie_aspm_configure_common_clock()  argument
206 	struct pci_dev *child, *parent = link->pdev;  in pcie_aspm_configure_common_clock()
257 	if (pcie_retrain_link(link->pdev, true)) {  in pcie_aspm_configure_common_clock()
367 	struct pcie_link_state *link;  in pcie_aspm_check_latency()  local
374 	link = endpoint->bus->self->link_state;  in pcie_aspm_check_latency()
384 	while (link) {  in pcie_aspm_check_latency()
385 		struct pci_dev *dev = pci_function_0(link->pdev->subordinate);  in pcie_aspm_check_latency()
388 		pcie_capability_read_dword(link->pdev, PCI_EXP_LNKCAP,  in pcie_aspm_check_latency()
398 		if ((link->aspm_capable & ASPM_STATE_L0S_UP) &&  in pcie_aspm_check_latency()
400 			link->aspm_capable &= ~ASPM_STATE_L0S_UP;  in pcie_aspm_check_latency()
403 		if ((link->aspm_capable & ASPM_STATE_L0S_DW) &&  in pcie_aspm_check_latency()
405 			link->aspm_capable &= ~ASPM_STATE_L0S_DW;  in pcie_aspm_check_latency()
420 		if ((link->aspm_capable & ASPM_STATE_L1) &&  in pcie_aspm_check_latency()
422 			link->aspm_capable &= ~ASPM_STATE_L1;  in pcie_aspm_check_latency()
425 		link = link->parent;  in pcie_aspm_check_latency()
430 static void aspm_calc_l12_info(struct pcie_link_state *link,  in aspm_calc_l12_info()  argument
433 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in aspm_calc_l12_info()
464 	 * Link from L0 to L1.2 and back to L0 so we enter L1.2 only if  in aspm_calc_l12_info()
529 static void aspm_l1ss_init(struct pcie_link_state *link)  in aspm_l1ss_init()  argument
531 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in aspm_l1ss_init()
558 		link->aspm_support |= ASPM_STATE_L1_1;  in aspm_l1ss_init()
560 		link->aspm_support |= ASPM_STATE_L1_2;  in aspm_l1ss_init()
562 		link->aspm_support |= ASPM_STATE_L1_1_PCIPM;  in aspm_l1ss_init()
564 		link->aspm_support |= ASPM_STATE_L1_2_PCIPM;  in aspm_l1ss_init()
574 		link->aspm_enabled |= ASPM_STATE_L1_1;  in aspm_l1ss_init()
576 		link->aspm_enabled |= ASPM_STATE_L1_2;  in aspm_l1ss_init()
578 		link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM;  in aspm_l1ss_init()
580 		link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM;  in aspm_l1ss_init()
582 	if (link->aspm_support & ASPM_STATE_L1_2_MASK)  in aspm_l1ss_init()
583 		aspm_calc_l12_info(link, parent_l1ss_cap, child_l1ss_cap);  in aspm_l1ss_init()
586 static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)  in pcie_aspm_cap_init()  argument
588 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in pcie_aspm_cap_init()
595 		link->aspm_enabled = ASPM_STATE_ALL;  in pcie_aspm_cap_init()
596 		link->aspm_disable = ASPM_STATE_ALL;  in pcie_aspm_cap_init()
601 	 * If ASPM not supported, don't mess with the clocks and link,  in pcie_aspm_cap_init()
610 	pcie_aspm_configure_common_clock(link);  in pcie_aspm_cap_init()
615 	 * read-only Link Capabilities may change depending on common clock  in pcie_aspm_cap_init()
627 	 * given link unless components on both sides of the link each  in pcie_aspm_cap_init()
631 		link->aspm_support |= ASPM_STATE_L0S;  in pcie_aspm_cap_init()
634 		link->aspm_enabled |= ASPM_STATE_L0S_UP;  in pcie_aspm_cap_init()
636 		link->aspm_enabled |= ASPM_STATE_L0S_DW;  in pcie_aspm_cap_init()
640 		link->aspm_support |= ASPM_STATE_L1;  in pcie_aspm_cap_init()
643 		link->aspm_enabled |= ASPM_STATE_L1;  in pcie_aspm_cap_init()
645 	aspm_l1ss_init(link);  in pcie_aspm_cap_init()
648 	link->aspm_default = link->aspm_enabled;  in pcie_aspm_cap_init()
651 	link->aspm_capable = link->aspm_support;  in pcie_aspm_cap_init()
664 static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state)  in pcie_config_aspm_l1ss()  argument
667 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in pcie_config_aspm_l1ss()
669 	enable_req = (link->aspm_enabled ^ state) & state;  in pcie_config_aspm_l1ss()
723 static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state)  in pcie_config_aspm_link()  argument
726 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in pcie_config_aspm_link()
730 	state &= (link->aspm_capable & ~link->aspm_disable);  in pcie_config_aspm_link()
739 		state |= (link->aspm_enabled & ASPM_STATE_L1_SS_PCIPM);  in pcie_config_aspm_link()
742 	/* Nothing to do if the link is already in the requested state */  in pcie_config_aspm_link()
743 	if (link->aspm_enabled == state)  in pcie_config_aspm_link()
755 	if (link->aspm_capable & ASPM_STATE_L1SS)  in pcie_config_aspm_link()
756 		pcie_config_aspm_l1ss(link, state);  in pcie_config_aspm_link()
771 	link->aspm_enabled = state;  in pcie_config_aspm_link()
774 static void pcie_config_aspm_path(struct pcie_link_state *link)  in pcie_config_aspm_path()  argument
776 	while (link) {  in pcie_config_aspm_path()
777 		pcie_config_aspm_link(link, policy_to_aspm_state(link));  in pcie_config_aspm_path()
778 		link = link->parent;  in pcie_config_aspm_path()
782 static void free_link_state(struct pcie_link_state *link)  in free_link_state()  argument
784 	link->pdev->link_state = NULL;  in free_link_state()
785 	kfree(link);  in free_link_state()
825 	struct pcie_link_state *link;  in alloc_pcie_link_state()  local
827 	link = kzalloc(sizeof(*link), GFP_KERNEL);  in alloc_pcie_link_state()
828 	if (!link)  in alloc_pcie_link_state()
831 	INIT_LIST_HEAD(&link->sibling);  in alloc_pcie_link_state()
832 	link->pdev = pdev;  in alloc_pcie_link_state()
833 	link->downstream = pci_function_0(pdev->subordinate);  in alloc_pcie_link_state()
839 	 * a switch may become the root of the link state chain for all  in alloc_pcie_link_state()
845 		link->root = link;  in alloc_pcie_link_state()
851 			kfree(link);  in alloc_pcie_link_state()
855 		link->parent = parent;  in alloc_pcie_link_state()
856 		link->root = link->parent->root;  in alloc_pcie_link_state()
859 	list_add(&link->sibling, &link_list);  in alloc_pcie_link_state()
860 	pdev->link_state = link;  in alloc_pcie_link_state()
861 	return link;  in alloc_pcie_link_state()
873  * pcie_aspm_init_link_state: Initiate PCI express link state.
879 	struct pcie_link_state *link;  in pcie_aspm_init_link_state()  local
890 	 * end of a Link, so there's nothing to do unless this device is  in pcie_aspm_init_link_state()
906 	link = alloc_pcie_link_state(pdev);  in pcie_aspm_init_link_state()
907 	if (!link)  in pcie_aspm_init_link_state()
914 	pcie_aspm_cap_init(link, blacklist);  in pcie_aspm_init_link_state()
917 	pcie_clkpm_cap_init(link, blacklist);  in pcie_aspm_init_link_state()
921 	 * link policy setting. Enabling ASPM on broken hardware can cripple  in pcie_aspm_init_link_state()
929 		pcie_config_aspm_path(link);  in pcie_aspm_init_link_state()
930 		pcie_set_clkpm(link, policy_to_clkpm_state(link));  in pcie_aspm_init_link_state()
944 	struct pcie_link_state *link;  in pcie_update_aspm_capable()  local
946 	list_for_each_entry(link, &link_list, sibling) {  in pcie_update_aspm_capable()
947 		if (link->root != root)  in pcie_update_aspm_capable()
949 		link->aspm_capable = link->aspm_support;  in pcie_update_aspm_capable()
951 	list_for_each_entry(link, &link_list, sibling) {  in pcie_update_aspm_capable()
953 		struct pci_bus *linkbus = link->pdev->subordinate;  in pcie_update_aspm_capable()
954 		if (link->root != root)  in pcie_update_aspm_capable()
969 	struct pcie_link_state *link, *root, *parent_link;  in pcie_aspm_exit_link_state()  local
977 	link = parent->link_state;  in pcie_aspm_exit_link_state()
978 	root = link->root;  in pcie_aspm_exit_link_state()
979 	parent_link = link->parent;  in pcie_aspm_exit_link_state()
982 	 * link->downstream is a pointer to the pci_dev of function 0.  If  in pcie_aspm_exit_link_state()
984 	 * so we can't use link->downstream again.  Free the link state to  in pcie_aspm_exit_link_state()
988 	 * retain the link state, but PCIe r6.0, sec 7.5.3.7, recommends  in pcie_aspm_exit_link_state()
992 	pcie_config_aspm_link(link, 0);  in pcie_aspm_exit_link_state()
993 	list_del(&link->sibling);  in pcie_aspm_exit_link_state()
994 	free_link_state(link);  in pcie_aspm_exit_link_state()
1012 	struct pcie_link_state *link = pdev->link_state;  in pcie_aspm_pm_state_change()  local
1014 	if (aspm_disabled || !link)  in pcie_aspm_pm_state_change()
1023 	pcie_update_aspm_capable(link->root);  in pcie_aspm_pm_state_change()
1024 	pcie_config_aspm_path(link);  in pcie_aspm_pm_state_change()
1032 	struct pcie_link_state *link = pdev->link_state;  in pcie_aspm_powersave_config_link()  local
1034 	if (aspm_disabled || !link)  in pcie_aspm_powersave_config_link()
1043 	pcie_config_aspm_path(link);  in pcie_aspm_powersave_config_link()
1044 	pcie_set_clkpm(link, policy_to_clkpm_state(link));  in pcie_aspm_powersave_config_link()
1065 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in __pci_disable_link_state()  local
1067 	if (!link)  in __pci_disable_link_state()
1086 		link->aspm_disable |= ASPM_STATE_L0S;  in __pci_disable_link_state()
1089 		link->aspm_disable |= ASPM_STATE_L1 | ASPM_STATE_L1SS;  in __pci_disable_link_state()
1091 		link->aspm_disable |= ASPM_STATE_L1_1;  in __pci_disable_link_state()
1093 		link->aspm_disable |= ASPM_STATE_L1_2;  in __pci_disable_link_state()
1095 		link->aspm_disable |= ASPM_STATE_L1_1_PCIPM;  in __pci_disable_link_state()
1097 		link->aspm_disable |= ASPM_STATE_L1_2_PCIPM;  in __pci_disable_link_state()
1098 	pcie_config_aspm_link(link, policy_to_aspm_state(link));  in __pci_disable_link_state()
1101 		link->clkpm_disable = 1;  in __pci_disable_link_state()
1102 	pcie_set_clkpm(link, policy_to_clkpm_state(link));  in __pci_disable_link_state()
1119  * pci_disable_link_state - Disable device's link state, so the link will
1125  * @state: ASPM link state to disable
1135 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in __pci_enable_link_state()  local
1137 	if (!link)  in __pci_enable_link_state()
1153 	link->aspm_default = 0;  in __pci_enable_link_state()
1155 		link->aspm_default |= ASPM_STATE_L0S;  in __pci_enable_link_state()
1157 		link->aspm_default |= ASPM_STATE_L1;  in __pci_enable_link_state()
1160 		link->aspm_default |= ASPM_STATE_L1_1 | ASPM_STATE_L1;  in __pci_enable_link_state()
1162 		link->aspm_default |= ASPM_STATE_L1_2 | ASPM_STATE_L1;  in __pci_enable_link_state()
1164 		link->aspm_default |= ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1;  in __pci_enable_link_state()
1166 		link->aspm_default |= ASPM_STATE_L1_2_PCIPM | ASPM_STATE_L1;  in __pci_enable_link_state()
1167 	pcie_config_aspm_link(link, policy_to_aspm_state(link));  in __pci_enable_link_state()
1169 	link->clkpm_default = (state & PCIE_LINK_STATE_CLKPM) ? 1 : 0;  in __pci_enable_link_state()
1170 	pcie_set_clkpm(link, policy_to_clkpm_state(link));  in __pci_enable_link_state()
1179  * pci_enable_link_state - Clear and set the default device link state so that
1180  * the link may be allowed to enter the specified states. Note that if the
1186  * @state: Mask of ASPM link states to enable
1195  * pci_enable_link_state_locked - Clear and set the default device link state
1196  * so that the link may be allowed to enter the specified states. Note that if
1202  * @state: Mask of ASPM link states to enable
1218 	struct pcie_link_state *link;  in pcie_aspm_set_policy()  local
1231 	list_for_each_entry(link, &link_list, sibling) {  in pcie_aspm_set_policy()
1232 		pcie_config_aspm_link(link, policy_to_aspm_state(link));  in pcie_aspm_set_policy()
1233 		pcie_set_clkpm(link, policy_to_clkpm_state(link));  in pcie_aspm_set_policy()
1266 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in pcie_aspm_enabled()  local
1268 	if (!link)  in pcie_aspm_enabled()
1271 	return link->aspm_enabled;  in pcie_aspm_enabled()
1280 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in aspm_attr_show_common()  local
1282 	return sysfs_emit(buf, "%d\n", (link->aspm_enabled & state) ? 1 : 0);  in aspm_attr_show_common()
1290 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in aspm_attr_store_common()  local
1300 		link->aspm_disable &= ~state;  in aspm_attr_store_common()
1303 			link->aspm_disable &= ~ASPM_STATE_L1;  in aspm_attr_store_common()
1305 		link->aspm_disable |= state;  in aspm_attr_store_common()
1307 			link->aspm_disable |= ASPM_STATE_L1SS;  in aspm_attr_store_common()
1310 	pcie_config_aspm_link(link, policy_to_aspm_state(link));  in aspm_attr_store_common()
1339 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in ASPM_ATTR()  local
1341 	return sysfs_emit(buf, "%d\n", link->clkpm_enabled);  in ASPM_ATTR()
1349 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in clkpm_store()  local
1358 	link->clkpm_disable = !state_enable;  in clkpm_store()
1359 	pcie_set_clkpm(link, policy_to_clkpm_state(link));  in clkpm_store()
1391 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in aspm_ctrl_attrs_are_visible()  local
1401 	if (aspm_disabled || !link)  in aspm_ctrl_attrs_are_visible()
1405 		return link->clkpm_capable ? a->mode : 0;  in aspm_ctrl_attrs_are_visible()
1407 	return link->aspm_capable & aspm_state_map[n - 1] ? a->mode : 0;  in aspm_ctrl_attrs_are_visible()
1411 	.name = "link",