131 	if (rd->rp->priv->type == RAPL_IF_TPMI) {  in get_pl_lock_prim()
148 	if (rd->rp->priv->limits[rd->id] & BIT(POWER_LIMIT2))  in get_pl_lock_prim()
159 		if (prim == PL_CLAMP && rd->rp->priv->type != RAPL_IF_TPMI)  in get_pl_prim()
173 		if (prim == PL_CLAMP && rd->rp->priv->type != RAPL_IF_TPMI)  in get_pl_prim()
216 static struct rapl_defaults *get_defaults(struct rapl_package *rp)  in get_defaults()  argument
218 	return rp->priv->defaults;  in get_defaults()
249 static void rapl_init_domains(struct rapl_package *rp);
264 static void package_power_limit_irq_save(struct rapl_package *rp);
310 	struct rapl_package *rp = rd->rp;  in release_zone()  local
317 		rp->domains = NULL;  in release_zone()
339 	struct rapl_defaults *defaults = get_defaults(rd->rp);  in set_domain_enable()
438 	struct rapl_package *rp;  in set_power_limit()  local
445 	rp = rd->rp;  in set_power_limit()
449 		package_power_limit_irq_save(rp);  in set_power_limit()
561 static int get_rid(struct rapl_package *rp)  in get_rid()  argument
563 	return rp->lead_cpu >= 0 ? rp->lead_cpu : rp->id;  in get_rid()
567 static void rapl_init_domains(struct rapl_package *rp)  in rapl_init_domains()  argument
571 	struct rapl_domain *rd = rp->domains;  in rapl_init_domains()
574 		unsigned int mask = rp->domain_map & (1 << i);  in rapl_init_domains()
580 		rd->rp = rp;  in rapl_init_domains()
582 		if (i == RAPL_DOMAIN_PLATFORM && rp->id > 0) {  in rapl_init_domains()
584 				rp->lead_cpu >= 0 ? topology_physical_package_id(rp->lead_cpu) :  in rapl_init_domains()
585 				rp->id);  in rapl_init_domains()
594 		rp->priv->limits[i] |= BIT(POWER_LIMIT1);  in rapl_init_domains()
597 			if (rp->priv->limits[i] & BIT(t))  in rapl_init_domains()
602 			rd->regs[j] = rp->priv->regs[i][j];  in rapl_init_domains()
612 	struct rapl_defaults *defaults = get_defaults(rd->rp);  in rapl_unit_xlate()
736 static struct rapl_primitive_info *get_rpi(struct rapl_package *rp, int prim)  in get_rpi()  argument
738 	struct rapl_primitive_info *rpi = rp->priv->rpi;  in get_rpi()
746 static int rapl_config(struct rapl_package *rp)  in rapl_config()  argument
748 	switch (rp->priv->type) {  in rapl_config()
752 		rp->priv->defaults = (void *)defaults_msr;  in rapl_config()
753 		rp->priv->rpi = (void *)rpi_msr;  in rapl_config()
756 		rp->priv->defaults = (void *)&defaults_tpmi;  in rapl_config()
757 		rp->priv->rpi = (void *)rpi_tpmi;  in rapl_config()
768 	struct rapl_defaults *defaults = get_defaults(rd->rp);  in prim_fixups()
812 	struct rapl_primitive_info *rpi = get_rpi(rd->rp, prim_fixed);  in rapl_read_data_raw()
830 	if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) {  in rapl_read_data_raw()
831 		pr_debug("failed to read reg 0x%llx for %s:%s\n", ra.reg.val, rd->rp->name, rd->name);  in rapl_read_data_raw()
851 	struct rapl_primitive_info *rpi = get_rpi(rd->rp, prim_fixed);  in rapl_write_data_raw()
869 	ret = rd->rp->priv->write_raw(get_rid(rd->rp), &ra);  in rapl_write_data_raw()
895 		pr_debug("%s:%s:%s locked by BIOS\n", rd->rp->name, rd->name, pl_names[pl]);  in rapl_write_pl_data()
919 	if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) {  in rapl_check_unit_core()
921 			ra.reg.val, rd->rp->name, rd->name);  in rapl_check_unit_core()
935 		 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);  in rapl_check_unit_core()
947 	if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) {  in rapl_check_unit_atom()
949 			ra.reg.val, rd->rp->name, rd->name);  in rapl_check_unit_atom()
963 		 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);  in rapl_check_unit_atom()
971 	struct rapl_package *rp = (struct rapl_package *)info;  in power_limit_irq_save_cpu()  local
975 	if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) {  in power_limit_irq_save_cpu()
976 		rp->power_limit_irq = l & PACKAGE_THERM_INT_PLN_ENABLE;  in power_limit_irq_save_cpu()
977 		rp->power_limit_irq |= PACKAGE_PLN_INT_SAVED;  in power_limit_irq_save_cpu()
993 static void package_power_limit_irq_save(struct rapl_package *rp)  in package_power_limit_irq_save()  argument
995 	if (rp->lead_cpu < 0)  in package_power_limit_irq_save()
1001 	smp_call_function_single(rp->lead_cpu, power_limit_irq_save_cpu, rp, 1);  in package_power_limit_irq_save()
1008 static void package_power_limit_irq_restore(struct rapl_package *rp)  in package_power_limit_irq_restore()  argument
1012 	if (rp->lead_cpu < 0)  in package_power_limit_irq_restore()
1019 	if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED))  in package_power_limit_irq_restore()
1024 	if (rp->power_limit_irq & PACKAGE_THERM_INT_PLN_ENABLE)  in package_power_limit_irq_restore()
1050 	struct rapl_defaults *defaults = get_defaults(rd->rp);  in set_floor_freq_atom()
1134 	if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) {  in rapl_check_unit_tpmi()
1136 			ra.reg.val, rd->rp->name, rd->name);  in rapl_check_unit_tpmi()
1150 		 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit);  in rapl_check_unit_tpmi()
1283 static void rapl_update_domain_data(struct rapl_package *rp)  in rapl_update_domain_data()  argument
1288 	for (dmn = 0; dmn < rp->nr_domains; dmn++) {  in rapl_update_domain_data()
1289 		pr_debug("update %s domain %s data\n", rp->name,  in rapl_update_domain_data()
1290 			 rp->domains[dmn].name);  in rapl_update_domain_data()
1293 			struct rapl_primitive_info *rpi = get_rpi(rp, prim);  in rapl_update_domain_data()
1295 			if (!rapl_read_data_raw(&rp->domains[dmn], prim,  in rapl_update_domain_data()
1297 				rp->domains[dmn].rdd.primitives[prim] = val;  in rapl_update_domain_data()
1303 static int rapl_package_register_powercap(struct rapl_package *rp)  in rapl_package_register_powercap()  argument
1310 	rapl_update_domain_data(rp);  in rapl_package_register_powercap()
1313 	for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {  in rapl_package_register_powercap()
1316 			pr_debug("register package domain %s\n", rp->name);  in rapl_package_register_powercap()
1318 					    rp->priv->control_type, rp->name,  in rapl_package_register_powercap()
1323 					 rp->name);  in rapl_package_register_powercap()
1327 			rp->power_zone = power_zone;  in rapl_package_register_powercap()
1337 	for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {  in rapl_package_register_powercap()
1338 		struct powercap_zone *parent = rp->power_zone;  in rapl_package_register_powercap()
1347 						    rp->priv->control_type,  in rapl_package_register_powercap()
1354 				 rp->name, rd->name);  in rapl_package_register_powercap()
1366 	while (--rd >= rp->domains) {  in rapl_package_register_powercap()
1367 		pr_debug("unregister %s domain %s\n", rp->name, rd->name);  in rapl_package_register_powercap()
1368 		powercap_unregister_zone(rp->priv->control_type,  in rapl_package_register_powercap()
1375 static int rapl_check_domain(int domain, struct rapl_package *rp)  in rapl_check_domain()  argument
1385 		ra.reg = rp->priv->regs[domain][RAPL_DOMAIN_REG_STATUS];  in rapl_check_domain()
1396 	if (rp->priv->read_raw(get_rid(rp), &ra) || !ra.value)  in rapl_check_domain()
1409 	struct rapl_defaults *defaults = get_defaults(rd->rp);  in rapl_get_domain_unit()
1413 		if (!rd->rp->priv->reg_unit.val) {  in rapl_get_domain_unit()
1417 		rd->regs[RAPL_DOMAIN_REG_UNIT] = rd->rp->priv->reg_unit;  in rapl_get_domain_unit()
1455 					rd->rp->name, rd->name, pl_names[i]);  in rapl_detect_powerlimit()
1467 static int rapl_detect_domains(struct rapl_package *rp)  in rapl_detect_domains()  argument
1474 		if (!rapl_check_domain(i, rp)) {  in rapl_detect_domains()
1475 			rp->domain_map |= 1 << i;  in rapl_detect_domains()
1479 	rp->nr_domains = bitmap_weight(&rp->domain_map, RAPL_DOMAIN_MAX);  in rapl_detect_domains()
1480 	if (!rp->nr_domains) {  in rapl_detect_domains()
1481 		pr_debug("no valid rapl domains found in %s\n", rp->name);  in rapl_detect_domains()
1484 	pr_debug("found %d domains on %s\n", rp->nr_domains, rp->name);  in rapl_detect_domains()
1486 	rp->domains = kcalloc(rp->nr_domains, sizeof(struct rapl_domain),  in rapl_detect_domains()
1488 	if (!rp->domains)  in rapl_detect_domains()
1491 	rapl_init_domains(rp);  in rapl_detect_domains()
1493 	for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {  in rapl_detect_domains()
1502 void rapl_remove_package(struct rapl_package *rp)  in rapl_remove_package()  argument
1506 	package_power_limit_irq_restore(rp);  in rapl_remove_package()
1508 	for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {  in rapl_remove_package()
1521 			 rp->name, rd->name);  in rapl_remove_package()
1522 		powercap_unregister_zone(rp->priv->control_type,  in rapl_remove_package()
1526 	powercap_unregister_zone(rp->priv->control_type,  in rapl_remove_package()
1528 	list_del(&rp->plist);  in rapl_remove_package()
1529 	kfree(rp);  in rapl_remove_package()
1536 	struct rapl_package *rp;  in rapl_find_package_domain()  local
1544 	list_for_each_entry(rp, &rapl_packages, plist) {  in rapl_find_package_domain()
1545 		if (rp->id == uid  in rapl_find_package_domain()
1546 		    && rp->priv->control_type == priv->control_type)  in rapl_find_package_domain()
1547 			return rp;  in rapl_find_package_domain()
1557 	struct rapl_package *rp;  in rapl_add_package()  local
1560 	rp = kzalloc(sizeof(struct rapl_package), GFP_KERNEL);  in rapl_add_package()
1561 	if (!rp)  in rapl_add_package()
1565 		rp->id = topology_logical_die_id(id);  in rapl_add_package()
1566 		rp->lead_cpu = id;  in rapl_add_package()
1568 			snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d-die-%d",  in rapl_add_package()
1571 			snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d",  in rapl_add_package()
1574 		rp->id = id;  in rapl_add_package()
1575 		rp->lead_cpu = -1;  in rapl_add_package()
1576 		snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d", id);  in rapl_add_package()
1579 	rp->priv = priv;  in rapl_add_package()
1580 	ret = rapl_config(rp);  in rapl_add_package()
1585 	if (rapl_detect_domains(rp)) {  in rapl_add_package()
1589 	ret = rapl_package_register_powercap(rp);  in rapl_add_package()
1591 		INIT_LIST_HEAD(&rp->plist);  in rapl_add_package()
1592 		list_add(&rp->plist, &rapl_packages);  in rapl_add_package()
1593 		return rp;  in rapl_add_package()
1597 	kfree(rp->domains);  in rapl_add_package()
1598 	kfree(rp);  in rapl_add_package()
1605 	struct rapl_package *rp;  in power_limit_state_save()  local
1610 	list_for_each_entry(rp, &rapl_packages, plist) {  in power_limit_state_save()
1611 		if (!rp->power_zone)  in power_limit_state_save()
1613 		rd = power_zone_to_rapl_domain(rp->power_zone);  in power_limit_state_save()
1626 	struct rapl_package *rp;  in power_limit_state_restore()  local
1631 	list_for_each_entry(rp, &rapl_packages, plist) {  in power_limit_state_restore()
1632 		if (!rp->power_zone)  in power_limit_state_restore()
1634 		rd = power_zone_to_rapl_domain(rp->power_zone);  in power_limit_state_restore()