124 #define cd_to_ni(cd) container_of((cd), struct arm_ni, cds[(cd)->id])  argument
127 #define cd_for_each_unit(cd, u) \  argument
128 	for (struct arm_ni_unit *u = cd->units; u < cd->units + cd->num_units; u++)
158 	struct arm_ni_cd *cd = pmu_to_cd(dev_get_drvdata(dev));  in arm_ni_event_attr_is_visible()  local
163 	cd_for_each_unit(cd, unit) {  in arm_ni_event_attr_is_visible()
221 	struct arm_ni_cd *cd = pmu_to_cd(dev_get_drvdata(dev));  in arm_ni_cpumask_show()  local
223 	return cpumap_print_to_pagebuf(true, buf, cpumask_of(cd->cpu));  in arm_ni_cpumask_show()
309 	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);  in arm_ni_event_init()  local
317 	event->cpu = cd->cpu;  in arm_ni_event_init()
321 	cd_for_each_unit(cd, unit) {  in arm_ni_event_init()
331 static u64 arm_ni_read_ccnt(struct arm_ni_cd *cd)  in arm_ni_read_ccnt()  argument
336 	u_new = readl_relaxed(cd->pmu_base + NI_PMCCNTR_U);  in arm_ni_read_ccnt()
339 		l = readl_relaxed(cd->pmu_base + NI_PMCCNTR_L);  in arm_ni_read_ccnt()
340 		u_new = readl_relaxed(cd->pmu_base + NI_PMCCNTR_U);  in arm_ni_read_ccnt()
349 	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);  in arm_ni_event_read()  local
357 			count = arm_ni_read_ccnt(cd);  in arm_ni_event_read()
359 			count = readl_relaxed(cd->pmu_base + NI_PMEVCNTR(hw->idx));  in arm_ni_event_read()
370 	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);  in arm_ni_event_start()  local
372 	writel_relaxed(1U << event->hw.idx, cd->pmu_base + NI_PMCNTENSET);  in arm_ni_event_start()
377 	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);  in arm_ni_event_stop()  local
379 	writel_relaxed(1U << event->hw.idx, cd->pmu_base + NI_PMCNTENCLR);  in arm_ni_event_stop()
384 static void arm_ni_init_ccnt(struct arm_ni_cd *cd)  in arm_ni_init_ccnt()  argument
386 	local64_set(&cd->ccnt->hw.prev_count, S64_MIN);  in arm_ni_init_ccnt()
387 	lo_hi_writeq_relaxed(S64_MIN, cd->pmu_base + NI_PMCCNTR_L);  in arm_ni_init_ccnt()
390 static void arm_ni_init_evcnt(struct arm_ni_cd *cd, int idx)  in arm_ni_init_evcnt()  argument
392 	local64_set(&cd->evcnt[idx]->hw.prev_count, S32_MIN);  in arm_ni_init_evcnt()
393 	writel_relaxed(S32_MIN, cd->pmu_base + NI_PMEVCNTR(idx));  in arm_ni_init_evcnt()
398 	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);  in arm_ni_event_add()  local
405 		if (cd->ccnt)  in arm_ni_event_add()
408 		cd->ccnt = event;  in arm_ni_event_add()
409 		arm_ni_init_ccnt(cd);  in arm_ni_event_add()
412 		while (cd->evcnt[hw->idx]) {  in arm_ni_event_add()
416 		cd->evcnt[hw->idx] = event;  in arm_ni_event_add()
419 		arm_ni_init_evcnt(cd, hw->idx);  in arm_ni_event_add()
424 		writel_relaxed(reg, cd->pmu_base + NI_PMEVTYPER(hw->idx));  in arm_ni_event_add()
433 	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);  in arm_ni_event_del()  local
439 		cd->ccnt = NULL;  in arm_ni_event_del()
441 		cd->evcnt[hw->idx] = NULL;  in arm_ni_event_del()
446 	struct arm_ni_cd *cd = dev_id;  in arm_ni_handle_irq()  local
448 	u32 reg = readl_relaxed(cd->pmu_base + NI_PMOVSCLR);  in arm_ni_handle_irq()
452 		if (!(WARN_ON(!cd->ccnt))) {  in arm_ni_handle_irq()
453 			arm_ni_event_read(cd->ccnt);  in arm_ni_handle_irq()
454 			arm_ni_init_ccnt(cd);  in arm_ni_handle_irq()
461 		if (!(WARN_ON(!cd->evcnt[i]))) {  in arm_ni_handle_irq()
462 			arm_ni_event_read(cd->evcnt[i]);  in arm_ni_handle_irq()
463 			arm_ni_init_evcnt(cd, i);  in arm_ni_handle_irq()
466 	writel_relaxed(reg, cd->pmu_base + NI_PMOVSCLR);  in arm_ni_handle_irq()
472 	struct arm_ni_cd *cd = ni->cds + node->id;  in arm_ni_init_cd()  local
476 	cd->id = node->id;  in arm_ni_init_cd()
477 	cd->num_units = node->num_components;  in arm_ni_init_cd()
478 	cd->units = devm_kcalloc(ni->dev, cd->num_units, sizeof(*(cd->units)), GFP_KERNEL);  in arm_ni_init_cd()
479 	if (!cd->units)  in arm_ni_init_cd()
482 	for (int i = 0; i < cd->num_units; i++) {  in arm_ni_init_cd()
485 		struct arm_ni_unit *unit = cd->units + i;  in arm_ni_init_cd()
495 				dev_info(ni->dev, "No access to PMU %d\n", cd->id);  in arm_ni_init_cd()
496 				devm_kfree(ni->dev, cd->units);  in arm_ni_init_cd()
500 			cd->pmu_base = unit_base;  in arm_ni_init_cd()
524 	res_start += cd->pmu_base - ni->base;  in arm_ni_init_cd()
531 		       cd->pmu_base + NI_PMCR);  in arm_ni_init_cd()
532 	writel_relaxed(U32_MAX, cd->pmu_base + NI_PMCNTENCLR);  in arm_ni_init_cd()
533 	writel_relaxed(U32_MAX, cd->pmu_base + NI_PMOVSCLR);  in arm_ni_init_cd()
534 	writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENSET);  in arm_ni_init_cd()
536 	cd->irq = platform_get_irq(to_platform_device(ni->dev), cd->id);  in arm_ni_init_cd()
537 	if (cd->irq < 0)  in arm_ni_init_cd()
538 		return cd->irq;  in arm_ni_init_cd()
540 	err = devm_request_irq(ni->dev, cd->irq, arm_ni_handle_irq,  in arm_ni_init_cd()
542 			       dev_name(ni->dev), cd);  in arm_ni_init_cd()
546 	cd->cpu = cpumask_local_spread(0, dev_to_node(ni->dev));  in arm_ni_init_cd()
547 	cd->pmu = (struct pmu) {  in arm_ni_init_cd()
563 	name = devm_kasprintf(ni->dev, GFP_KERNEL, "arm_ni_%d_cd_%d", ni->id, cd->id);  in arm_ni_init_cd()
567 	err = cpuhp_state_add_instance_nocalls(arm_ni_hp_state, &cd->cpuhp_node);  in arm_ni_init_cd()
571 	err = perf_pmu_register(&cd->pmu, name, -1);  in arm_ni_init_cd()
573 		cpuhp_state_remove_instance_nocalls(arm_ni_hp_state, &cd->cpuhp_node);  in arm_ni_init_cd()
590 	struct arm_ni_node cfg, vd, pd, cd;  in arm_ni_probe()  local
657 				arm_ni_probe_domain(base + reg, &cd);  in arm_ni_probe()
658 				ret = arm_ni_init_cd(ni, &cd, res->start);  in arm_ni_probe()
673 		struct arm_ni_cd *cd = ni->cds + i;  in arm_ni_remove()  local
675 		if (!cd->pmu_base)  in arm_ni_remove()
678 		writel_relaxed(0, cd->pmu_base + NI_PMCR);  in arm_ni_remove()
679 		writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENCLR);  in arm_ni_remove()
680 		perf_pmu_unregister(&cd->pmu);  in arm_ni_remove()
681 		cpuhp_state_remove_instance_nocalls(arm_ni_hp_state, &cd->cpuhp_node);  in arm_ni_remove()
711 static void arm_ni_pmu_migrate(struct arm_ni_cd *cd, unsigned int cpu)  in arm_ni_pmu_migrate()  argument
713 	perf_pmu_migrate_context(&cd->pmu, cd->cpu, cpu);  in arm_ni_pmu_migrate()
714 	irq_set_affinity(cd->irq, cpumask_of(cpu));  in arm_ni_pmu_migrate()
715 	cd->cpu = cpu;  in arm_ni_pmu_migrate()
720 	struct arm_ni_cd *cd;  in arm_ni_pmu_online_cpu()  local
723 	cd = hlist_entry_safe(cpuhp_node, struct arm_ni_cd, cpuhp_node);  in arm_ni_pmu_online_cpu()
724 	node = dev_to_node(cd_to_ni(cd)->dev);  in arm_ni_pmu_online_cpu()
725 	if (cpu_to_node(cd->cpu) != node && cpu_to_node(cpu) == node)  in arm_ni_pmu_online_cpu()
726 		arm_ni_pmu_migrate(cd, cpu);  in arm_ni_pmu_online_cpu()
732 	struct arm_ni_cd *cd;  in arm_ni_pmu_offline_cpu()  local
736 	cd = hlist_entry_safe(cpuhp_node, struct arm_ni_cd, cpuhp_node);  in arm_ni_pmu_offline_cpu()
737 	if (cpu != cd->cpu)  in arm_ni_pmu_offline_cpu()
740 	node = dev_to_node(cd_to_ni(cd)->dev);  in arm_ni_pmu_offline_cpu()
746 		arm_ni_pmu_migrate(cd, target);  in arm_ni_pmu_offline_cpu()