1 // SPDX-License-Identifier: GPL-2.0
3  * Performance event support for s390x - CPU-measurement Counter Sets
32 struct cf_diag_csd {		/* Counter set data per CPU */
42  * - a two byte eye catcher (0xfeef)
43  * - a one byte counter set number
44  * - a two byte counter set size (indicates the number of counters in this set)
45  * - a three byte reserved value (must be zero) to make the header the same
51  * - flag field indicating valid fields when corresponding bit set
52  * - the counter facility first and second version number
53  * - the CPU speed if nonzero
54  * - the time stamp the counter sets have been collected
55  * - the time of day (TOD) base value
56  * - the machine type.
59  * CPU and saved again when the process is going to be removed from a CPU.
64 struct cf_ctrset_entry {	/* CPU-M CF counter set entry (8 byte) */
65 	unsigned int def:16;	/* 0-15  Data Entry Format */
66 	unsigned int set:16;	/* 16-31 Counter set identifier */
67 	unsigned int ctr:16;	/* 32-47 Number of stored counters */
68 	unsigned int res1:16;	/* 48-63 Reserved */
71 struct cf_trailer_entry {	/* CPU-M CF_DIAG trailer (64 byte) */
72 	/* 0 - 7 */
76 			unsigned int speed:1;		/* CPU speed set */
82 		unsigned long flags;	/* 0-63    All indicators */
84 	/* 8 - 15 */
85 	unsigned int cfvn:16;			/* 64-79   Ctr First Version */
86 	unsigned int csvn:16;			/* 80-95   Ctr Second Version */
87 	unsigned int cpu_speed:32;		/* 96-127  CPU speed */
88 	/* 16 - 23 */
89 	unsigned long timestamp;		/* 128-191 Timestamp (TOD) */
90 	/* 24 - 55 */
100 	/* 56 - 63 */
112 	te->cfvn = cpuhw->info.cfvn;		/* Counter version numbers */  in cf_diag_trailer()
113 	te->csvn = cpuhw->info.csvn;  in cf_diag_trailer()
116 	te->mach_type = cpuid.machine;  in cf_diag_trailer()
117 	te->cpu_speed = cf_diag_cpu_speed;  in cf_diag_trailer()
118 	if (te->cpu_speed)  in cf_diag_trailer()
119 		te->speed = 1;  in cf_diag_trailer()
120 	te->clock_base = 1;			/* Save clock base */  in cf_diag_trailer()
121 	memcpy(&te->tod_base, &tod_clock_base[1], 8);  in cf_diag_trailer()
122 	store_tod_clock((__u64 *)&te->timestamp);  in cf_diag_trailer()
127  * Enable and activate the CPU-counter sets according
128  * to the per-cpu control state.
136 			    "%s pmu %p cpu %d flags %#x state %#llx\n",  in cf_diag_enable()
137 			    __func__, pmu, smp_processor_id(), cpuhw->flags,  in cf_diag_enable()
138 			    cpuhw->state);  in cf_diag_enable()
139 	if (cpuhw->flags & PMU_F_ENABLED)  in cf_diag_enable()
142 	err = lcctl(cpuhw->state);  in cf_diag_enable()
148 	cpuhw->flags |= PMU_F_ENABLED;  in cf_diag_enable()
153  * Disable and enable (inactive) the CPU-counter sets according
154  * to the per-cpu control state.
163 			    "%s pmu %p cpu %d flags %#x state %#llx\n",  in cf_diag_disable()
164 			    __func__, pmu, smp_processor_id(), cpuhw->flags,  in cf_diag_disable()
165 			    cpuhw->state);  in cf_diag_disable()
166 	if (!(cpuhw->flags & PMU_F_ENABLED))  in cf_diag_disable()
169 	inactive = cpuhw->state & ~((1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1);  in cf_diag_disable()
176 	cpuhw->flags &= ~PMU_F_ENABLED;  in cf_diag_disable()
186 			    "%s event %p cpu %d cf_diag_events %d\n",  in cf_diag_perf_event_destroy()
187 			    __func__, event, event->cpu,  in cf_diag_perf_event_destroy()
199 	struct perf_event_attr *attr = &event->attr;  in __hw_perf_event_init()
204 	debug_sprintf_event(cf_diag_dbg, 5, "%s event %p cpu %d\n", __func__,  in __hw_perf_event_init()
205 			    event, event->cpu);  in __hw_perf_event_init()
207 	event->hw.config = attr->config;  in __hw_perf_event_init()
208 	event->hw.config_base = 0;  in __hw_perf_event_init()
211 	 * the hardware init function is either called per-cpu or just once  in __hw_perf_event_init()
212 	 * for all CPUS (event->cpu == -1).  This depends on the whether  in __hw_perf_event_init()
214 	 * the perf event moves from one CPU to another CPU.  in __hw_perf_event_init()
215 	 * Checking the authorization on any CPU is fine as the hardware  in __hw_perf_event_init()
220 		if (cpuhw->info.auth_ctl & cpumf_ctr_ctl[i])  in __hw_perf_event_init()
221 			event->hw.config_base |= cpumf_ctr_ctl[i];  in __hw_perf_event_init()
225 	if (!event->hw.config_base) {  in __hw_perf_event_init()
226 		err = -EINVAL;  in __hw_perf_event_init()
231 	event->hw.sample_period = attr->sample_period;  in __hw_perf_event_init()
232 	local64_set(&event->hw.period_left, event->hw.sample_period);  in __hw_perf_event_init()
233 	event->hw.last_period  = event->hw.sample_period;  in __hw_perf_event_init()
236 			    __func__, err, event->hw.config_base);  in __hw_perf_event_init()
242 	struct perf_event_attr *attr = &event->attr;  in cf_diag_event_init()
243 	int err = -ENOENT;  in cf_diag_event_init()
246 			    "%s event %p cpu %d config %#llx type:%u "  in cf_diag_event_init()
248 			    event, event->cpu, attr->config, event->pmu->type,  in cf_diag_event_init()
249 			    attr->sample_type, atomic_read(&cf_diag_events));  in cf_diag_event_init()
251 	if (event->attr.config != PERF_EVENT_CPUM_CF_DIAG ||  in cf_diag_event_init()
252 	    event->attr.type != event->pmu->type)  in cf_diag_event_init()
260 	if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv ||  in cf_diag_event_init()
261 	    !(attr->sample_type & (PERF_SAMPLE_CPU | PERF_SAMPLE_RAW))) {  in cf_diag_event_init()
262 		err = -EOPNOTSUPP;  in cf_diag_event_init()
266 	/* Initialize for using the CPU-measurement counter facility */  in cf_diag_event_init()
270 			err = -EBUSY;  in cf_diag_event_init()
274 	event->destroy = cf_diag_perf_event_destroy;  in cf_diag_event_init()
278 		event->destroy(event);  in cf_diag_event_init()
299 		if (info->cfvn >= 1)  in cf_diag_ctrset_size()
303 		if (info->cfvn == 1)  in cf_diag_ctrset_size()
305 		else if (info->cfvn >= 3)  in cf_diag_ctrset_size()
309 		if (info->csvn >= 1 && info->csvn <= 5)  in cf_diag_ctrset_size()
311 		else if (info->csvn == 6)  in cf_diag_ctrset_size()
315 		if (info->csvn == 1)  in cf_diag_ctrset_size()
317 		else if (info->csvn == 2)  in cf_diag_ctrset_size()
319 		else if (info->csvn >= 3 && info->csvn <= 5)  in cf_diag_ctrset_size()
321 		else if (info->csvn == 6)  in cf_diag_ctrset_size()
325 		if (info->csvn > 3)  in cf_diag_ctrset_size()
358  * the CPUM-CF first and second version number determine the number of
375 	ctrdata->def = CF_DIAG_CTRSET_DEF;  in cf_diag_getctrset()
376 	ctrdata->set = ctrset;  in cf_diag_getctrset()
377 	ctrdata->res1 = 0;  in cf_diag_getctrset()
378 	ctrset_size = cf_diag_ctrset_size(ctrset, &cpuhw->info);  in cf_diag_getctrset()
386 			ctrdata->ctr = ctrset_size;  in cf_diag_getctrset()
394 			    __func__, ctrset, ctrset_size, cpuhw->info.cfvn,  in cf_diag_getctrset()
395 			    cpuhw->info.csvn, need, rc);  in cf_diag_getctrset()
405 	size_t offset = 0, done;  in cf_diag_getctr()  local
409 	sz -= sizeof(*trailer);			/* Always room for trailer */  in cf_diag_getctr()
411 		struct cf_ctrset_entry *ctrdata = data + offset;  in cf_diag_getctr()
416 		done = cf_diag_getctrset(ctrdata, i, sz - offset);  in cf_diag_getctr()
417 		offset += done;  in cf_diag_getctr()
419 				    "%s ctrset %d offset %zu done %zu\n",  in cf_diag_getctr()
420 				     __func__, i, offset, done);  in cf_diag_getctr()
422 	trailer = data + offset;  in cf_diag_getctr()
424 	return offset + sizeof(*trailer);  in cf_diag_getctr()
430 	for (; --counters >= 0; ++pstart, ++pstop)  in cf_diag_diffctrset()
432 			*pstop -= *pstart;  in cf_diag_diffctrset()
434 			*pstop = *pstart - *pstop;  in cf_diag_diffctrset()
447 	size_t offset = 0;  in cf_diag_diffctr()  local
449 	auth &= (1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1;  in cf_diag_diffctr()
451 		ctrstart = (struct cf_ctrset_entry *)(csd->start + offset);  in cf_diag_diffctr()
452 		ctrstop = (struct cf_ctrset_entry *)(csd->data + offset);  in cf_diag_diffctr()
456 				"in set %i\n", ctrstart->set);  in cf_diag_diffctr()
459 		auth &= ~cpumf_ctr_ctl[ctrstart->set];  in cf_diag_diffctr()
460 		if (ctrstart->def == CF_DIAG_CTRSET_DEF) {  in cf_diag_diffctr()
462 					  (u64 *)(ctrstop + 1), ctrstart->ctr);  in cf_diag_diffctr()
463 			offset += ctrstart->ctr * sizeof(u64) +  in cf_diag_diffctr()
467 				    "%s set %d ctr %d offset %zu auth %lx\n",  in cf_diag_diffctr()
468 				    __func__, ctrstart->set, ctrstart->ctr,  in cf_diag_diffctr()
469 				    offset, auth);  in cf_diag_diffctr()
470 	} while (ctrstart->def && auth);  in cf_diag_diffctr()
473 	trailer_start = (struct cf_trailer_entry *)(csd->start + offset);  in cf_diag_diffctr()
474 	trailer_stop = (struct cf_trailer_entry *)(csd->data + offset);  in cf_diag_diffctr()
475 	trailer_stop->progusage[0] = trailer_start->timestamp;  in cf_diag_diffctr()
485  * Return non-zero if an event overflow occurred.
496 	perf_sample_data_init(&data, 0, event->hw.last_period);  in cf_diag_push_sample()
500 	if (event->attr.sample_type & PERF_SAMPLE_CPU)  in cf_diag_push_sample()
501 		data.cpu_entry.cpu = event->cpu;  in cf_diag_push_sample()
502 	if (event->attr.sample_type & PERF_SAMPLE_RAW) {  in cf_diag_push_sample()
503 		raw.frag.size = csd->used;  in cf_diag_push_sample()
504 		raw.frag.data = csd->data;  in cf_diag_push_sample()
505 		raw.size = csd->used;  in cf_diag_push_sample()
511 			    "%s event %p cpu %d sample_type %#llx raw %d "  in cf_diag_push_sample()
512 			    "ov %d\n", __func__, event, event->cpu,  in cf_diag_push_sample()
513 			    event->attr.sample_type, raw.size, overflow);  in cf_diag_push_sample()
515 		event->pmu->stop(event, 0);  in cf_diag_push_sample()
525 	struct hw_perf_event *hwc = &event->hw;  in cf_diag_start()
528 			    "%s event %p cpu %d flags %#x hwc-state %#x\n",  in cf_diag_start()
529 			    __func__, event, event->cpu, flags, hwc->state);  in cf_diag_start()
530 	if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))  in cf_diag_start()
533 	/* (Re-)enable and activate all counter sets */  in cf_diag_start()
535 	hwc->state = 0;  in cf_diag_start()
536 	ctr_set_multiple_enable(&cpuhw->state, hwc->config_base);  in cf_diag_start()
537 	lcctl(cpuhw->state);	/* Enable counter sets */  in cf_diag_start()
538 	csd->used = cf_diag_getctr(csd->start, sizeof(csd->start),  in cf_diag_start()
539 				   event->hw.config_base);  in cf_diag_start()
540 	ctr_set_multiple_start(&cpuhw->state, hwc->config_base);  in cf_diag_start()
548 	struct hw_perf_event *hwc = &event->hw;  in cf_diag_stop()
551 			    "%s event %p cpu %d flags %#x hwc-state %#x\n",  in cf_diag_stop()
552 			    __func__, event, event->cpu, flags, hwc->state);  in cf_diag_stop()
555 	ctr_set_multiple_stop(&cpuhw->state, hwc->config_base);  in cf_diag_stop()
556 	local64_inc(&event->count);  in cf_diag_stop()
557 	csd->used = cf_diag_getctr(csd->data, sizeof(csd->data),  in cf_diag_stop()
558 				   event->hw.config_base);  in cf_diag_stop()
559 	if (cf_diag_diffctr(csd, event->hw.config_base))  in cf_diag_stop()
561 	hwc->state |= PERF_HES_STOPPED;  in cf_diag_stop()
570 			    "%s event %p cpu %d flags %#x cpuhw %p\n",  in cf_diag_add()
571 			    __func__, event, event->cpu, flags, cpuhw);  in cf_diag_add()
573 	if (cpuhw->flags & PMU_F_IN_USE) {  in cf_diag_add()
574 		err = -EAGAIN;  in cf_diag_add()
578 	event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;  in cf_diag_add()
580 	cpuhw->flags |= PMU_F_IN_USE;  in cf_diag_add()
593 			    "%s event %p cpu %d flags %#x\n",  in cf_diag_del()
594 			   __func__, event, event->cpu, flags);  in cf_diag_del()
597 	ctr_set_multiple_stop(&cpuhw->state, event->hw.config_base);  in cf_diag_del()
598 	ctr_set_multiple_disable(&cpuhw->state, event->hw.config_base);  in cf_diag_del()
599 	cpuhw->flags &= ~PMU_F_IN_USE;  in cf_diag_del()
609 PMU_FORMAT_ATTR(event, "config:0-63");
645 /* Get the CPU speed, try sampling facility first and CPU attributes second. */
658 	if (test_facility(34)) {		/* CPU speed extract static part */  in cf_diag_get_cpu_speed()
661 		if (mhz != -1UL)  in cf_diag_get_cpu_speed()
667  * event raw data. This relies on the CPU Measurement Counter Facility device
677 		return -ENODEV;  in cf_diag_init()
682 	if (need > sizeof(((struct cf_diag_csd *)0)->start)) {  in cf_diag_init()
685 		return -ENOMEM;  in cf_diag_init()
692 		return -ENOMEM;  in cf_diag_init()
696 	rc = perf_pmu_register(&cf_diag, "cpum_cf_diag", -1);  in cf_diag_init()