Lines Matching +full:counter +full:- +full:1
1 // SPDX-License-Identifier: GPL-2.0
3 * Performance event support for s390x - CPU-measurement Counter Facility
26 CPUMF_CTR_SET_BASIC = 0, /* Basic Counter Set */
27 CPUMF_CTR_SET_USER = 1, /* Problem-State Counter Set */
28 CPUMF_CTR_SET_CRYPTO = 2, /* Crypto-Activity Counter Set */
29 CPUMF_CTR_SET_EXT = 3, /* Extended Counter Set */
30 CPUMF_CTR_SET_MT_DIAG = 4, /* MT-diagnostic Counter Set */
32 /* Maximum number of counter sets */
86 unsigned char start[PAGE_SIZE]; /* Counter set at event add */
87 unsigned char stop[PAGE_SIZE]; /* Counter set at event delete */
88 unsigned char data[PAGE_SIZE]; /* Counter set at /dev/hwctr */
89 unsigned int sets; /* # Counter set saved in memory */
96 * The CPU Measurement query counter information instruction contains
98 * does not change when running on a particular machine, such as counter
100 * of counter sets. Extract this information at device driver initialization.
129 * Get pointer to per-cpu structure.
132 * - cfset_copy_all(): This function is protected by cpus_read_lock(), so
137 * - pmu_{en|dis}able(), pmu_{add|del}()and pmu_{start|stop}():
139 * - cfset_ioctl_{on|off}, cfset_cpu_read(): see comment cfset_copy_all().
150 return q->cpucf; in get_cpu_cfhw()
160 /* Disable counter sets on dedicated CPU */
174 on_each_cpu(cpum_cf_reset_cpu, NULL, 1); in cpum_cf_free_root()
196 refcount_set(&cpu_cf_root.refcnt, 1); in cpum_cf_alloc_root()
197 on_each_cpu(cpum_cf_reset_cpu, NULL, 1); in cpum_cf_alloc_root()
200 rc = -ENOMEM; in cpum_cf_alloc_root()
206 /* Free CPU counter data structure for a PMU */
221 cpuhw = p->cpucf; in cpum_cf_free_cpu()
229 if (refcount_dec_and_test(&cpuhw->refcnt)) { in cpum_cf_free_cpu()
231 p->cpucf = NULL; in cpum_cf_free_cpu()
238 /* Allocate CPU counter data structure for a PMU. Called under mutex lock. */
250 cpuhw = p->cpucf; in cpum_cf_alloc_cpu()
255 p->cpucf = cpuhw; in cpum_cf_alloc_cpu()
256 refcount_set(&cpuhw->refcnt, 1); in cpum_cf_alloc_cpu()
258 rc = -ENOMEM; in cpum_cf_alloc_cpu()
261 refcount_inc(&cpuhw->refcnt); in cpum_cf_alloc_cpu()
267 * invoked. Adjust the reference counter for the anchor. in cpum_cf_alloc_cpu()
279 * If cpu is -1, track task on all available CPUs. This requires
282 * If cpu is non-zero install event on this CPU only. This setup handles
290 if (cpu == -1) { in cpum_cf_alloc()
292 return -ENOMEM; in cpum_cf_alloc()
311 if (cpu == -1) { in cpum_cf_free()
319 #define CF_DIAG_CTRSET_DEF 0xfeef /* Counter set header mark */
322 /* Counter sets are stored as data stream in a page sized memory buffer and
324 * Each counter set starts with an eight byte header consisting of:
325 * - a two byte eye catcher (0xfeef)
326 * - a one byte counter set number
327 * - a two byte counter set size (indicates the number of counters in this set)
328 * - a three byte reserved value (must be zero) to make the header the same
329 * size as a counter value.
330 * All counter values are eight byte in size.
332 * All counter sets are followed by a 64 byte trailer.
334 * - flag field indicating valid fields when corresponding bit set
335 * - the counter facility first and second version number
336 * - the CPU speed if nonzero
337 * - the time stamp the counter sets have been collected
338 * - the time of day (TOD) base value
339 * - the machine type.
341 * The counter sets are saved when the process is prepared to be executed on a
343 * The difference of both counter sets are calculated and stored in the event
346 struct cf_ctrset_entry { /* CPU-M CF counter set entry (8 byte) */
347 unsigned int def:16; /* 0-15 Data Entry Format */
348 unsigned int set:16; /* 16-31 Counter set identifier */
349 unsigned int ctr:16; /* 32-47 Number of stored counters */
350 unsigned int res1:16; /* 48-63 Reserved */
353 struct cf_trailer_entry { /* CPU-M CF_DIAG trailer (64 byte) */
354 /* 0 - 7 */
357 unsigned int clock_base:1; /* TOD clock base set */
358 unsigned int speed:1; /* CPU speed set */
360 unsigned int mtda:1; /* Loss of MT ctr. data alert */
361 unsigned int caca:1; /* Counter auth. change alert */
362 unsigned int lcda:1; /* Loss of counter data alert */
364 unsigned long flags; /* 0-63 All indicators */
366 /* 8 - 15 */
367 unsigned int cfvn:16; /* 64-79 Ctr First Version */
368 unsigned int csvn:16; /* 80-95 Ctr Second Version */
369 unsigned int cpu_speed:32; /* 96-127 CPU speed */
370 /* 16 - 23 */
371 unsigned long timestamp; /* 128-191 Timestamp (TOD) */
372 /* 24 - 55 */
382 /* 56 - 63 */
393 te->cfvn = cpumf_ctr_info.cfvn; /* Counter version numbers */ in cfdiag_trailer()
394 te->csvn = cpumf_ctr_info.csvn; in cfdiag_trailer()
397 te->mach_type = cpuid.machine; in cfdiag_trailer()
398 te->cpu_speed = cfdiag_cpu_speed; in cfdiag_trailer()
399 if (te->cpu_speed) in cfdiag_trailer()
400 te->speed = 1; in cfdiag_trailer()
401 te->clock_base = 1; /* Save clock base */ in cfdiag_trailer()
402 te->tod_base = tod_clock_base.tod; in cfdiag_trailer()
403 te->timestamp = get_tod_clock_fast(); in cfdiag_trailer()
407 * The number of counters per counter set varies between machine generations,
409 * Determine each counter set size at device driver initialization and
419 if (cpumf_ctr_info.cfvn >= 1) in cpum_cf_make_setsize()
423 if (cpumf_ctr_info.cfvn == 1) in cpum_cf_make_setsize()
429 if (cpumf_ctr_info.csvn >= 1 && cpumf_ctr_info.csvn <= 5) in cpum_cf_make_setsize()
435 if (cpumf_ctr_info.csvn == 1) in cpum_cf_make_setsize()
455 * Return the maximum possible counter set size (in number of 8 byte counters)
463 /* Read a counter set. The counter set number determines the counter set and
464 * the CPUM-CF first and second version number determine the number of
465 * available counters in each counter set.
466 * Each counter set starts with header containing the counter set number and
469 * The functions returns the number of bytes occupied by this counter set
471 * If there is no counter in the counter set, this counter set is useless and
474 * Note that the counter sets may not be enabled or active and the stcctm
484 ctrdata->def = CF_DIAG_CTRSET_DEF; in cfdiag_getctrset()
485 ctrdata->set = ctrset; in cfdiag_getctrset()
486 ctrdata->res1 = 0; in cfdiag_getctrset()
493 (u64 *)(ctrdata + 1)); in cfdiag_getctrset()
496 ctrdata->ctr = ctrset_size; in cfdiag_getctrset()
512 /* Read out all counter sets and save them in the provided data buffer.
523 sz -= sizeof(*trailer); /* Always room for trailer */ in cfdiag_getctr()
528 continue; /* Counter set not authorized */ in cfdiag_getctr()
530 done = cfdiag_getctrset(ctrdata, i, sz - offset, error_ok); in cfdiag_getctr()
538 /* Calculate the difference for each counter in a counter set. */
541 for (; --counters >= 0; ++pstart, ++pstop) in cfdiag_diffctrset()
543 *pstop -= *pstart; in cfdiag_diffctrset()
545 *pstop = *pstart - *pstop + 1; in cfdiag_diffctrset()
548 /* Scan the counter sets and calculate the difference of each counter
549 * in each set. The result is the increment of each counter during the
550 * period the counter set has been activated.
560 auth &= (1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1; in cfdiag_diffctr()
562 ctrstart = (struct cf_ctrset_entry *)(cpuhw->start + offset); in cfdiag_diffctr()
563 ctrstop = (struct cf_ctrset_entry *)(cpuhw->stop + offset); in cfdiag_diffctr()
566 pr_err_once("cpum_cf_diag counter set compare error " in cfdiag_diffctr()
567 "in set %i\n", ctrstart->set); in cfdiag_diffctr()
570 auth &= ~cpumf_ctr_ctl[ctrstart->set]; in cfdiag_diffctr()
571 if (ctrstart->def == CF_DIAG_CTRSET_DEF) { in cfdiag_diffctr()
572 cfdiag_diffctrset((u64 *)(ctrstart + 1), in cfdiag_diffctr()
573 (u64 *)(ctrstop + 1), ctrstart->ctr); in cfdiag_diffctr()
574 offset += ctrstart->ctr * sizeof(u64) + in cfdiag_diffctr()
577 } while (ctrstart->def && auth); in cfdiag_diffctr()
580 trailer_start = (struct cf_trailer_entry *)(cpuhw->start + offset); in cfdiag_diffctr()
581 trailer_stop = (struct cf_trailer_entry *)(cpuhw->stop + offset); in cfdiag_diffctr()
582 trailer_stop->progusage[0] = trailer_start->timestamp; in cfdiag_diffctr()
584 return 1; in cfdiag_diffctr()
610 /* check required version for counter sets */ in validate_ctr_version()
614 if (cpumf_ctr_info.cfvn < 1) in validate_ctr_version()
615 err = -EOPNOTSUPP; in validate_ctr_version()
618 if ((cpumf_ctr_info.csvn >= 1 && cpumf_ctr_info.csvn <= 5 && in validate_ctr_version()
620 err = -EOPNOTSUPP; in validate_ctr_version()
623 if (cpumf_ctr_info.csvn < 1) in validate_ctr_version()
624 err = -EOPNOTSUPP; in validate_ctr_version()
625 if ((cpumf_ctr_info.csvn == 1 && config > 159) || in validate_ctr_version()
630 err = -EOPNOTSUPP; in validate_ctr_version()
634 err = -EOPNOTSUPP; in validate_ctr_version()
636 * MT-diagnostic counters are read-only. The counter set in validate_ctr_version()
639 * also disables the counter set. State changes are ignored in validate_ctr_version()
641 * a kernel parameter only, the counter set is either disabled in validate_ctr_version()
645 * counter set is enabled and active. in validate_ctr_version()
651 err = -EOPNOTSUPP; in validate_ctr_version()
654 err = -EOPNOTSUPP; in validate_ctr_version()
662 * Enable and activate the CPU-counter sets according
663 * to the per-cpu control state.
670 if (!cpuhw || (cpuhw->flags & PMU_F_ENABLED)) in cpumf_pmu_enable()
673 err = lcctl(cpuhw->state | cpuhw->dev_state); in cpumf_pmu_enable()
677 cpuhw->flags |= PMU_F_ENABLED; in cpumf_pmu_enable()
682 * Disable and enable (inactive) the CPU-counter sets according
683 * to the per-cpu control state.
691 if (!cpuhw || !(cpuhw->flags & PMU_F_ENABLED)) in cpumf_pmu_disable()
694 inactive = cpuhw->state & ~((1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1); in cpumf_pmu_disable()
695 inactive |= cpuhw->dev_state; in cpumf_pmu_disable()
700 cpuhw->flags &= ~PMU_F_ENABLED; in cpumf_pmu_disable()
706 cpum_cf_free(event->cpu); in hw_perf_event_destroy()
709 /* CPUMF <-> perf event mappings for kernel+userspace (basic set) */
712 [PERF_COUNT_HW_INSTRUCTIONS] = 1,
713 [PERF_COUNT_HW_CACHE_REFERENCES] = -1,
714 [PERF_COUNT_HW_CACHE_MISSES] = -1,
715 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = -1,
716 [PERF_COUNT_HW_BRANCH_MISSES] = -1,
717 [PERF_COUNT_HW_BUS_CYCLES] = -1,
719 /* CPUMF <-> perf event mappings for userspace (problem-state set) */
723 [PERF_COUNT_HW_CACHE_REFERENCES] = -1,
724 [PERF_COUNT_HW_CACHE_MISSES] = -1,
725 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = -1,
726 [PERF_COUNT_HW_BRANCH_MISSES] = -1,
727 [PERF_COUNT_HW_BUS_CYCLES] = -1,
738 struct perf_event_attr *attr = &event->attr; in __hw_perf_event_init()
739 struct hw_perf_event *hwc = &event->hw; in __hw_perf_event_init()
747 if (attr->exclude_kernel || attr->exclude_user || in __hw_perf_event_init()
748 attr->exclude_hv) in __hw_perf_event_init()
749 return -EOPNOTSUPP; in __hw_perf_event_init()
750 ev = attr->config; in __hw_perf_event_init()
755 return -ENOENT; in __hw_perf_event_init()
756 ev = attr->config; in __hw_perf_event_init()
757 if (!attr->exclude_user && attr->exclude_kernel) { in __hw_perf_event_init()
759 * Count user space (problem-state) only in __hw_perf_event_init()
760 * Handle events 32 and 33 as 0:u and 1:u in __hw_perf_event_init()
764 return -EOPNOTSUPP; in __hw_perf_event_init()
767 } else if (!attr->exclude_kernel && attr->exclude_user) { in __hw_perf_event_init()
769 return -EOPNOTSUPP; in __hw_perf_event_init()
774 return -EOPNOTSUPP; in __hw_perf_event_init()
781 return -ENOENT; in __hw_perf_event_init()
784 if (ev == -1) in __hw_perf_event_init()
785 return -ENOENT; in __hw_perf_event_init()
788 return -ENOENT; in __hw_perf_event_init()
790 /* Obtain the counter set to which the specified counter belongs */ in __hw_perf_event_init()
800 * counter number in the 'config' member and the counter in __hw_perf_event_init()
802 * It is later used to enable/disable the counter(s). in __hw_perf_event_init()
804 hwc->config = ev; in __hw_perf_event_init()
805 hwc->config_base = cpumf_ctr_ctl[set]; in __hw_perf_event_init()
808 /* The counter could not be associated to a counter set */ in __hw_perf_event_init()
809 return -EINVAL; in __hw_perf_event_init()
812 /* Initialize for using the CPU-measurement counter facility */ in __hw_perf_event_init()
813 if (cpum_cf_alloc(event->cpu)) in __hw_perf_event_init()
814 return -ENOMEM; in __hw_perf_event_init()
815 event->destroy = hw_perf_event_destroy; in __hw_perf_event_init()
818 * Finally, validate version and authorization of the counter set. in __hw_perf_event_init()
819 * If the particular CPU counter set is not authorized, in __hw_perf_event_init()
820 * return with -ENOENT in order to fall back to other in __hw_perf_event_init()
823 if (!(hwc->config_base & cpumf_ctr_info.auth_ctl)) in __hw_perf_event_init()
824 return -ENOENT; in __hw_perf_event_init()
825 return validate_ctr_version(hwc->config, set); in __hw_perf_event_init()
830 * - PERF_TYPE_HARDWARE:
831 * - pmu->type:
838 u64 ev = event->attr.config; in cpumf_pmu_event_type()
850 unsigned int type = event->attr.type; in cpumf_pmu_event_init()
855 else if (event->pmu->type == type) in cpumf_pmu_event_init()
859 return -ENOENT; in cpumf_pmu_event_init()
861 if (unlikely(err) && event->destroy) in cpumf_pmu_event_init()
862 event->destroy(event); in cpumf_pmu_event_init()
873 prev = local64_read(&event->hw.prev_count); in hw_perf_event_reset()
874 err = ecctr(event->hw.config, &new); in hw_perf_event_reset()
878 /* The counter is not (yet) available. This in hw_perf_event_reset()
879 * might happen if the counter set to which in hw_perf_event_reset()
880 * this counter belongs is in the disabled in hw_perf_event_reset()
885 } while (local64_cmpxchg(&event->hw.prev_count, prev, new) != prev); in hw_perf_event_reset()
896 prev = local64_read(&event->hw.prev_count); in hw_perf_event_update()
897 err = ecctr(event->hw.config, &new); in hw_perf_event_update()
900 } while (local64_cmpxchg(&event->hw.prev_count, prev, new) != prev); in hw_perf_event_update()
902 delta = (prev <= new) ? new - prev in hw_perf_event_update()
903 : (-1ULL - prev) + new + 1; /* overflow */ in hw_perf_event_update()
904 local64_add(delta, &event->count); in hw_perf_event_update()
909 if (event->hw.state & PERF_HES_STOPPED) in cpumf_pmu_read()
918 struct hw_perf_event *hwc = &event->hw; in cpumf_pmu_start()
921 if (!(hwc->state & PERF_HES_STOPPED)) in cpumf_pmu_start()
924 hwc->state = 0; in cpumf_pmu_start()
926 /* (Re-)enable and activate the counter set */ in cpumf_pmu_start()
927 ctr_set_enable(&cpuhw->state, hwc->config_base); in cpumf_pmu_start()
928 ctr_set_start(&cpuhw->state, hwc->config_base); in cpumf_pmu_start()
930 /* The counter set to which this counter belongs can be already active. in cpumf_pmu_start()
931 * Because all counters in a set are active, the event->hw.prev_count in cpumf_pmu_start()
932 * needs to be synchronized. At this point, the counter set can be in in cpumf_pmu_start()
935 if (hwc->config == PERF_EVENT_CPUM_CF_DIAG) { in cpumf_pmu_start()
936 cpuhw->usedss = cfdiag_getctr(cpuhw->start, in cpumf_pmu_start()
937 sizeof(cpuhw->start), in cpumf_pmu_start()
938 hwc->config_base, true); in cpumf_pmu_start()
943 /* Increment refcount for counter sets */ in cpumf_pmu_start()
945 if ((hwc->config_base & cpumf_ctr_ctl[i])) in cpumf_pmu_start()
946 atomic_inc(&cpuhw->ctr_set[i]); in cpumf_pmu_start()
949 /* Create perf event sample with the counter sets as raw data. The sample
954 * Return non-zero if an event overflow occurred.
965 perf_sample_data_init(&data, 0, event->hw.last_period); in cfdiag_push_sample()
969 if (event->attr.sample_type & PERF_SAMPLE_CPU) in cfdiag_push_sample()
970 data.cpu_entry.cpu = event->cpu; in cfdiag_push_sample()
971 if (event->attr.sample_type & PERF_SAMPLE_RAW) { in cfdiag_push_sample()
972 raw.frag.size = cpuhw->usedss; in cfdiag_push_sample()
973 raw.frag.data = cpuhw->stop; in cfdiag_push_sample()
979 event->pmu->stop(event, 0); in cfdiag_push_sample()
988 struct hw_perf_event *hwc = &event->hw; in cpumf_pmu_stop()
991 if (!(hwc->state & PERF_HES_STOPPED)) { in cpumf_pmu_stop()
992 /* Decrement reference count for this counter set and if this in cpumf_pmu_stop()
993 * is the last used counter in the set, clear activation in cpumf_pmu_stop()
994 * control and set the counter set state to inactive. in cpumf_pmu_stop()
997 if (!(hwc->config_base & cpumf_ctr_ctl[i])) in cpumf_pmu_stop()
999 if (!atomic_dec_return(&cpuhw->ctr_set[i])) in cpumf_pmu_stop()
1000 ctr_set_stop(&cpuhw->state, cpumf_ctr_ctl[i]); in cpumf_pmu_stop()
1002 hwc->state |= PERF_HES_STOPPED; in cpumf_pmu_stop()
1005 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { in cpumf_pmu_stop()
1006 if (hwc->config == PERF_EVENT_CPUM_CF_DIAG) { in cpumf_pmu_stop()
1007 local64_inc(&event->count); in cpumf_pmu_stop()
1008 cpuhw->usedss = cfdiag_getctr(cpuhw->stop, in cpumf_pmu_stop()
1009 sizeof(cpuhw->stop), in cpumf_pmu_stop()
1010 event->hw.config_base, in cpumf_pmu_stop()
1012 if (cfdiag_diffctr(cpuhw, event->hw.config_base)) in cpumf_pmu_stop()
1017 hwc->state |= PERF_HES_UPTODATE; in cpumf_pmu_stop()
1025 ctr_set_enable(&cpuhw->state, event->hw.config_base); in cpumf_pmu_add()
1026 event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; in cpumf_pmu_add()
1041 /* Check if any counter in the counter set is still used. If not used, in cpumf_pmu_del()
1042 * change the counter set to the disabled state. This also clears the in cpumf_pmu_del()
1047 * cpumf_pmu_start() always has to reenable a counter set. in cpumf_pmu_del()
1050 if (!atomic_read(&cpuhw->ctr_set[i])) in cpumf_pmu_del()
1051 ctr_set_disable(&cpuhw->state, cpumf_ctr_ctl[i]); in cpumf_pmu_del()
1068 static struct cfset_session { /* CPUs and counter set bit mask */
1092 * - CPU add: Nothing is done since a file descriptor can not be created
1094 * - CPU delete: Handled by common code via pmu_disable(), pmu_stop() and
1140 /* Return true if store counter set multiple instruction is available */
1146 /* CPU-measurement alerts for the counter facility */
1165 /* counter authorization change alert */ in cpumf_measurement_alert()
1169 /* loss of counter data alert */ in cpumf_measurement_alert()
1171 pr_err("CPU[%i] Counter data was lost\n", smp_processor_id()); in cpumf_measurement_alert()
1173 /* loss of MT counter data alert */ in cpumf_measurement_alert()
1175 pr_warn("CPU[%i] MT counter data was lost\n", in cpumf_measurement_alert()
1184 /* Extract counter measurement facility information */ in cpumf_pmu_init()
1186 return -ENODEV; in cpumf_pmu_init()
1188 /* Determine and store counter set sizes for later reference */ in cpumf_pmu_init()
1193 * Clear bit 15 of cr0 to unauthorize problem-state to in cpumf_pmu_init()
1198 /* register handler for measurement-alert interruptions */ in cpumf_pmu_init()
1202 pr_err("Registering for CPU-measurement alerts failed with rc=%i\n", rc); in cpumf_pmu_init()
1207 cf_dbg = debug_register(KMSG_COMPONENT, 2, 1, 128); in cpumf_pmu_init()
1210 rc = -ENOMEM; in cpumf_pmu_init()
1216 rc = perf_pmu_register(&cpumf_pmu, "cpum_cf", -1); in cpumf_pmu_init()
1220 } else if (stccm_avail()) { /* Setup counter set device */ in cpumf_pmu_init()
1237 /* Support for the CPU Measurement Facility counter set extraction using
1239 * counter set via normal file operations.
1243 unsigned int sets; /* Counter set bit mask */
1247 struct cfset_request { /* CPUs and counter set bit mask */
1248 unsigned long ctrset; /* Bit mask of counter set to read */
1258 /* Remove current request from global bookkeeping. Maintain a counter set bit
1264 list_del(&p->node); in cfset_session_del()
1267 /* Add current request to global bookkeeping. Maintain a counter set bit mask
1273 list_add(&p->node, &cfset_session.head); in cfset_session_add()
1278 * The cpu_cf_events::dev_state is used to denote counter sets in use by this
1280 * value is zero and no additional counter sets will be included.
1288 * CPU Measurement counter facility.
1298 * When /dev/hwctr interface is also used at the same time, the counter sets
1300 * However this is not a problem and does not lead to wrong counter values
1301 * for the perf_event_open() SVC. The current counter value will be recorded
1302 * during schedule-in. At schedule-out time the current counter value is
1305 /* Stop all counter sets via ioctl interface */
1312 /* Check if any counter set used by /dev/hwctr */ in cfset_ioctl_off()
1314 if ((p->sets & cpumf_ctr_ctl[rc])) { in cfset_ioctl_off()
1315 if (!atomic_dec_return(&cpuhw->ctr_set[rc])) { in cfset_ioctl_off()
1316 ctr_set_disable(&cpuhw->dev_state, in cfset_ioctl_off()
1318 ctr_set_stop(&cpuhw->dev_state, in cfset_ioctl_off()
1322 /* Keep perf_event_open counter sets */ in cfset_ioctl_off()
1323 rc = lcctl(cpuhw->dev_state | cpuhw->state); in cfset_ioctl_off()
1325 pr_err("Counter set stop %#llx of /dev/%s failed rc=%i\n", in cfset_ioctl_off()
1326 cpuhw->state, S390_HWCTR_DEVICE, rc); in cfset_ioctl_off()
1327 if (!cpuhw->dev_state) in cfset_ioctl_off()
1328 cpuhw->flags &= ~PMU_F_IN_USE; in cfset_ioctl_off()
1331 /* Start counter sets on particular CPU */
1338 cpuhw->flags |= PMU_F_IN_USE; in cfset_ioctl_on()
1339 ctr_set_enable(&cpuhw->dev_state, p->sets); in cfset_ioctl_on()
1340 ctr_set_start(&cpuhw->dev_state, p->sets); in cfset_ioctl_on()
1342 if ((p->sets & cpumf_ctr_ctl[rc])) in cfset_ioctl_on()
1343 atomic_inc(&cpuhw->ctr_set[rc]); in cfset_ioctl_on()
1344 rc = lcctl(cpuhw->dev_state | cpuhw->state); /* Start counter sets */ in cfset_ioctl_on()
1346 atomic_inc(&p->cpus_ack); in cfset_ioctl_on()
1348 pr_err("Counter set start %#llx of /dev/%s failed rc=%i\n", in cfset_ioctl_on()
1349 cpuhw->dev_state | cpuhw->state, S390_HWCTR_DEVICE, rc); in cfset_ioctl_on()
1357 cpuhw->dev_state = 0; in cfset_release_cpu()
1358 rc = lcctl(cpuhw->state); /* Keep perf_event_open counter sets */ in cfset_release_cpu()
1360 pr_err("Counter set release %#llx of /dev/%s failed rc=%i\n", in cfset_release_cpu()
1361 cpuhw->state, S390_HWCTR_DEVICE, rc); in cfset_release_cpu()
1372 .sets = req->ctrset, in cfset_all_stop()
1375 cpumask_and(&req->mask, &req->mask, cpu_online_mask); in cfset_all_stop()
1376 on_each_cpu_mask(&req->mask, cfset_ioctl_off, &p, 1); in cfset_all_stop()
1386 if (file->private_data) { in cfset_release()
1387 cfset_all_stop(file->private_data); in cfset_release()
1388 cfset_session_del(file->private_data); in cfset_release()
1389 kfree(file->private_data); in cfset_release()
1390 file->private_data = NULL; in cfset_release()
1393 on_each_cpu(cfset_release_cpu, NULL, 1); in cfset_release()
1394 cpum_cf_free(-1); in cfset_release()
1412 return -EPERM; in cfset_open()
1413 file->private_data = NULL; in cfset_open()
1417 rc = cpum_cf_alloc(-1); in cfset_open()
1420 refcount_set(&cfset_opencnt, 1); in cfset_open()
1432 .sets = req->ctrset, in cfset_all_start()
1439 return -ENOMEM; in cfset_all_start()
1440 cpumask_and(mask, &req->mask, cpu_online_mask); in cfset_all_start()
1441 on_each_cpu_mask(mask, cfset_ioctl_on, &p, 1); in cfset_all_start()
1443 on_each_cpu_mask(mask, cfset_ioctl_off, &p, 1); in cfset_all_start()
1444 rc = -EIO; in cfset_all_start()
1452 * To find out the size of the counter sets, any one CPU will do. They
1453 * all have the same counter sets.
1464 sizeof(((struct s390_ctrset_setdata *)0)->set) + in cfset_needspace()
1465 sizeof(((struct s390_ctrset_setdata *)0)->no_cnts); in cfset_needspace()
1467 bytes = sizeof(((struct s390_ctrset_read *)0)->no_cpus) + nr_cpu_ids * in cfset_needspace()
1468 (bytes + sizeof(((struct s390_ctrset_cpudata *)0)->cpu_nr) + in cfset_needspace()
1469 sizeof(((struct s390_ctrset_cpudata *)0)->no_sets)); in cfset_needspace()
1480 uptr = ctrset_read->data; in cfset_all_copy()
1486 rc = put_user(cpu, &ctrset_cpudata->cpu_nr); in cfset_all_copy()
1487 rc |= put_user(cpuhw->sets, &ctrset_cpudata->no_sets); in cfset_all_copy()
1488 rc |= copy_to_user(ctrset_cpudata->data, cpuhw->data, in cfset_all_copy()
1489 cpuhw->used); in cfset_all_copy()
1491 rc = -EFAULT; in cfset_all_copy()
1494 uptr += sizeof(struct s390_ctrset_cpudata) + cpuhw->used; in cfset_all_copy()
1498 if (put_user(cpus, &ctrset_read->no_cpus)) in cfset_all_copy()
1499 rc = -EFAULT; in cfset_all_copy()
1508 int rc = -1; in cfset_cpuset_read()
1512 p->set = cpumf_ctr_ctl[ctrset]; in cfset_cpuset_read()
1513 p->no_cnts = ctrset_size; in cfset_cpuset_read()
1514 rc = ctr_stcctm(ctrset, ctrset_size, (u64 *)p->cv); in cfset_cpuset_read()
1521 /* Read all counter sets. */
1530 cpuhw->used = 0; in cfset_cpu_read()
1531 cpuhw->sets = 0; in cfset_cpu_read()
1532 memset(cpuhw->data, 0, sizeof(cpuhw->data)); in cfset_cpu_read()
1534 /* Scan the counter sets */ in cfset_cpu_read()
1536 struct s390_ctrset_setdata *sp = (void *)cpuhw->data + in cfset_cpu_read()
1537 cpuhw->used; in cfset_cpu_read()
1539 if (!(p->sets & cpumf_ctr_ctl[set])) in cfset_cpu_read()
1540 continue; /* Counter set not in list */ in cfset_cpu_read()
1542 space = sizeof(cpuhw->data) - cpuhw->used; in cfset_cpu_read()
1545 cpuhw->used += space; in cfset_cpu_read()
1546 cpuhw->sets += 1; in cfset_cpu_read()
1558 return -ENOMEM; in cfset_all_read()
1560 p.sets = req->ctrset; in cfset_all_read()
1561 cpumask_and(mask, &req->mask, cpu_online_mask); in cfset_all_read()
1562 on_each_cpu_mask(mask, cfset_cpu_read, &p, 1); in cfset_all_read()
1570 int ret = -ENODATA; in cfset_ioctl_read()
1572 if (req && req->ctrset) in cfset_ioctl_read()
1579 struct cfset_request *req = file->private_data; in cfset_ioctl_stop()
1580 int ret = -ENXIO; in cfset_ioctl_stop()
1586 file->private_data = NULL; in cfset_ioctl_stop()
1602 if (file->private_data) in cfset_ioctl_start()
1603 return -EBUSY; in cfset_ioctl_start()
1606 return -EFAULT; in cfset_ioctl_start()
1608 return -EINVAL; in cfset_ioctl_start()
1614 return -EINVAL; /* Invalid counter set */ in cfset_ioctl_start()
1616 return -EINVAL; /* No counter set at all? */ in cfset_ioctl_start()
1620 return -ENOMEM; in cfset_ioctl_start()
1621 cpumask_clear(&preq->mask); in cfset_ioctl_start()
1624 if (copy_from_user(&preq->mask, umask, len)) { in cfset_ioctl_start()
1626 return -EFAULT; in cfset_ioctl_start()
1628 if (cpumask_empty(&preq->mask)) { in cfset_ioctl_start()
1630 return -EINVAL; in cfset_ioctl_start()
1633 if (put_user(need, &ustart->data_bytes)) { in cfset_ioctl_start()
1635 return -EFAULT; in cfset_ioctl_start()
1637 preq->ctrset = start.counter_sets; in cfset_ioctl_start()
1641 file->private_data = preq; in cfset_ioctl_start()
1650 * S390_HWCTR_START: Start the specified counter sets on a CPU list. The
1651 * counter set keeps running until explicitly stopped. Returns the number
1652 * of bytes needed to store the counter values. If another S390_HWCTR_START
1654 * command on the same file descriptor, -EBUSY is returned.
1655 * S390_HWCTR_READ: Read the counter set values from specified CPU list given
1657 * S390_HWCTR_STOP: Stops the counter sets on the CPU list given with the
1674 ret = cfset_ioctl_read(arg, file->private_data); in cfset_ioctl()
1677 ret = -ENOTTY; in cfset_ioctl()
1711 p.sets = rp->ctrset; in cfset_online_cpu()
1713 cpumask_set_cpu(cpu, &rp->mask); in cfset_online_cpu()
1730 p.sets = rp->ctrset; in cfset_offline_cpu()
1732 cpumask_clear_cpu(cpu, &rp->mask); in cfset_offline_cpu()
1754 /* Setup the event. Test for authorized counter sets and only include counter
1756 * counter sets result in specification exception (and panic).
1760 struct perf_event_attr *attr = &event->attr; in cfdiag_event_init2()
1764 event->hw.config = attr->config; in cfdiag_event_init2()
1765 event->hw.sample_period = attr->sample_period; in cfdiag_event_init2()
1766 local64_set(&event->hw.period_left, event->hw.sample_period); in cfdiag_event_init2()
1767 local64_set(&event->count, 0); in cfdiag_event_init2()
1768 event->hw.last_period = event->hw.sample_period; in cfdiag_event_init2()
1770 /* Add all authorized counter sets to config_base. The in cfdiag_event_init2()
1771 * the hardware init function is either called per-cpu or just once in cfdiag_event_init2()
1772 * for all CPUS (event->cpu == -1). This depends on the whether in cfdiag_event_init2()
1778 event->hw.config_base = get_authctrsets(); in cfdiag_event_init2()
1780 /* No authorized counter sets, nothing to count/sample */ in cfdiag_event_init2()
1781 if (!event->hw.config_base) in cfdiag_event_init2()
1782 err = -EINVAL; in cfdiag_event_init2()
1789 struct perf_event_attr *attr = &event->attr; in cfdiag_event_init()
1790 int err = -ENOENT; in cfdiag_event_init()
1792 if (event->attr.config != PERF_EVENT_CPUM_CF_DIAG || in cfdiag_event_init()
1793 event->attr.type != event->pmu->type) in cfdiag_event_init()
1798 * This event is useless without PERF_SAMPLE_RAW to return counter set in cfdiag_event_init()
1801 if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv || in cfdiag_event_init()
1802 !(attr->sample_type & (PERF_SAMPLE_CPU | PERF_SAMPLE_RAW))) { in cfdiag_event_init()
1803 err = -EOPNOTSUPP; in cfdiag_event_init()
1807 /* Initialize for using the CPU-measurement counter facility */ in cfdiag_event_init()
1808 if (cpum_cf_alloc(event->cpu)) in cfdiag_event_init()
1809 return -ENOMEM; in cfdiag_event_init()
1810 event->destroy = hw_perf_event_destroy; in cfdiag_event_init()
1814 event->destroy(event); in cfdiag_event_init()
1819 /* Create cf_diag/events/CF_DIAG event sysfs file. This counter is used
1820 * to collect the complete counter sets for a scheduled process. Target
1821 * are complete counter sets attached as raw data to the artificial event.
1822 * This results in complete counter sets available when a process is
1823 * scheduled. Contains the delta of every counter while the process was
1833 PMU_FORMAT_ATTR(event, "config:0-63");
1877 /* Calculate memory needed to store all counter sets together with header and
1878 * trailer data. This is independent of the counter set authorization which
1915 if (mhz != -1UL) in cfdiag_get_cpu_speed()
1925 /* Make sure the counter set data fits into predefined buffer. */ in cfset_init()
1927 if (need > sizeof(((struct cpu_cf_events *)0)->start)) { in cfset_init()
1930 return -ENOMEM; in cfset_init()
1940 rc = perf_pmu_register(&cf_diag, "cpum_cf_diag", -1); in cfset_init()