1 // SPDX-License-Identifier: GPL-2.0
3  * trace_hwlat.c - A simple Hardware Latency detector.
20  * Although certain hardware-inducing latencies are necessary (for example,
22  * and remote management) they can wreak havoc upon any OS-level performance
23  * guarantees toward low-latency, especially when the OS is not even made
27  * sampling the built-in CPU timer, looking for discontiguous readings.
31  *          environment requiring any kind of low-latency performance
34  * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. <jcm@redhat.com>
35  * Copyright (C) 2013-2016 Steven Rostedt, Red Hat, Inc. <srostedt@redhat.com>
55 #define DEFAULT_LAT_THRESHOLD	10			/* 10us */
60 static struct dentry *hwlat_sample_width;	/* sample width us */
61 static struct dentry *hwlat_sample_window;	/* sample window us */
78 /* Individual latency samples are stored here when detected. */
108 	struct trace_buffer *buffer = tr->array_buffer.buffer;  in trace_hwlat_sample()
110 	struct hwlat_entry *entry;  in trace_hwlat_sample()  local
117 	event = trace_buffer_lock_reserve(buffer, TRACE_HWLAT, sizeof(*entry),  in trace_hwlat_sample()
121 	entry	= ring_buffer_event_data(event);  in trace_hwlat_sample()
122 	entry->seqnum			= sample->seqnum;  in trace_hwlat_sample()
123 	entry->duration			= sample->duration;  in trace_hwlat_sample()
124 	entry->outer_duration		= sample->outer_duration;  in trace_hwlat_sample()
125 	entry->timestamp		= sample->timestamp;  in trace_hwlat_sample()
126 	entry->nmi_total_ts		= sample->nmi_total_ts;  in trace_hwlat_sample()
127 	entry->nmi_count		= sample->nmi_count;  in trace_hwlat_sample()
128 	entry->count			= sample->count;  in trace_hwlat_sample()
130 	if (!call_filter_check_discard(call, entry, buffer, event))  in trace_hwlat_sample()
138 #define time_sub(a, b)	((a) - (b))
155 			nmi_total_ts += time_get() - nmi_ts_start;  in trace_hwlat_callback()
163  * get_sample - sample the CPU TSC and look for likely hardware latencies
166  * hardware-induced latency. Called with interrupts disabled and with
178 	int ret = -1;  in get_sample()
248 	/* If we exceed the threshold value, we have found a hardware latency */  in get_sample()
250 		u64 latency;  in get_sample()  local
267 		latency = max(sample, outer_sample);  in get_sample()
269 		/* Keep a running maximum ever recorded hardware latency */  in get_sample()
270 		if (latency > tr->max_latency) {  in get_sample()
271 			tr->max_latency = latency;  in get_sample()
296 	if (!cpumask_equal(current_mask, current->cpus_ptr))  in move_to_next_cpu()
300 	cpumask_and(current_mask, cpu_online_mask, tr->tracing_cpumask);  in move_to_next_cpu()
321  * kthread_fn - The CPU time sampling/hardware latency detection kernel thread
324  * disable interrupts, which does (intentionally) introduce latency since we
343 		interval = hwlat_data.sample_window - hwlat_data.sample_width;  in kthread_fn()
360  * start_kthread - Kick off the hardware latency sampling/detector kthread
376 	cpumask_and(current_mask, cpu_online_mask, tr->tracing_cpumask);  in start_kthread()
383 		return -ENOMEM;  in start_kthread()
388 	sched_setaffinity(kthread->pid, current_mask);  in start_kthread()
397  * stop_kthread - Inform the hardware latency samping/detector kthread to stop
399  * This kicks the running hardware latency sampling/detector kernel thread and
411  * hwlat_read - Wrapper read function for reading both window and width
424 	u64 *entry = filp->private_data;  in hwlat_read()  local
428 	if (!entry)  in hwlat_read()
429 		return -EFAULT;  in hwlat_read()
434 	val = *entry;  in hwlat_read()
442  * hwlat_width_write - Write function for "width" entry
449  * to the hardware latency detector. It can be used to configure
450  * for how many us of the total window us we will actively sample for any
451  * hardware-induced latency periods. Obviously, it is not possible to
471 		err = -EINVAL;  in hwlat_width_write()
481  * hwlat_window_write - Write function for "window" entry
488  * to the hardware latency detetector. The window is the total time
489  * in us that will be considered one sample period. Conceptually, windows
490  * occur back-to-back and contain a sample width period during which
510 		err = -EINVAL;  in hwlat_window_write()
532  * init_tracefs - A function to initialize the tracefs interface files
546 		return -ENOMEM;  in init_tracefs()
550 		return -ENOMEM;  in init_tracefs()
570 	return -ENOMEM;  in init_tracefs()
593 		return -EBUSY;  in hwlat_tracer_init()
599 	tr->max_latency = 0;  in hwlat_tracer_init()