3  * Generic ring buffer
39  * The "absolute" timestamp in the buffer is only 59 bits.
67  * The ring buffer header is special. We must manually up keep it.
89  * The ring buffer is made up of a list of pages. A separate list of pages is
90  * allocated for each CPU. A writer may only write to a buffer that is
92  * from any per cpu buffer.
94  * The reader is special. For each per cpu buffer, the reader has its own
96  * page is swapped with another page in the ring buffer.
100  * again (as long as it is out of the ring buffer).
105  *   |reader|          RING BUFFER
116  *   |reader|          RING BUFFER
127  *   |reader|          RING BUFFER
138  *   |buffer|          RING BUFFER
150  * and swap that into the ring buffer.
288 /* inline for ring buffer fast paths */
312 #define for_each_buffer_cpu(buffer, cpu)		\  argument
313 	for_each_cpu(cpu, buffer->cpumask)
315 #define for_each_online_buffer_cpu(buffer, cpu)		\  argument
316 	for_each_cpu_and(cpu, buffer->cpumask, cpu_online_mask)
343 	unsigned char	 data[] RB_ALIGN_DATA;	/* data of buffer page */
352  * Note, the buffer_page list must be first. The buffer pages
353  * are allocated in cache lines, which means that each buffer
356  * add flags in the list struct pointers, to make the ring buffer
360 	struct list_head list;		/* list of buffer pages */
372  * The buffer page counters, write and entries, must be reset
439  *  ABSOLUTE - the buffer requests all events to have absolute time stamps
475  * head_page == tail_page && head == tail then buffer is empty.
481 	struct trace_buffer	*buffer;  member
526 	/* ring buffer pages to update, > 0 to add, < 0 to remove */
579 int ring_buffer_print_page_header(struct trace_buffer *buffer, struct trace_seq *s)  in ring_buffer_print_page_header()  argument
603 			 (unsigned int)buffer->subbuf_size,  in ring_buffer_print_page_header()
621  * is on the buffer that it passed in.
678 static inline u64 rb_time_stamp(struct trace_buffer *buffer);
682  * @buffer: The buffer that the event is on
686  * committed to the ring buffer. And must be called from the same
693  * the max nesting, then the write_stamp of the buffer is returned,
697 u64 ring_buffer_event_time_stamp(struct trace_buffer *buffer,  in ring_buffer_event_time_stamp()  argument
700 	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[smp_processor_id()];  in ring_buffer_event_time_stamp()
729  * ring_buffer_nr_dirty_pages - get the number of used pages in the ring buffer
730  * @buffer: The ring_buffer to get the number of pages from
733  * Returns the number of pages that have content in the ring buffer.
735 size_t ring_buffer_nr_dirty_pages(struct trace_buffer *buffer, int cpu)  in ring_buffer_nr_dirty_pages()  argument
741 	read = local_read(&buffer->buffers[cpu]->pages_read);  in ring_buffer_nr_dirty_pages()
742 	lost = local_read(&buffer->buffers[cpu]->pages_lost);  in ring_buffer_nr_dirty_pages()
743 	cnt = local_read(&buffer->buffers[cpu]->pages_touched);  in ring_buffer_nr_dirty_pages()
759 static __always_inline bool full_hit(struct trace_buffer *buffer, int cpu, int full)  in full_hit()  argument
761 	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];  in full_hit()
770 	 * Add one as dirty will never equal nr_pages, as the sub-buffer  in full_hit()
774 	dirty = ring_buffer_nr_dirty_pages(buffer, cpu) + 1;  in full_hit()
780  * rb_wake_up_waiters - wake up tasks waiting for ring buffer input
783  * ring buffer waiters queue.
812  * ring_buffer_wake_waiters - wake up any waiters on this ring buffer
813  * @buffer: The ring buffer to wake waiters on
814  * @cpu: The CPU buffer to wake waiters on
816  * In the case of a file that represents a ring buffer is closing,
819 void ring_buffer_wake_waiters(struct trace_buffer *buffer, int cpu)  in ring_buffer_wake_waiters()  argument
824 	if (!buffer)  in ring_buffer_wake_waiters()
830 		for_each_buffer_cpu(buffer, cpu)  in ring_buffer_wake_waiters()
831 			ring_buffer_wake_waiters(buffer, cpu);  in ring_buffer_wake_waiters()
833 		rbwork = &buffer->irq_work;  in ring_buffer_wake_waiters()
835 		if (WARN_ON_ONCE(!buffer->buffers))  in ring_buffer_wake_waiters()
840 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_wake_waiters()
841 		/* The CPU buffer may not have been initialized yet */  in ring_buffer_wake_waiters()
851 static bool rb_watermark_hit(struct trace_buffer *buffer, int cpu, int full)  in rb_watermark_hit()  argument
858 		return !ring_buffer_empty(buffer);  in rb_watermark_hit()
860 	cpu_buffer = buffer->buffers[cpu];  in rb_watermark_hit()
862 	if (!ring_buffer_empty_cpu(buffer, cpu)) {  in rb_watermark_hit()
871 		ret = !pagebusy && full_hit(buffer, cpu, full);  in rb_watermark_hit()
883 rb_wait_cond(struct rb_irq_work *rbwork, struct trace_buffer *buffer,  in rb_wait_cond()  argument
886 	if (rb_watermark_hit(buffer, cpu, full))  in rb_wait_cond()
899 	 * We don't clear it even if the buffer is no longer  in rb_wait_cond()
938  * ring_buffer_wait - wait for input to the ring buffer
939  * @buffer: buffer to wait on
940  * @cpu: the cpu buffer to wait on
946  * as data is added to any of the @buffer's cpu buffers. Otherwise
947  * it will wait for data to be added to a specific cpu buffer.
949 int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full,  in ring_buffer_wait()  argument
960 	 * data in any cpu buffer, or a specific buffer, put the  in ring_buffer_wait()
964 		rbwork = &buffer->irq_work;  in ring_buffer_wait()
968 		if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_wait()
970 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_wait()
988 				rb_wait_cond(rbwork, buffer, cpu, full, cond, data));  in ring_buffer_wait()
994  * ring_buffer_poll_wait - poll on buffer input
995  * @buffer: buffer to wait on
996  * @cpu: the cpu buffer to wait on
1002  * as data is added to any of the @buffer's cpu buffers. Otherwise
1003  * it will wait for data to be added to a specific cpu buffer.
1008 __poll_t ring_buffer_poll_wait(struct trace_buffer *buffer, int cpu,  in ring_buffer_poll_wait()  argument
1015 		rbwork = &buffer->irq_work;  in ring_buffer_poll_wait()
1018 		if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_poll_wait()
1021 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_poll_wait()
1028 		if (rb_watermark_hit(buffer, cpu, full))  in ring_buffer_poll_wait()
1034 		 * compare the amount in the ring buffer to shortest_full.  in ring_buffer_poll_wait()
1035 		 * If the amount in the ring buffer is greater than the  in ring_buffer_poll_wait()
1052 	 * checking if the ring buffer is empty.  Once the waiters_pending bit  in ring_buffer_poll_wait()
1059 	 * the buffer goes from empty to having content.  But as this race is  in ring_buffer_poll_wait()
1065 	if ((cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer)) ||  in ring_buffer_poll_wait()
1066 	    (cpu != RING_BUFFER_ALL_CPUS && !ring_buffer_empty_cpu(buffer, cpu)))  in ring_buffer_poll_wait()
1071 /* buffer may be either ring_buffer or ring_buffer_per_cpu */
1079 				atomic_inc(&__b->buffer->record_disabled); \
1090 static inline u64 rb_time_stamp(struct trace_buffer *buffer)  in rb_time_stamp()  argument
1095 	if (IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) && likely(buffer->clock == trace_clock_local))  in rb_time_stamp()
1098 		ts = buffer->clock();  in rb_time_stamp()
1104 u64 ring_buffer_time_stamp(struct trace_buffer *buffer)  in ring_buffer_time_stamp()  argument
1109 	time = rb_time_stamp(buffer);  in ring_buffer_time_stamp()
1116 void ring_buffer_normalize_time_stamp(struct trace_buffer *buffer,  in ring_buffer_normalize_time_stamp()  argument
1125  * Making the ring buffer lockless makes things tricky.
1130  * The reader page is always off the ring buffer, but when the
1132  * a new one from the buffer. The reader needs to take from
1182  *  the reader page with a page in the buffer, but before it
1498  * rb_check_pages - integrity check of buffer pages
1499  * @cpu_buffer: CPU buffer with pages to test
1512 	 * Walk the linked list underpinning the ring buffer and validate all  in rb_check_pages()
1519 	 * ring buffer size. Therefore, the code releases and re-acquires the  in rb_check_pages()
1583 static void *rb_range_meta(struct trace_buffer *buffer, int nr_pages, int cpu)  in rb_range_meta()  argument
1585 	int subbuf_size = buffer->subbuf_size + BUF_PAGE_HDR_SIZE;  in rb_range_meta()
1591 	bmeta = buffer->meta;  in rb_range_meta()
1645  * Return a specific sub-buffer for a given @cpu defined by @idx.
1653 	meta = rb_range_meta(cpu_buffer->buffer, 0, cpu_buffer->cpu);  in rb_range_buffer()
1662 	/* Map this buffer to the order that's in meta->buffers[] */  in rb_range_buffer()
1668 	if (ptr + subbuf_size > cpu_buffer->buffer->range_addr_end)  in rb_range_buffer()
1678 static bool rb_meta_init(struct trace_buffer *buffer, int scratch_size)  in rb_meta_init()  argument
1680 	unsigned long ptr = buffer->range_addr_start;  in rb_meta_init()
1686 	buffer->meta = bmeta;  in rb_meta_init()
1688 	total_size = buffer->range_addr_end - buffer->range_addr_start;  in rb_meta_init()
1693 	/* The first buffer will start word size after the meta page */  in rb_meta_init()
1699 		pr_info("Ring buffer boot meta mismatch of magic\n");  in rb_meta_init()
1704 		pr_info("Ring buffer boot meta mismatch of struct size\n");  in rb_meta_init()
1709 		pr_info("Ring buffer boot meta mismatch of total size\n");  in rb_meta_init()
1714 		pr_info("Ring buffer boot meta mismatch of offset outside of total size\n");  in rb_meta_init()
1719 		pr_info("Ring buffer boot meta mismatch of first buffer offset\n");  in rb_meta_init()
1738  * See if the existing memory contains valid ring buffer data.
1744 			      struct trace_buffer *buffer, int nr_pages,  in rb_cpu_meta_valid()  argument
1762 		pr_info("Ring buffer boot meta [%d] head buffer out of range\n", cpu);  in rb_cpu_meta_valid()
1768 		pr_info("Ring buffer boot meta [%d] commit buffer out of range\n", cpu);  in rb_cpu_meta_valid()
1780 			pr_info("Ring buffer boot meta [%d] array out of range\n", cpu);  in rb_cpu_meta_valid()
1785 			pr_info("Ring buffer boot meta [%d] buffer invalid commit\n", cpu);  in rb_cpu_meta_valid()
1790 			pr_info("Ring buffer boot meta [%d] array has duplicates\n", cpu);  in rb_cpu_meta_valid()
1881 		pr_info("Ring buffer reader page is invalid\n");  in rb_meta_validate_events()
1906 			pr_info("Ring buffer meta [%d] invalid buffer page\n",  in rb_meta_validate_events()
1911 		/* If the buffer has content, update pages_touched */  in rb_meta_validate_events()
1924 		pr_info("Ring buffer meta [%d] commit page not found\n",  in rb_meta_validate_events()
1932 	pr_info("Ring buffer meta [%d] is from previous boot!\n", cpu_buffer->cpu);  in rb_meta_validate_events()
1951 static void rb_range_meta_init(struct trace_buffer *buffer, int nr_pages, int scratch_size)  in rb_range_meta_init()  argument
1965 	if (rb_meta_init(buffer, scratch_size))  in rb_range_meta_init()
1971 		meta = rb_range_meta(buffer, nr_pages, cpu);  in rb_range_meta_init()
1973 		if (valid && rb_cpu_meta_valid(meta, cpu, buffer, nr_pages, subbuf_mask)) {  in rb_range_meta_init()
1984 			next_meta = rb_range_meta(buffer, nr_pages, cpu + 1);  in rb_range_meta_init()
1986 			next_meta = (void *)buffer->range_addr_end;  in rb_range_meta_init()
2001 		 * location of the ring buffer. Although their addresses  in rb_range_meta_init()
2056 	seq_printf(m, "buffer[%ld]:    %d\n", val, meta->buffers[val]);  in rbm_show()
2072 int ring_buffer_meta_seq_init(struct file *file, struct trace_buffer *buffer, int cpu)  in ring_buffer_meta_seq_init()  argument
2082 	m->private = buffer->buffers[cpu];  in ring_buffer_meta_seq_init()
2105 	struct trace_buffer *buffer = cpu_buffer->buffer;  in __rb_allocate_pages()  local
2142 	if (buffer->range_addr_start)  in __rb_allocate_pages()
2143 		meta = rb_range_meta(buffer, nr_pages, cpu_buffer->cpu);  in __rb_allocate_pages()
2162 			/* A range was given. Use that for the buffer page */  in __rb_allocate_pages()
2174 						cpu_buffer->buffer->subbuf_order);  in __rb_allocate_pages()
2180 		bpage->order = cpu_buffer->buffer->subbuf_order;  in __rb_allocate_pages()
2212 	 * The ring buffer page list is a circular list that does not  in rb_allocate_pages()
2227 rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu)  in rb_allocate_cpu_buffer()  argument
2241 	cpu_buffer->buffer = buffer;  in rb_allocate_cpu_buffer()
2243 	lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key);  in rb_allocate_cpu_buffer()
2261 	if (buffer->range_addr_start) {  in rb_allocate_cpu_buffer()
2267 		cpu_buffer->ring_meta = rb_range_meta(buffer, nr_pages, cpu);  in rb_allocate_cpu_buffer()
2277 					cpu_buffer->buffer->subbuf_order);  in rb_allocate_cpu_buffer()
2299 			pr_warn("Ring buffer meta buffers not all mapped\n");  in rb_allocate_cpu_buffer()
2317 		/* The valid meta buffer still needs to activate the head page */  in rb_allocate_cpu_buffer()
2362 	struct trace_buffer *buffer;  in alloc_buffer()  local
2370 	buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()),  in alloc_buffer()
2372 	if (!buffer)  in alloc_buffer()
2375 	if (!zalloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))  in alloc_buffer()
2378 	buffer->subbuf_order = order;  in alloc_buffer()
2380 	buffer->subbuf_size = subbuf_size - BUF_PAGE_HDR_SIZE;  in alloc_buffer()
2382 	/* Max payload is buffer page size - header (8bytes) */  in alloc_buffer()
2383 	buffer->max_data_size = buffer->subbuf_size - (sizeof(u32) * 2);  in alloc_buffer()
2385 	buffer->flags = flags;  in alloc_buffer()
2386 	buffer->clock = trace_clock_local;  in alloc_buffer()
2387 	buffer->reader_lock_key = key;  in alloc_buffer()
2389 	init_irq_work(&buffer->irq_work.work, rb_wake_up_waiters);  in alloc_buffer()
2390 	init_waitqueue_head(&buffer->irq_work.waiters);  in alloc_buffer()
2392 	buffer->cpus = nr_cpu_ids;  in alloc_buffer()
2395 	buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()),  in alloc_buffer()
2397 	if (!buffer->buffers)  in alloc_buffer()
2412 		/* Subtract the buffer meta data and word aligned */  in alloc_buffer()
2451 		buffer->range_addr_start = start;  in alloc_buffer()
2452 		buffer->range_addr_end = end;  in alloc_buffer()
2454 		rb_range_meta_init(buffer, nr_pages, scratch_size);  in alloc_buffer()
2458 		nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size);  in alloc_buffer()
2464 	cpumask_set_cpu(cpu, buffer->cpumask);  in alloc_buffer()
2465 	buffer->buffers[cpu] = rb_allocate_cpu_buffer(buffer, nr_pages, cpu);  in alloc_buffer()
2466 	if (!buffer->buffers[cpu])  in alloc_buffer()
2469 	ret = cpuhp_state_add_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node);  in alloc_buffer()
2473 	mutex_init(&buffer->mutex);  in alloc_buffer()
2475 	return buffer;  in alloc_buffer()
2478 	for_each_buffer_cpu(buffer, cpu) {  in alloc_buffer()
2479 		if (buffer->buffers[cpu])  in alloc_buffer()
2480 			rb_free_cpu_buffer(buffer->buffers[cpu]);  in alloc_buffer()
2482 	kfree(buffer->buffers);  in alloc_buffer()
2485 	free_cpumask_var(buffer->cpumask);  in alloc_buffer()
2488 	kfree(buffer);  in alloc_buffer()
2495  * @flags: attributes to set for the ring buffer.
2496  * @key: ring buffer reader_lock_key.
2499  * flag. This flag means that the buffer will overwrite old data
2500  * when the buffer wraps. If this flag is not set, the buffer will
2506 	/* Default buffer page size - one system page */  in __ring_buffer_alloc()
2515  * @flags: attributes to set for the ring buffer.
2516  * @order: sub-buffer order
2520  * @key: ring buffer reader_lock_key.
2523  * flag. This flag means that the buffer will overwrite old data
2524  * when the buffer wraps. If this flag is not set, the buffer will
2537 void *ring_buffer_meta_scratch(struct trace_buffer *buffer, unsigned int *size)  in ring_buffer_meta_scratch()  argument
2542 	if (!buffer || !buffer->meta)  in ring_buffer_meta_scratch()
2545 	meta = buffer->meta;  in ring_buffer_meta_scratch()
2556  * ring_buffer_free - free a ring buffer.
2557  * @buffer: the buffer to free.
2560 ring_buffer_free(struct trace_buffer *buffer)  in ring_buffer_free()  argument
2564 	cpuhp_state_remove_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node);  in ring_buffer_free()
2566 	irq_work_sync(&buffer->irq_work.work);  in ring_buffer_free()
2568 	for_each_buffer_cpu(buffer, cpu)  in ring_buffer_free()
2569 		rb_free_cpu_buffer(buffer->buffers[cpu]);  in ring_buffer_free()
2571 	kfree(buffer->buffers);  in ring_buffer_free()
2572 	free_cpumask_var(buffer->cpumask);  in ring_buffer_free()
2574 	kfree(buffer);  in ring_buffer_free()
2578 void ring_buffer_set_clock(struct trace_buffer *buffer,  in ring_buffer_set_clock()  argument
2581 	buffer->clock = clock;  in ring_buffer_set_clock()
2584 void ring_buffer_set_time_stamp_abs(struct trace_buffer *buffer, bool abs)  in ring_buffer_set_time_stamp_abs()  argument
2586 	buffer->time_stamp_abs = abs;  in ring_buffer_set_time_stamp_abs()
2589 bool ring_buffer_time_stamp_abs(struct trace_buffer *buffer)  in ring_buffer_time_stamp_abs()  argument
2591 	return buffer->time_stamp_abs;  in ring_buffer_time_stamp_abs()
2631 	 * from the ring buffer  in rb_remove_pages()
2660 	/* make sure pages points to a valid page in the ring buffer */  in rb_remove_pages()
2675 	/* last buffer page to remove */  in rb_remove_pages()
2692 			 * bytes consumed in ring buffer from here.  in rb_remove_pages()
2726 	 * in the ring buffer. Now we are racing with the writer trying to  in rb_insert_pages()
2819  * ring_buffer_resize - resize the ring buffer
2820  * @buffer: the buffer to resize.
2822  * @cpu_id: the cpu buffer to resize
2824  * Minimum size is 2 * buffer->subbuf_size.
2828 int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size,  in ring_buffer_resize()  argument
2836 	 * Always succeed at resizing a non-existent buffer:  in ring_buffer_resize()
2838 	if (!buffer)  in ring_buffer_resize()
2841 	/* Make sure the requested buffer exists */  in ring_buffer_resize()
2843 	    !cpumask_test_cpu(cpu_id, buffer->cpumask))  in ring_buffer_resize()
2846 	nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size);  in ring_buffer_resize()
2852 	/* prevent another thread from changing buffer sizes */  in ring_buffer_resize()
2853 	mutex_lock(&buffer->mutex);  in ring_buffer_resize()
2854 	atomic_inc(&buffer->resizing);  in ring_buffer_resize()
2859 		 * manipulating the ring buffer and is expecting a sane state while  in ring_buffer_resize()
2862 		for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_resize()
2863 			cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
2871 		for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_resize()
2872 			cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
2900 		 * since we can change their buffer sizes without any race.  in ring_buffer_resize()
2902 		for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_resize()
2903 			cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
2926 		for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_resize()
2927 			cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
2938 		cpu_buffer = buffer->buffers[cpu_id];  in ring_buffer_resize()
2945 		 * manipulating the ring buffer and is expecting a sane state while  in ring_buffer_resize()
2989 	 * The ring buffer resize can happen with the ring buffer  in ring_buffer_resize()
2991 	 * as possible. But if the buffer is disabled, we do not need  in ring_buffer_resize()
2993 	 * that the buffer is not corrupt.  in ring_buffer_resize()
2995 	if (atomic_read(&buffer->record_disabled)) {  in ring_buffer_resize()
2996 		atomic_inc(&buffer->record_disabled);  in ring_buffer_resize()
2998 		 * Even though the buffer was disabled, we must make sure  in ring_buffer_resize()
3004 		for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_resize()
3005 			cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
3008 		atomic_dec(&buffer->record_disabled);  in ring_buffer_resize()
3011 	atomic_dec(&buffer->resizing);  in ring_buffer_resize()
3012 	mutex_unlock(&buffer->mutex);  in ring_buffer_resize()
3016 	for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_resize()
3019 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
3032 	atomic_dec(&buffer->resizing);  in ring_buffer_resize()
3033 	mutex_unlock(&buffer->mutex);  in ring_buffer_resize()
3038 void ring_buffer_change_overwrite(struct trace_buffer *buffer, int val)  in ring_buffer_change_overwrite()  argument
3040 	mutex_lock(&buffer->mutex);  in ring_buffer_change_overwrite()
3042 		buffer->flags |= RB_FL_OVERWRITE;  in ring_buffer_change_overwrite()
3044 		buffer->flags &= ~RB_FL_OVERWRITE;  in ring_buffer_change_overwrite()
3045 	mutex_unlock(&buffer->mutex);  in ring_buffer_change_overwrite()
3137 	addr &= (PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1;  in rb_event_index()
3162 /* Return the index into the sub-buffers for a given sub-buffer */
3376 	unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size);  in rb_reset_tail()
3445 	/* Set write to end of buffer */  in rb_reset_tail()
3461 	struct trace_buffer *buffer = cpu_buffer->buffer;  in rb_move_tail()  local
3471 	 * it all the way around the buffer, bail, and warn  in rb_move_tail()
3484 	 * page with the buffer head.  in rb_move_tail()
3490 	 * the buffer, unless the commit page is still on the  in rb_move_tail()
3504 			if (!(buffer->flags & RB_FL_OVERWRITE)) {  in rb_move_tail()
3520 			 * page. We could have a small buffer, and  in rb_move_tail()
3521 			 * have filled up the buffer with events  in rb_move_tail()
3619 		 * is added to the buffer, it will lose those bits.  in rb_add_timestamp()
3635 				pr_warn("Ring buffer clock went backwards: %llu -> %llu\n",  in rb_add_timestamp()
3650  * @cpu_buffer: The per cpu buffer of the @event
3655  * is the actual size that is written to the ring buffer,
3730 	addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1);  in rb_try_to_discard()
3904 rb_wakeups(struct trace_buffer *buffer, struct ring_buffer_per_cpu *cpu_buffer)  in rb_wakeups()  argument
3906 	if (buffer->irq_work.waiters_pending) {  in rb_wakeups()
3907 		buffer->irq_work.waiters_pending = false;  in rb_wakeups()
3909 		irq_work_queue(&buffer->irq_work.work);  in rb_wakeups()
3929 	if (!full_hit(buffer, cpu_buffer->cpu, cpu_buffer->shortest_full))  in rb_wakeups()
3994  * if an interrupt comes in while NORMAL bit is set and the ring buffer
4046  * @buffer: The ring buffer to modify
4048  * The ring buffer has a safety mechanism to prevent recursion.
4057 void ring_buffer_nest_start(struct trace_buffer *buffer)  in ring_buffer_nest_start()  argument
4065 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_nest_start()
4072  * @buffer: The ring buffer to modify
4077 void ring_buffer_nest_end(struct trace_buffer *buffer)  in ring_buffer_nest_end()  argument
4084 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_nest_end()
4092  * @buffer: The buffer to commit to
4094  * This commits the data to the ring buffer, and releases any locks held.
4098 int ring_buffer_unlock_commit(struct trace_buffer *buffer)  in ring_buffer_unlock_commit()  argument
4103 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_unlock_commit()
4107 	rb_wakeups(buffer, cpu_buffer);  in ring_buffer_unlock_commit()
4264  * the buffer page.
4342 	info->ts = rb_time_stamp(cpu_buffer->buffer);  in __rb_reserve_next()
4353 			/* Use the sub-buffer timestamp */  in __rb_reserve_next()
4376 	/* See if we shot pass the end of this buffer page */  in __rb_reserve_next()
4377 	if (unlikely(write > cpu_buffer->buffer->subbuf_size)) {  in __rb_reserve_next()
4412 		ts = rb_time_stamp(cpu_buffer->buffer);  in __rb_reserve_next()
4449 	/* We reserved something on the buffer */  in __rb_reserve_next()
4470 rb_reserve_next_event(struct trace_buffer *buffer,  in rb_reserve_next_event()  argument
4480 	 * ring buffer does cmpxchg as well as atomic64 operations  in rb_reserve_next_event()
4495 	 * Due to the ability to swap a cpu buffer from a buffer  in rb_reserve_next_event()
4501 	if (unlikely(READ_ONCE(cpu_buffer->buffer) != buffer)) {  in rb_reserve_next_event()
4510 	if (ring_buffer_time_stamp_abs(cpu_buffer->buffer)) {  in rb_reserve_next_event()
4513 		if (info.length > cpu_buffer->buffer->max_data_size)  in rb_reserve_next_event()
4551  * ring_buffer_lock_reserve - reserve a part of the buffer
4552  * @buffer: the ring buffer to reserve from
4555  * Returns a reserved event on the ring buffer to copy directly to.
4566 ring_buffer_lock_reserve(struct trace_buffer *buffer, unsigned long length)  in ring_buffer_lock_reserve()  argument
4575 	if (unlikely(atomic_read(&buffer->record_disabled)))  in ring_buffer_lock_reserve()
4580 	if (unlikely(!cpumask_test_cpu(cpu, buffer->cpumask)))  in ring_buffer_lock_reserve()
4583 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_lock_reserve()
4588 	if (unlikely(length > buffer->max_data_size))  in ring_buffer_lock_reserve()
4594 	event = rb_reserve_next_event(buffer, cpu_buffer, length);  in ring_buffer_lock_reserve()
4622 	addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1);  in rb_decrement_entry()
4644 	/* commit not part of this buffer?? */  in rb_decrement_entry()
4650  * @buffer: the ring buffer
4653  * Sometimes an event that is in the ring buffer needs to be ignored.
4654  * This function lets the user discard an event in the ring buffer
4658  * committed. It will try to free the event from the ring buffer
4667 void ring_buffer_discard_commit(struct trace_buffer *buffer,  in ring_buffer_discard_commit()  argument
4677 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_discard_commit()
4684 	RB_WARN_ON(buffer, !local_read(&cpu_buffer->committing));  in ring_buffer_discard_commit()
4701  * ring_buffer_write - write data to the buffer without reserving
4702  * @buffer: The ring buffer to write to.
4704  * @data: The data to write to the buffer.
4707  * one function. If you already have the data to write to the buffer, it
4713 int ring_buffer_write(struct trace_buffer *buffer,  in ring_buffer_write()  argument
4725 	if (atomic_read(&buffer->record_disabled))  in ring_buffer_write()
4730 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_write()
4733 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_write()
4738 	if (length > buffer->max_data_size)  in ring_buffer_write()
4744 	event = rb_reserve_next_event(buffer, cpu_buffer, length);  in ring_buffer_write()
4754 	rb_wakeups(buffer, cpu_buffer);  in ring_buffer_write()
4769  * The total entries in the ring buffer is the running counter
4770  * of entries entered into the ring buffer, minus the sum of
4771  * the entries read from the ring buffer and the number of
4787  * ring_buffer_record_disable - stop all writes into the buffer
4788  * @buffer: The ring buffer to stop writes to.
4790  * This prevents all writes to the buffer. Any attempt to write
4791  * to the buffer after this will fail and return NULL.
4795 void ring_buffer_record_disable(struct trace_buffer *buffer)  in ring_buffer_record_disable()  argument
4797 	atomic_inc(&buffer->record_disabled);  in ring_buffer_record_disable()
4802  * ring_buffer_record_enable - enable writes to the buffer
4803  * @buffer: The ring buffer to enable writes
4808 void ring_buffer_record_enable(struct trace_buffer *buffer)  in ring_buffer_record_enable()  argument
4810 	atomic_dec(&buffer->record_disabled);  in ring_buffer_record_enable()
4815  * ring_buffer_record_off - stop all writes into the buffer
4816  * @buffer: The ring buffer to stop writes to.
4818  * This prevents all writes to the buffer. Any attempt to write
4819  * to the buffer after this will fail and return NULL.
4825 void ring_buffer_record_off(struct trace_buffer *buffer)  in ring_buffer_record_off()  argument
4830 	rd = atomic_read(&buffer->record_disabled);  in ring_buffer_record_off()
4833 	} while (!atomic_try_cmpxchg(&buffer->record_disabled, &rd, new_rd));  in ring_buffer_record_off()
4838  * ring_buffer_record_on - restart writes into the buffer
4839  * @buffer: The ring buffer to start writes to.
4841  * This enables all writes to the buffer that was disabled by
4848 void ring_buffer_record_on(struct trace_buffer *buffer)  in ring_buffer_record_on()  argument
4853 	rd = atomic_read(&buffer->record_disabled);  in ring_buffer_record_on()
4856 	} while (!atomic_try_cmpxchg(&buffer->record_disabled, &rd, new_rd));  in ring_buffer_record_on()
4861  * ring_buffer_record_is_on - return true if the ring buffer can write
4862  * @buffer: The ring buffer to see if write is enabled
4864  * Returns true if the ring buffer is in a state that it accepts writes.
4866 bool ring_buffer_record_is_on(struct trace_buffer *buffer)  in ring_buffer_record_is_on()  argument
4868 	return !atomic_read(&buffer->record_disabled);  in ring_buffer_record_is_on()
4872  * ring_buffer_record_is_set_on - return true if the ring buffer is set writable
4873  * @buffer: The ring buffer to see if write is set enabled
4875  * Returns true if the ring buffer is set writable by ring_buffer_record_on().
4878  * It may return true when the ring buffer has been disabled by
4880  * the ring buffer.
4882 bool ring_buffer_record_is_set_on(struct trace_buffer *buffer)  in ring_buffer_record_is_set_on()  argument
4884 	return !(atomic_read(&buffer->record_disabled) & RB_BUFFER_OFF);  in ring_buffer_record_is_set_on()
4889  * @buffer: The ring buffer to stop writes to.
4890  * @cpu: The CPU buffer to stop
4892  * This prevents all writes to the buffer. Any attempt to write
4893  * to the buffer after this will fail and return NULL.
4897 void ring_buffer_record_disable_cpu(struct trace_buffer *buffer, int cpu)  in ring_buffer_record_disable_cpu()  argument
4901 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_record_disable_cpu()
4904 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_record_disable_cpu()
4910  * ring_buffer_record_enable_cpu - enable writes to the buffer
4911  * @buffer: The ring buffer to enable writes
4917 void ring_buffer_record_enable_cpu(struct trace_buffer *buffer, int cpu)  in ring_buffer_record_enable_cpu()  argument
4921 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_record_enable_cpu()
4924 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_record_enable_cpu()
4930  * ring_buffer_oldest_event_ts - get the oldest event timestamp from the buffer
4931  * @buffer: The ring buffer
4932  * @cpu: The per CPU buffer to read from.
4934 u64 ring_buffer_oldest_event_ts(struct trace_buffer *buffer, int cpu)  in ring_buffer_oldest_event_ts()  argument
4941 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_oldest_event_ts()
4944 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_oldest_event_ts()
4963  * ring_buffer_bytes_cpu - get the number of bytes unconsumed in a cpu buffer
4964  * @buffer: The ring buffer
4965  * @cpu: The per CPU buffer to read from.
4967 unsigned long ring_buffer_bytes_cpu(struct trace_buffer *buffer, int cpu)  in ring_buffer_bytes_cpu()  argument
4972 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_bytes_cpu()
4975 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_bytes_cpu()
4983  * ring_buffer_entries_cpu - get the number of entries in a cpu buffer
4984  * @buffer: The ring buffer
4985  * @cpu: The per CPU buffer to get the entries from.
4987 unsigned long ring_buffer_entries_cpu(struct trace_buffer *buffer, int cpu)  in ring_buffer_entries_cpu()  argument
4991 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_entries_cpu()
4994 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_entries_cpu()
5002  * buffer wrapping around (only if RB_FL_OVERWRITE is on).
5003  * @buffer: The ring buffer
5004  * @cpu: The per CPU buffer to get the number of overruns from
5006 unsigned long ring_buffer_overrun_cpu(struct trace_buffer *buffer, int cpu)  in ring_buffer_overrun_cpu()  argument
5011 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_overrun_cpu()
5014 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_overrun_cpu()
5023  * commits failing due to the buffer wrapping around while there are uncommitted
5025  * @buffer: The ring buffer
5026  * @cpu: The per CPU buffer to get the number of overruns from
5029 ring_buffer_commit_overrun_cpu(struct trace_buffer *buffer, int cpu)  in ring_buffer_commit_overrun_cpu()  argument
5034 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_commit_overrun_cpu()
5037 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_commit_overrun_cpu()
5046  * the ring buffer filling up (only if RB_FL_OVERWRITE is off).
5047  * @buffer: The ring buffer
5048  * @cpu: The per CPU buffer to get the number of overruns from
5051 ring_buffer_dropped_events_cpu(struct trace_buffer *buffer, int cpu)  in ring_buffer_dropped_events_cpu()  argument
5056 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_dropped_events_cpu()
5059 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_dropped_events_cpu()
5068  * @buffer: The ring buffer
5069  * @cpu: The per CPU buffer to get the number of events read
5072 ring_buffer_read_events_cpu(struct trace_buffer *buffer, int cpu)  in ring_buffer_read_events_cpu()  argument
5076 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_read_events_cpu()
5079 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_read_events_cpu()
5085  * ring_buffer_entries - get the number of entries in a buffer
5086  * @buffer: The ring buffer
5088  * Returns the total number of entries in the ring buffer
5091 unsigned long ring_buffer_entries(struct trace_buffer *buffer)  in ring_buffer_entries()  argument
5097 	/* if you care about this being correct, lock the buffer */  in ring_buffer_entries()
5098 	for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_entries()
5099 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_entries()
5108  * ring_buffer_overruns - get the number of overruns in buffer
5109  * @buffer: The ring buffer
5111  * Returns the total number of overruns in the ring buffer
5114 unsigned long ring_buffer_overruns(struct trace_buffer *buffer)  in ring_buffer_overruns()  argument
5120 	/* if you care about this being correct, lock the buffer */  in ring_buffer_overruns()
5121 	for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_overruns()
5122 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_overruns()
5287 	unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size);  in rb_get_reader_page()
5324 	/* Don't bother swapping if the ring buffer is empty */  in rb_get_reader_page()
5347 	 * cpu_buffer->pages just needs to point to the buffer, it  in rb_get_reader_page()
5348 	 *  has no specific buffer page to point to. Lets move it out  in rb_get_reader_page()
5463 	/* This function should not be called when buffer is empty */  in rb_advance_reader()
5495 	 * Check if we are at the end of the buffer.  in rb_advance_iter()
5562 			ring_buffer_normalize_time_stamp(cpu_buffer->buffer,  in rb_buffer_peek()
5572 			ring_buffer_normalize_time_stamp(cpu_buffer->buffer,  in rb_buffer_peek()
5590 	struct trace_buffer *buffer;  in rb_iter_peek()  local
5599 	buffer = cpu_buffer->buffer;  in rb_iter_peek()
5602 	 * Check if someone performed a consuming read to the buffer  in rb_iter_peek()
5603 	 * or removed some pages from the buffer. In these cases,  in rb_iter_peek()
5619 	 * the ring buffer with an active write as the consumer is.  in rb_iter_peek()
5655 			ring_buffer_normalize_time_stamp(cpu_buffer->buffer,  in rb_iter_peek()
5665 			ring_buffer_normalize_time_stamp(buffer,  in rb_iter_peek()
5686 	 * If an NMI die dumps out the content of the ring buffer  in rb_reader_lock()
5688 	 * preempted a task that holds the ring buffer locks. If  in rb_reader_lock()
5690 	 * to do the read, but this can corrupt the ring buffer,  in rb_reader_lock()
5697 	/* Continue without locking, but disable the ring buffer */  in rb_reader_lock()
5711  * @buffer: The ring buffer to read
5720 ring_buffer_peek(struct trace_buffer *buffer, int cpu, u64 *ts,  in ring_buffer_peek()  argument
5723 	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];  in ring_buffer_peek()
5728 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_peek()
5747  * @iter: The ring buffer iterator
5762  * @iter: The ring buffer iterator
5788  * @buffer: The ring buffer to get the next event from
5789  * @cpu: the cpu to read the buffer from
5793  * Returns the next event in the ring buffer, and that event is consumed.
5795  * and eventually empty the ring buffer if the producer is slower.
5798 ring_buffer_consume(struct trace_buffer *buffer, int cpu, u64 *ts,  in ring_buffer_consume()  argument
5810 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_consume()
5813 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_consume()
5837  * ring_buffer_read_prepare - Prepare for a non consuming read of the buffer
5838  * @buffer: The ring buffer to read from
5839  * @cpu: The cpu buffer to iterate over
5843  * through the buffer.  Memory is allocated, buffer resizing
5854 ring_buffer_read_prepare(struct trace_buffer *buffer, int cpu, gfp_t flags)  in ring_buffer_read_prepare()  argument
5859 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_read_prepare()
5867 	iter->event_size = buffer->subbuf_size;  in ring_buffer_read_prepare()
5874 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_read_prepare()
5899  * ring_buffer_read_start - start a non consuming read of the buffer
5902  * This finalizes the startup of an iteration through the buffer.
5929  * ring_buffer_read_finish - finish reading the iterator of the buffer
5932  * This re-enables resizing of the buffer, and frees the iterator.
5939 	/* Use this opportunity to check the integrity of the ring buffer. */  in ring_buffer_read_finish()
5950  * @iter: The ring buffer iterator
5969  * ring_buffer_size - return the size of the ring buffer (in bytes)
5970  * @buffer: The ring buffer.
5971  * @cpu: The CPU to get ring buffer size from.
5973 unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu)  in ring_buffer_size()  argument
5975 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_size()
5978 	return buffer->subbuf_size * buffer->buffers[cpu]->nr_pages;  in ring_buffer_size()
5984  * @buffer: The ring buffer.
5988 unsigned long ring_buffer_max_event_size(struct trace_buffer *buffer)  in ring_buffer_max_event_size()  argument
5991 	if (ring_buffer_time_stamp_abs(buffer))  in ring_buffer_max_event_size()
5992 		return buffer->max_data_size - RB_LEN_TIME_EXTEND;  in ring_buffer_max_event_size()
5993 	return buffer->max_data_size;  in ring_buffer_max_event_size()
6080 /* Must have disabled the cpu buffer then done a synchronize_rcu */
6101  * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
6102  * @buffer: The ring buffer to reset a per cpu buffer of
6103  * @cpu: The CPU buffer to be reset
6105 void ring_buffer_reset_cpu(struct trace_buffer *buffer, int cpu)  in ring_buffer_reset_cpu()  argument
6107 	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];  in ring_buffer_reset_cpu()
6109 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_reset_cpu()
6112 	/* prevent another thread from changing buffer sizes */  in ring_buffer_reset_cpu()
6113 	mutex_lock(&buffer->mutex);  in ring_buffer_reset_cpu()
6126 	mutex_unlock(&buffer->mutex);  in ring_buffer_reset_cpu()
6134  * ring_buffer_reset_online_cpus - reset a ring buffer per CPU buffer
6135  * @buffer: The ring buffer to reset a per cpu buffer of
6137 void ring_buffer_reset_online_cpus(struct trace_buffer *buffer)  in ring_buffer_reset_online_cpus()  argument
6142 	/* prevent another thread from changing buffer sizes */  in ring_buffer_reset_online_cpus()
6143 	mutex_lock(&buffer->mutex);  in ring_buffer_reset_online_cpus()
6145 	for_each_online_buffer_cpu(buffer, cpu) {  in ring_buffer_reset_online_cpus()
6146 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_reset_online_cpus()
6155 	for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_reset_online_cpus()
6156 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_reset_online_cpus()
6171 	mutex_unlock(&buffer->mutex);  in ring_buffer_reset_online_cpus()
6175  * ring_buffer_reset - reset a ring buffer
6176  * @buffer: The ring buffer to reset all cpu buffers
6178 void ring_buffer_reset(struct trace_buffer *buffer)  in ring_buffer_reset()  argument
6183 	/* prevent another thread from changing buffer sizes */  in ring_buffer_reset()
6184 	mutex_lock(&buffer->mutex);  in ring_buffer_reset()
6186 	for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_reset()
6187 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_reset()
6196 	for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_reset()
6197 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_reset()
6205 	mutex_unlock(&buffer->mutex);  in ring_buffer_reset()
6210  * ring_buffer_empty - is the ring buffer empty?
6211  * @buffer: The ring buffer to test
6213 bool ring_buffer_empty(struct trace_buffer *buffer)  in ring_buffer_empty()  argument
6221 	/* yes this is racy, but if you don't like the race, lock the buffer */  in ring_buffer_empty()
6222 	for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_empty()
6223 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_empty()
6239  * ring_buffer_empty_cpu - is a cpu buffer of a ring buffer empty?
6240  * @buffer: The ring buffer
6241  * @cpu: The CPU buffer to test
6243 bool ring_buffer_empty_cpu(struct trace_buffer *buffer, int cpu)  in ring_buffer_empty_cpu()  argument
6250 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_empty_cpu()
6253 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_empty_cpu()
6266  * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers
6267  * @buffer_a: One buffer to swap with
6268  * @buffer_b: The other buffer to swap with
6272  * of a CPU buffer and has another back up buffer lying around.
6273  * it is expected that the tracer handles the cpu buffer not being
6334 	 * it will mess the state of the cpu buffer.  in ring_buffer_swap_cpu()
6344 	cpu_buffer_b->buffer = buffer_a;  in ring_buffer_swap_cpu()
6345 	cpu_buffer_a->buffer = buffer_b;  in ring_buffer_swap_cpu()
6359  * ring_buffer_alloc_read_page - allocate a page to read from buffer
6360  * @buffer: the buffer to allocate for.
6361  * @cpu: the cpu buffer to allocate.
6364  * When reading a full page from the ring buffer, these functions
6367  * needs to get pages from the ring buffer, it passes the result
6369  * the page that was allocated, with the read page of the buffer.
6375 ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu)  in ring_buffer_alloc_read_page()  argument
6382 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_alloc_read_page()
6389 	bpage->order = buffer->subbuf_order;  in ring_buffer_alloc_read_page()
6390 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_alloc_read_page()
6407 				cpu_buffer->buffer->subbuf_order);  in ring_buffer_alloc_read_page()
6424  * @buffer: the buffer the page was allocate for
6425  * @cpu: the cpu buffer the page came from
6430 void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu,  in ring_buffer_free_read_page()  argument
6438 	if (!buffer || !buffer->buffers || !buffer->buffers[cpu])  in ring_buffer_free_read_page()
6441 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_free_read_page()
6445 	 * is different from the subbuffer order of the buffer -  in ring_buffer_free_read_page()
6448 	if (page_ref_count(page) > 1 || data_page->order != buffer->subbuf_order)  in ring_buffer_free_read_page()
6469  * ring_buffer_read_page - extract a page from the ring buffer
6470  * @buffer: buffer to extract from
6473  * @cpu: the cpu of the buffer to extract
6476  * This function will pull out a page from the ring buffer and consume it.
6479  * to swap with a page in the ring buffer.
6482  *	rpage = ring_buffer_alloc_read_page(buffer, cpu);
6485  *	ret = ring_buffer_read_page(buffer, rpage, len, cpu, 0);
6488  *	ring_buffer_free_read_page(buffer, cpu, rpage);
6494  *  The ring buffer can be used anywhere in the kernel and can not
6495  *  blindly call wake_up. The layer that uses the ring buffer must be
6502 int ring_buffer_read_page(struct trace_buffer *buffer,  in ring_buffer_read_page()  argument
6506 	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];  in ring_buffer_read_page()
6517 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_read_page()
6531 	if (data_page->order != buffer->subbuf_order)  in ring_buffer_read_page()
6556 	 * we must copy the data from the page to the buffer.  in ring_buffer_read_page()
6655 		if (buffer->subbuf_size - commit >= sizeof(missed_events)) {  in ring_buffer_read_page()
6667 	if (commit < buffer->subbuf_size)  in ring_buffer_read_page()
6668 		memset(&bpage->data[commit], 0, buffer->subbuf_size - commit);  in ring_buffer_read_page()
6691  * ring_buffer_subbuf_size_get - get size of the sub buffer.
6692  * @buffer: the buffer to get the sub buffer size from
6694  * Returns size of the sub buffer, in bytes.
6696 int ring_buffer_subbuf_size_get(struct trace_buffer *buffer)  in ring_buffer_subbuf_size_get()  argument
6698 	return buffer->subbuf_size + BUF_PAGE_HDR_SIZE;  in ring_buffer_subbuf_size_get()
6703  * ring_buffer_subbuf_order_get - get order of system sub pages in one buffer page.
6704  * @buffer: The ring_buffer to get the system sub page order from
6706  * By default, one ring buffer sub page equals to one system page. This parameter
6707  * is configurable, per ring buffer. The size of the ring buffer sub page can be
6710  * Returns the order of buffer sub page size, in system pages:
6711  * 0 means the sub buffer size is 1 system page and so forth.
6714 int ring_buffer_subbuf_order_get(struct trace_buffer *buffer)  in ring_buffer_subbuf_order_get()  argument
6716 	if (!buffer)  in ring_buffer_subbuf_order_get()
6719 	return buffer->subbuf_order;  in ring_buffer_subbuf_order_get()
6724  * ring_buffer_subbuf_order_set - set the size of ring buffer sub page.
6725  * @buffer: The ring_buffer to set the new page size.
6726  * @order: Order of the system pages in one sub buffer page
6728  * By default, one ring buffer pages equals to one system page. This API can be
6729  * used to set new size of the ring buffer page. The size must be order of
6731  * system pages that are allocated for one ring buffer page:
6739 int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order)  in ring_buffer_subbuf_order_set()  argument
6749 	if (!buffer || order < 0)  in ring_buffer_subbuf_order_set()
6752 	if (buffer->subbuf_order == order)  in ring_buffer_subbuf_order_set()
6763 	old_order = buffer->subbuf_order;  in ring_buffer_subbuf_order_set()
6764 	old_size = buffer->subbuf_size;  in ring_buffer_subbuf_order_set()
6766 	/* prevent another thread from changing buffer sizes */  in ring_buffer_subbuf_order_set()
6767 	mutex_lock(&buffer->mutex);  in ring_buffer_subbuf_order_set()
6768 	atomic_inc(&buffer->record_disabled);  in ring_buffer_subbuf_order_set()
6773 	buffer->subbuf_order = order;  in ring_buffer_subbuf_order_set()
6774 	buffer->subbuf_size = psize - BUF_PAGE_HDR_SIZE;  in ring_buffer_subbuf_order_set()
6777 	for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_subbuf_order_set()
6779 		if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_subbuf_order_set()
6782 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_subbuf_order_set()
6790 		nr_pages = old_size * buffer->buffers[cpu]->nr_pages;  in ring_buffer_subbuf_order_set()
6791 		nr_pages = DIV_ROUND_UP(nr_pages, buffer->subbuf_size);  in ring_buffer_subbuf_order_set()
6802 		/* Allocate the new size buffer */  in ring_buffer_subbuf_order_set()
6812 	for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_subbuf_order_set()
6817 		if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_subbuf_order_set()
6820 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_subbuf_order_set()
6871 	atomic_dec(&buffer->record_disabled);  in ring_buffer_subbuf_order_set()
6872 	mutex_unlock(&buffer->mutex);  in ring_buffer_subbuf_order_set()
6877 	buffer->subbuf_order = old_order;  in ring_buffer_subbuf_order_set()
6878 	buffer->subbuf_size = old_size;  in ring_buffer_subbuf_order_set()
6880 	atomic_dec(&buffer->record_disabled);  in ring_buffer_subbuf_order_set()
6881 	mutex_unlock(&buffer->mutex);  in ring_buffer_subbuf_order_set()
6883 	for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_subbuf_order_set()
6884 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_subbuf_order_set()
6951 	meta->subbuf_size = cpu_buffer->buffer->subbuf_size + BUF_PAGE_HDR_SIZE;  in rb_setup_ids_meta_page()
6958 rb_get_mapped_buffer(struct trace_buffer *buffer, int cpu)  in rb_get_mapped_buffer()  argument
6962 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in rb_get_mapped_buffer()
6965 	cpu_buffer = buffer->buffers[cpu];  in rb_get_mapped_buffer()
7003 	mutex_lock(&cpu_buffer->buffer->mutex);  in __rb_inc_dec_mapped()
7015 	mutex_unlock(&cpu_buffer->buffer->mutex);  in __rb_inc_dec_mapped()
7046 	subbuf_order = cpu_buffer->buffer->subbuf_order;  in __rb_map_vma()
7136 int ring_buffer_map(struct trace_buffer *buffer, int cpu,  in ring_buffer_map()  argument
7143 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_map()
7146 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_map()
7158 	/* prevent another thread from changing buffer/sub-buffer sizes */  in ring_buffer_map()
7159 	mutex_lock(&buffer->mutex);  in ring_buffer_map()
7199 	mutex_unlock(&buffer->mutex);  in ring_buffer_map()
7205 int ring_buffer_unmap(struct trace_buffer *buffer, int cpu)  in ring_buffer_unmap()  argument
7211 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_unmap()
7214 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_unmap()
7226 	mutex_lock(&buffer->mutex);  in ring_buffer_unmap()
7241 	mutex_unlock(&buffer->mutex);  in ring_buffer_unmap()
7249 int ring_buffer_map_get_reader(struct trace_buffer *buffer, int cpu)  in ring_buffer_map_get_reader()  argument
7257 	cpu_buffer = rb_get_mapped_buffer(buffer, cpu);  in ring_buffer_map_get_reader()
7303 			if (buffer->subbuf_size - commit >= sizeof(missed_events)) {  in ring_buffer_map_get_reader()
7325 				buffer->subbuf_size + BUF_PAGE_HDR_SIZE);  in ring_buffer_map_get_reader()
7337  * If we were to free the buffer, then the user would lose any trace that was in
7338  * the buffer.
7342 	struct trace_buffer *buffer;  in trace_rb_cpu_prepare()  local
7347 	buffer = container_of(node, struct trace_buffer, node);  in trace_rb_cpu_prepare()
7348 	if (cpumask_test_cpu(cpu, buffer->cpumask))  in trace_rb_cpu_prepare()
7354 	for_each_buffer_cpu(buffer, cpu_i) {  in trace_rb_cpu_prepare()
7357 			nr_pages = buffer->buffers[cpu_i]->nr_pages;  in trace_rb_cpu_prepare()
7358 		if (nr_pages != buffer->buffers[cpu_i]->nr_pages) {  in trace_rb_cpu_prepare()
7366 	buffer->buffers[cpu] =  in trace_rb_cpu_prepare()
7367 		rb_allocate_cpu_buffer(buffer, nr_pages, cpu);  in trace_rb_cpu_prepare()
7368 	if (!buffer->buffers[cpu]) {  in trace_rb_cpu_prepare()
7369 		WARN(1, "failed to allocate ring buffer on CPU %u\n",  in trace_rb_cpu_prepare()
7374 	cpumask_set_cpu(cpu, buffer->cpumask);  in trace_rb_cpu_prepare()
7380  * This is a basic integrity check of the ring buffer.
7383  * writing to the per cpu ring buffer various sizes of data.
7387  * IPIs to the other CPUs to also write into the ring buffer.
7388  * this is to test the nesting ability of the buffer.
7391  * ring buffer should happen that's not expected, a big warning
7397 	struct trace_buffer *buffer;  member
7450 	/* read rb_test_started before checking buffer enabled */  in rb_write_something()
7453 	event = ring_buffer_lock_reserve(data->buffer, len);  in rb_write_something()
7467 	if (RB_WARN_ON(data->buffer, event_len < len))  in rb_write_something()
7493 	ring_buffer_unlock_commit(data->buffer);  in rb_write_something()
7539 	struct trace_buffer *buffer;  in test_ringbuffer()  local
7544 		pr_warn("Lockdown is enabled, skipping ring buffer tests\n");  in test_ringbuffer()
7548 	pr_info("Running ring buffer tests...\n");  in test_ringbuffer()
7550 	buffer = ring_buffer_alloc(RB_TEST_BUFFER_SIZE, RB_FL_OVERWRITE);  in test_ringbuffer()
7551 	if (WARN_ON(!buffer))  in test_ringbuffer()
7554 	/* Disable buffer so that threads can't write to it yet */  in test_ringbuffer()
7555 	ring_buffer_record_off(buffer);  in test_ringbuffer()
7558 		rb_data[cpu].buffer = buffer;  in test_ringbuffer()
7578 	ring_buffer_record_on(buffer);  in test_ringbuffer()
7580 	 * Show buffer is enabled before setting rb_test_started.  in test_ringbuffer()
7583 	 * buffer gets enabled, there will always be some kind of  in test_ringbuffer()
7586 	 * the threads see that the buffer is active.  in test_ringbuffer()
7604 		ring_buffer_free(buffer);  in test_ringbuffer()
7644 		if (RB_WARN_ON(buffer, total_dropped))  in test_ringbuffer()
7649 		while ((event = ring_buffer_consume(buffer, cpu, NULL, &lost))) {  in test_ringbuffer()
7656 				pr_info("buffer had: %.*s\n", item->size, item->str);  in test_ringbuffer()
7658 				RB_WARN_ON(buffer, 1);  in test_ringbuffer()
7678 			if (RB_WARN_ON(buffer, total_len != total_alloc ||  in test_ringbuffer()
7682 		if (RB_WARN_ON(buffer, total_lost + total_read != total_events))  in test_ringbuffer()
7688 		pr_info("Ring buffer PASSED!\n");  in test_ringbuffer()
7690 	ring_buffer_free(buffer);  in test_ringbuffer()