Lines Matching full:buffer
3 * Generic ring buffer
40 * The "absolute" timestamp in the buffer is only 59 bits.
68 * The ring buffer header is special. We must manually up keep it.
90 * The ring buffer is made up of a list of pages. A separate list of pages is
91 * allocated for each CPU. A writer may only write to a buffer that is
93 * from any per cpu buffer.
95 * The reader is special. For each per cpu buffer, the reader has its own
97 * page is swapped with another page in the ring buffer.
101 * again (as long as it is out of the ring buffer).
106 * |reader| RING BUFFER
117 * |reader| RING BUFFER
128 * |reader| RING BUFFER
139 * |buffer| RING BUFFER
151 * and swap that into the ring buffer.
289 /* inline for ring buffer fast paths */
313 #define for_each_buffer_cpu(buffer, cpu) \ argument
314 for_each_cpu(cpu, buffer->cpumask)
316 #define for_each_online_buffer_cpu(buffer, cpu) \ argument
317 for_each_cpu_and(cpu, buffer->cpumask, cpu_online_mask)
344 unsigned char data[] RB_ALIGN_DATA; /* data of buffer page */
353 * Note, the buffer_page list must be first. The buffer pages
354 * are allocated in cache lines, which means that each buffer
357 * add flags in the list struct pointers, to make the ring buffer
361 struct list_head list; /* list of buffer pages */
373 * The buffer page counters, write and entries, must be reset
406 * For best performance, allocate cpu buffer data cache line sized
475 * ABSOLUTE - the buffer requests all events to have absolute time stamps
511 * head_page == tail_page && head == tail then buffer is empty.
517 struct trace_buffer *buffer; member
562 /* ring buffer pages to update, > 0 to add, < 0 to remove */
615 int ring_buffer_print_page_header(struct trace_buffer *buffer, struct trace_seq *s) in ring_buffer_print_page_header() argument
639 (unsigned int)buffer->subbuf_size, in ring_buffer_print_page_header()
657 * is on the buffer that it passed in.
714 static inline u64 rb_time_stamp(struct trace_buffer *buffer);
718 * @buffer: The buffer that the event is on
722 * committed to the ring buffer. And must be called from the same
729 * the max nesting, then the write_stamp of the buffer is returned,
733 u64 ring_buffer_event_time_stamp(struct trace_buffer *buffer, in ring_buffer_event_time_stamp() argument
736 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[smp_processor_id()]; in ring_buffer_event_time_stamp()
765 * ring_buffer_nr_dirty_pages - get the number of used pages in the ring buffer
766 * @buffer: The ring_buffer to get the number of pages from
769 * Returns the number of pages that have content in the ring buffer.
771 size_t ring_buffer_nr_dirty_pages(struct trace_buffer *buffer, int cpu) in ring_buffer_nr_dirty_pages() argument
777 read = local_read(&buffer->buffers[cpu]->pages_read); in ring_buffer_nr_dirty_pages()
778 lost = local_read(&buffer->buffers[cpu]->pages_lost); in ring_buffer_nr_dirty_pages()
779 cnt = local_read(&buffer->buffers[cpu]->pages_touched); in ring_buffer_nr_dirty_pages()
795 static __always_inline bool full_hit(struct trace_buffer *buffer, int cpu, int full) in full_hit() argument
797 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; in full_hit()
806 * Add one as dirty will never equal nr_pages, as the sub-buffer in full_hit()
810 dirty = ring_buffer_nr_dirty_pages(buffer, cpu) + 1; in full_hit()
816 * rb_wake_up_waiters - wake up tasks waiting for ring buffer input
819 * ring buffer waiters queue.
848 * ring_buffer_wake_waiters - wake up any waiters on this ring buffer
849 * @buffer: The ring buffer to wake waiters on
850 * @cpu: The CPU buffer to wake waiters on
852 * In the case of a file that represents a ring buffer is closing,
855 void ring_buffer_wake_waiters(struct trace_buffer *buffer, int cpu) in ring_buffer_wake_waiters() argument
860 if (!buffer) in ring_buffer_wake_waiters()
866 for_each_buffer_cpu(buffer, cpu) in ring_buffer_wake_waiters()
867 ring_buffer_wake_waiters(buffer, cpu); in ring_buffer_wake_waiters()
869 rbwork = &buffer->irq_work; in ring_buffer_wake_waiters()
871 if (WARN_ON_ONCE(!buffer->buffers)) in ring_buffer_wake_waiters()
876 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_wake_waiters()
877 /* The CPU buffer may not have been initialized yet */ in ring_buffer_wake_waiters()
887 static bool rb_watermark_hit(struct trace_buffer *buffer, int cpu, int full) in rb_watermark_hit() argument
894 return !ring_buffer_empty(buffer); in rb_watermark_hit()
896 cpu_buffer = buffer->buffers[cpu]; in rb_watermark_hit()
898 if (!ring_buffer_empty_cpu(buffer, cpu)) { in rb_watermark_hit()
907 ret = !pagebusy && full_hit(buffer, cpu, full); in rb_watermark_hit()
919 rb_wait_cond(struct rb_irq_work *rbwork, struct trace_buffer *buffer, in rb_wait_cond() argument
922 if (rb_watermark_hit(buffer, cpu, full)) in rb_wait_cond()
935 * We don't clear it even if the buffer is no longer in rb_wait_cond()
974 * ring_buffer_wait - wait for input to the ring buffer
975 * @buffer: buffer to wait on
976 * @cpu: the cpu buffer to wait on
982 * as data is added to any of the @buffer's cpu buffers. Otherwise
983 * it will wait for data to be added to a specific cpu buffer.
985 int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full, in ring_buffer_wait() argument
996 * data in any cpu buffer, or a specific buffer, put the in ring_buffer_wait()
1000 rbwork = &buffer->irq_work; in ring_buffer_wait()
1004 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_wait()
1006 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_wait()
1024 rb_wait_cond(rbwork, buffer, cpu, full, cond, data)); in ring_buffer_wait()
1030 * ring_buffer_poll_wait - poll on buffer input
1031 * @buffer: buffer to wait on
1032 * @cpu: the cpu buffer to wait on
1038 * as data is added to any of the @buffer's cpu buffers. Otherwise
1039 * it will wait for data to be added to a specific cpu buffer.
1044 __poll_t ring_buffer_poll_wait(struct trace_buffer *buffer, int cpu, in ring_buffer_poll_wait() argument
1051 rbwork = &buffer->irq_work; in ring_buffer_poll_wait()
1054 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_poll_wait()
1057 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_poll_wait()
1064 if (rb_watermark_hit(buffer, cpu, full)) in ring_buffer_poll_wait()
1070 * compare the amount in the ring buffer to shortest_full. in ring_buffer_poll_wait()
1071 * If the amount in the ring buffer is greater than the in ring_buffer_poll_wait()
1088 * checking if the ring buffer is empty. Once the waiters_pending bit in ring_buffer_poll_wait()
1095 * the buffer goes from empty to having content. But as this race is in ring_buffer_poll_wait()
1101 if ((cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer)) || in ring_buffer_poll_wait()
1102 (cpu != RING_BUFFER_ALL_CPUS && !ring_buffer_empty_cpu(buffer, cpu))) in ring_buffer_poll_wait()
1107 /* buffer may be either ring_buffer or ring_buffer_per_cpu */
1115 atomic_inc(&__b->buffer->record_disabled); \
1126 static inline u64 rb_time_stamp(struct trace_buffer *buffer) in rb_time_stamp() argument
1131 if (IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) && likely(buffer->clock == trace_clock_local)) in rb_time_stamp()
1134 ts = buffer->clock(); in rb_time_stamp()
1140 u64 ring_buffer_time_stamp(struct trace_buffer *buffer) in ring_buffer_time_stamp() argument
1145 time = rb_time_stamp(buffer); in ring_buffer_time_stamp()
1152 void ring_buffer_normalize_time_stamp(struct trace_buffer *buffer, in ring_buffer_normalize_time_stamp() argument
1161 * Making the ring buffer lockless makes things tricky.
1166 * The reader page is always off the ring buffer, but when the
1168 * a new one from the buffer. The reader needs to take from
1218 * the reader page with a page in the buffer, but before it
1541 * rb_check_pages - integrity check of buffer pages
1542 * @cpu_buffer: CPU buffer with pages to test
1555 * Walk the linked list underpinning the ring buffer and validate all in rb_check_pages()
1562 * ring buffer size. Therefore, the code releases and re-acquires the in rb_check_pages()
1626 static void *rb_range_meta(struct trace_buffer *buffer, int nr_pages, int cpu) in rb_range_meta() argument
1628 int subbuf_size = buffer->subbuf_size + BUF_PAGE_HDR_SIZE; in rb_range_meta()
1634 bmeta = buffer->meta; in rb_range_meta()
1688 * Return a specific sub-buffer for a given @cpu defined by @idx.
1696 meta = rb_range_meta(cpu_buffer->buffer, 0, cpu_buffer->cpu); in rb_range_buffer()
1705 /* Map this buffer to the order that's in meta->buffers[] */ in rb_range_buffer()
1711 if (ptr + subbuf_size > cpu_buffer->buffer->range_addr_end) in rb_range_buffer()
1721 static bool rb_meta_init(struct trace_buffer *buffer, int scratch_size) in rb_meta_init() argument
1723 unsigned long ptr = buffer->range_addr_start; in rb_meta_init()
1729 buffer->meta = bmeta; in rb_meta_init()
1731 total_size = buffer->range_addr_end - buffer->range_addr_start; in rb_meta_init()
1736 /* The first buffer will start word size after the meta page */ in rb_meta_init()
1742 pr_info("Ring buffer boot meta mismatch of magic\n"); in rb_meta_init()
1747 pr_info("Ring buffer boot meta mismatch of struct size\n"); in rb_meta_init()
1752 pr_info("Ring buffer boot meta mismatch of total size\n"); in rb_meta_init()
1757 pr_info("Ring buffer boot meta mismatch of offset outside of total size\n"); in rb_meta_init()
1762 pr_info("Ring buffer boot meta mismatch of first buffer offset\n"); in rb_meta_init()
1781 * See if the existing memory contains valid ring buffer data.
1787 struct trace_buffer *buffer, int nr_pages, in rb_cpu_meta_valid() argument
1805 pr_info("Ring buffer boot meta [%d] head buffer out of range\n", cpu); in rb_cpu_meta_valid()
1811 pr_info("Ring buffer boot meta [%d] commit buffer out of range\n", cpu); in rb_cpu_meta_valid()
1823 pr_info("Ring buffer boot meta [%d] array out of range\n", cpu); in rb_cpu_meta_valid()
1828 pr_info("Ring buffer boot meta [%d] buffer invalid commit\n", cpu); in rb_cpu_meta_valid()
1833 pr_info("Ring buffer boot meta [%d] array has duplicates\n", cpu); in rb_cpu_meta_valid()
1931 pr_info("Ring buffer reader page is invalid\n"); in rb_meta_validate_events()
1976 pr_info("Ring buffer [%d] rewound %d pages\n", cpu_buffer->cpu, i); in rb_meta_validate_events()
1983 * If the ring buffer was rewound, then inject the reader page in rb_meta_validate_events()
2045 pr_info("Ring buffer meta [%d] invalid buffer page\n", in rb_meta_validate_events()
2050 /* If the buffer has content, update pages_touched */ in rb_meta_validate_events()
2063 pr_info("Ring buffer meta [%d] commit page not found\n", in rb_meta_validate_events()
2071 pr_info("Ring buffer meta [%d] is from previous boot!\n", cpu_buffer->cpu); in rb_meta_validate_events()
2090 static void rb_range_meta_init(struct trace_buffer *buffer, int nr_pages, int scratch_size) in rb_range_meta_init() argument
2104 if (rb_meta_init(buffer, scratch_size)) in rb_range_meta_init()
2110 meta = rb_range_meta(buffer, nr_pages, cpu); in rb_range_meta_init()
2112 if (valid && rb_cpu_meta_valid(meta, cpu, buffer, nr_pages, subbuf_mask)) { in rb_range_meta_init()
2123 next_meta = rb_range_meta(buffer, nr_pages, cpu + 1); in rb_range_meta_init()
2125 next_meta = (void *)buffer->range_addr_end; in rb_range_meta_init()
2140 * location of the ring buffer. Although their addresses in rb_range_meta_init()
2195 seq_printf(m, "buffer[%ld]: %d\n", val, meta->buffers[val]); in rbm_show()
2211 int ring_buffer_meta_seq_init(struct file *file, struct trace_buffer *buffer, int cpu) in ring_buffer_meta_seq_init() argument
2221 m->private = buffer->buffers[cpu]; in ring_buffer_meta_seq_init()
2244 struct trace_buffer *buffer = cpu_buffer->buffer; in __rb_allocate_pages() local
2273 if (buffer->range_addr_start) in __rb_allocate_pages()
2274 meta = rb_range_meta(buffer, nr_pages, cpu_buffer->cpu); in __rb_allocate_pages()
2291 /* A range was given. Use that for the buffer page */ in __rb_allocate_pages()
2301 int order = cpu_buffer->buffer->subbuf_order; in __rb_allocate_pages()
2306 bpage->order = cpu_buffer->buffer->subbuf_order; in __rb_allocate_pages()
2338 * The ring buffer page list is a circular list that does not in rb_allocate_pages()
2353 rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) in rb_allocate_cpu_buffer() argument
2365 cpu_buffer->buffer = buffer; in rb_allocate_cpu_buffer()
2367 lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key); in rb_allocate_cpu_buffer()
2384 if (buffer->range_addr_start) { in rb_allocate_cpu_buffer()
2390 cpu_buffer->ring_meta = rb_range_meta(buffer, nr_pages, cpu); in rb_allocate_cpu_buffer()
2398 int order = cpu_buffer->buffer->subbuf_order; in rb_allocate_cpu_buffer()
2419 pr_warn("Ring buffer meta buffers not all mapped\n"); in rb_allocate_cpu_buffer()
2437 /* The valid meta buffer still needs to activate the head page */ in rb_allocate_cpu_buffer()
2480 struct trace_buffer *buffer __free(kfree) = NULL; in alloc_buffer()
2488 buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()), in alloc_buffer()
2490 if (!buffer) in alloc_buffer()
2493 if (!zalloc_cpumask_var(&buffer->cpumask, GFP_KERNEL)) in alloc_buffer()
2496 buffer->subbuf_order = order; in alloc_buffer()
2498 buffer->subbuf_size = subbuf_size - BUF_PAGE_HDR_SIZE; in alloc_buffer()
2500 /* Max payload is buffer page size - header (8bytes) */ in alloc_buffer()
2501 buffer->max_data_size = buffer->subbuf_size - (sizeof(u32) * 2); in alloc_buffer()
2503 buffer->flags = flags; in alloc_buffer()
2504 buffer->clock = trace_clock_local; in alloc_buffer()
2505 buffer->reader_lock_key = key; in alloc_buffer()
2507 init_irq_work(&buffer->irq_work.work, rb_wake_up_waiters); in alloc_buffer()
2508 init_waitqueue_head(&buffer->irq_work.waiters); in alloc_buffer()
2510 buffer->cpus = nr_cpu_ids; in alloc_buffer()
2513 buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()), in alloc_buffer()
2515 if (!buffer->buffers) in alloc_buffer()
2530 /* Subtract the buffer meta data and word aligned */ in alloc_buffer()
2569 buffer->range_addr_start = start; in alloc_buffer()
2570 buffer->range_addr_end = end; in alloc_buffer()
2572 rb_range_meta_init(buffer, nr_pages, scratch_size); in alloc_buffer()
2576 nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size); in alloc_buffer()
2582 cpumask_set_cpu(cpu, buffer->cpumask); in alloc_buffer()
2583 buffer->buffers[cpu] = rb_allocate_cpu_buffer(buffer, nr_pages, cpu); in alloc_buffer()
2584 if (!buffer->buffers[cpu]) in alloc_buffer()
2587 ret = cpuhp_state_add_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node); in alloc_buffer()
2591 mutex_init(&buffer->mutex); in alloc_buffer()
2593 return_ptr(buffer); in alloc_buffer()
2596 for_each_buffer_cpu(buffer, cpu) { in alloc_buffer()
2597 if (buffer->buffers[cpu]) in alloc_buffer()
2598 rb_free_cpu_buffer(buffer->buffers[cpu]); in alloc_buffer()
2600 kfree(buffer->buffers); in alloc_buffer()
2603 free_cpumask_var(buffer->cpumask); in alloc_buffer()
2611 * @flags: attributes to set for the ring buffer.
2612 * @key: ring buffer reader_lock_key.
2615 * flag. This flag means that the buffer will overwrite old data
2616 * when the buffer wraps. If this flag is not set, the buffer will
2622 /* Default buffer page size - one system page */ in __ring_buffer_alloc()
2631 * @flags: attributes to set for the ring buffer.
2632 * @order: sub-buffer order
2636 * @key: ring buffer reader_lock_key.
2639 * flag. This flag means that the buffer will overwrite old data
2640 * when the buffer wraps. If this flag is not set, the buffer will
2653 void *ring_buffer_meta_scratch(struct trace_buffer *buffer, unsigned int *size) in ring_buffer_meta_scratch() argument
2658 if (!buffer || !buffer->meta) in ring_buffer_meta_scratch()
2661 meta = buffer->meta; in ring_buffer_meta_scratch()
2672 * ring_buffer_free - free a ring buffer.
2673 * @buffer: the buffer to free.
2676 ring_buffer_free(struct trace_buffer *buffer) in ring_buffer_free() argument
2680 cpuhp_state_remove_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node); in ring_buffer_free()
2682 irq_work_sync(&buffer->irq_work.work); in ring_buffer_free()
2684 for_each_buffer_cpu(buffer, cpu) in ring_buffer_free()
2685 rb_free_cpu_buffer(buffer->buffers[cpu]); in ring_buffer_free()
2687 kfree(buffer->buffers); in ring_buffer_free()
2688 free_cpumask_var(buffer->cpumask); in ring_buffer_free()
2690 kfree(buffer); in ring_buffer_free()
2694 void ring_buffer_set_clock(struct trace_buffer *buffer, in ring_buffer_set_clock() argument
2697 buffer->clock = clock; in ring_buffer_set_clock()
2700 void ring_buffer_set_time_stamp_abs(struct trace_buffer *buffer, bool abs) in ring_buffer_set_time_stamp_abs() argument
2702 buffer->time_stamp_abs = abs; in ring_buffer_set_time_stamp_abs()
2705 bool ring_buffer_time_stamp_abs(struct trace_buffer *buffer) in ring_buffer_time_stamp_abs() argument
2707 return buffer->time_stamp_abs; in ring_buffer_time_stamp_abs()
2747 * from the ring buffer in rb_remove_pages()
2776 /* make sure pages points to a valid page in the ring buffer */ in rb_remove_pages()
2791 /* last buffer page to remove */ in rb_remove_pages()
2808 * bytes consumed in ring buffer from here. in rb_remove_pages()
2842 * in the ring buffer. Now we are racing with the writer trying to in rb_insert_pages()
2935 * ring_buffer_resize - resize the ring buffer
2936 * @buffer: the buffer to resize.
2938 * @cpu_id: the cpu buffer to resize
2940 * Minimum size is 2 * buffer->subbuf_size.
2944 int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, in ring_buffer_resize() argument
2952 * Always succeed at resizing a non-existent buffer: in ring_buffer_resize()
2954 if (!buffer) in ring_buffer_resize()
2957 /* Make sure the requested buffer exists */ in ring_buffer_resize()
2959 !cpumask_test_cpu(cpu_id, buffer->cpumask)) in ring_buffer_resize()
2962 nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size); in ring_buffer_resize()
2974 /* prevent another thread from changing buffer sizes */ in ring_buffer_resize()
2975 mutex_lock(&buffer->mutex); in ring_buffer_resize()
2976 atomic_inc(&buffer->resizing); in ring_buffer_resize()
2981 * manipulating the ring buffer and is expecting a sane state while in ring_buffer_resize()
2984 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
2985 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
2993 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
2994 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
3021 * since we can change their buffer sizes without any race. in ring_buffer_resize()
3023 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
3024 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
3047 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
3048 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
3058 cpu_buffer = buffer->buffers[cpu_id]; in ring_buffer_resize()
3065 * manipulating the ring buffer and is expecting a sane state while in ring_buffer_resize()
3106 * The ring buffer resize can happen with the ring buffer in ring_buffer_resize()
3108 * as possible. But if the buffer is disabled, we do not need in ring_buffer_resize()
3110 * that the buffer is not corrupt. in ring_buffer_resize()
3112 if (atomic_read(&buffer->record_disabled)) { in ring_buffer_resize()
3113 atomic_inc(&buffer->record_disabled); in ring_buffer_resize()
3115 * Even though the buffer was disabled, we must make sure in ring_buffer_resize()
3121 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
3122 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
3125 atomic_dec(&buffer->record_disabled); in ring_buffer_resize()
3128 atomic_dec(&buffer->resizing); in ring_buffer_resize()
3129 mutex_unlock(&buffer->mutex); in ring_buffer_resize()
3133 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
3136 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
3151 atomic_dec(&buffer->resizing); in ring_buffer_resize()
3152 mutex_unlock(&buffer->mutex); in ring_buffer_resize()
3157 void ring_buffer_change_overwrite(struct trace_buffer *buffer, int val) in ring_buffer_change_overwrite() argument
3159 mutex_lock(&buffer->mutex); in ring_buffer_change_overwrite()
3161 buffer->flags |= RB_FL_OVERWRITE; in ring_buffer_change_overwrite()
3163 buffer->flags &= ~RB_FL_OVERWRITE; in ring_buffer_change_overwrite()
3164 mutex_unlock(&buffer->mutex); in ring_buffer_change_overwrite()
3256 addr &= (PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1; in rb_event_index()
3281 /* Return the index into the sub-buffers for a given sub-buffer */
3495 unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size); in rb_reset_tail()
3564 /* Set write to end of buffer */ in rb_reset_tail()
3580 struct trace_buffer *buffer = cpu_buffer->buffer; in rb_move_tail() local
3590 * it all the way around the buffer, bail, and warn in rb_move_tail()
3603 * page with the buffer head. in rb_move_tail()
3609 * the buffer, unless the commit page is still on the in rb_move_tail()
3623 if (!(buffer->flags & RB_FL_OVERWRITE)) { in rb_move_tail()
3639 * page. We could have a small buffer, and in rb_move_tail()
3640 * have filled up the buffer with events in rb_move_tail()
3738 * is added to the buffer, it will lose those bits. in rb_add_timestamp()
3754 pr_warn("Ring buffer clock went backwards: %llu -> %llu\n", in rb_add_timestamp()
3769 * @cpu_buffer: The per cpu buffer of the @event
3774 * is the actual size that is written to the ring buffer,
3849 addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1); in rb_try_to_discard()
4040 rb_wakeups(struct trace_buffer *buffer, struct ring_buffer_per_cpu *cpu_buffer) in rb_wakeups() argument
4042 if (buffer->irq_work.waiters_pending) { in rb_wakeups()
4043 buffer->irq_work.waiters_pending = false; in rb_wakeups()
4045 rb_irq_work_queue(&buffer->irq_work); in rb_wakeups()
4065 if (!full_hit(buffer, cpu_buffer->cpu, cpu_buffer->shortest_full)) in rb_wakeups()
4130 * if an interrupt comes in while NORMAL bit is set and the ring buffer
4182 * @buffer: The ring buffer to modify
4184 * The ring buffer has a safety mechanism to prevent recursion.
4193 void ring_buffer_nest_start(struct trace_buffer *buffer) in ring_buffer_nest_start() argument
4201 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_nest_start()
4208 * @buffer: The ring buffer to modify
4213 void ring_buffer_nest_end(struct trace_buffer *buffer) in ring_buffer_nest_end() argument
4220 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_nest_end()
4228 * @buffer: The buffer to commit to
4230 * This commits the data to the ring buffer, and releases any locks held.
4234 int ring_buffer_unlock_commit(struct trace_buffer *buffer) in ring_buffer_unlock_commit() argument
4239 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_unlock_commit()
4243 rb_wakeups(buffer, cpu_buffer); in ring_buffer_unlock_commit()
4400 * the buffer page.
4478 info->ts = rb_time_stamp(cpu_buffer->buffer); in __rb_reserve_next()
4489 /* Use the sub-buffer timestamp */ in __rb_reserve_next()
4512 /* See if we shot pass the end of this buffer page */ in __rb_reserve_next()
4513 if (unlikely(write > cpu_buffer->buffer->subbuf_size)) { in __rb_reserve_next()
4548 ts = rb_time_stamp(cpu_buffer->buffer); in __rb_reserve_next()
4585 /* We reserved something on the buffer */ in __rb_reserve_next()
4606 rb_reserve_next_event(struct trace_buffer *buffer, in rb_reserve_next_event() argument
4616 * ring buffer does cmpxchg as well as atomic64 operations in rb_reserve_next_event()
4631 * Due to the ability to swap a cpu buffer from a buffer in rb_reserve_next_event()
4637 if (unlikely(READ_ONCE(cpu_buffer->buffer) != buffer)) { in rb_reserve_next_event()
4646 if (ring_buffer_time_stamp_abs(cpu_buffer->buffer)) { in rb_reserve_next_event()
4649 if (info.length > cpu_buffer->buffer->max_data_size) in rb_reserve_next_event()
4687 * ring_buffer_lock_reserve - reserve a part of the buffer
4688 * @buffer: the ring buffer to reserve from
4691 * Returns a reserved event on the ring buffer to copy directly to.
4702 ring_buffer_lock_reserve(struct trace_buffer *buffer, unsigned long length) in ring_buffer_lock_reserve() argument
4711 if (unlikely(atomic_read(&buffer->record_disabled))) in ring_buffer_lock_reserve()
4716 if (unlikely(!cpumask_test_cpu(cpu, buffer->cpumask))) in ring_buffer_lock_reserve()
4719 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_lock_reserve()
4724 if (unlikely(length > buffer->max_data_size)) in ring_buffer_lock_reserve()
4730 event = rb_reserve_next_event(buffer, cpu_buffer, length); in ring_buffer_lock_reserve()
4758 addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1); in rb_decrement_entry()
4780 /* commit not part of this buffer?? */ in rb_decrement_entry()
4786 * @buffer: the ring buffer
4789 * Sometimes an event that is in the ring buffer needs to be ignored.
4790 * This function lets the user discard an event in the ring buffer
4794 * committed. It will try to free the event from the ring buffer
4803 void ring_buffer_discard_commit(struct trace_buffer *buffer, in ring_buffer_discard_commit() argument
4813 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_discard_commit()
4820 RB_WARN_ON(buffer, !local_read(&cpu_buffer->committing)); in ring_buffer_discard_commit()
4834 * ring_buffer_write - write data to the buffer without reserving
4835 * @buffer: The ring buffer to write to.
4837 * @data: The data to write to the buffer.
4840 * one function. If you already have the data to write to the buffer, it
4846 int ring_buffer_write(struct trace_buffer *buffer, in ring_buffer_write() argument
4858 if (atomic_read(&buffer->record_disabled)) in ring_buffer_write()
4863 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_write()
4866 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_write()
4871 if (length > buffer->max_data_size) in ring_buffer_write()
4877 event = rb_reserve_next_event(buffer, cpu_buffer, length); in ring_buffer_write()
4887 rb_wakeups(buffer, cpu_buffer); in ring_buffer_write()
4898 * The total entries in the ring buffer is the running counter
4899 * of entries entered into the ring buffer, minus the sum of
4900 * the entries read from the ring buffer and the number of
4916 * ring_buffer_record_disable - stop all writes into the buffer
4917 * @buffer: The ring buffer to stop writes to.
4919 * This prevents all writes to the buffer. Any attempt to write
4920 * to the buffer after this will fail and return NULL.
4924 void ring_buffer_record_disable(struct trace_buffer *buffer) in ring_buffer_record_disable() argument
4926 atomic_inc(&buffer->record_disabled); in ring_buffer_record_disable()
4931 * ring_buffer_record_enable - enable writes to the buffer
4932 * @buffer: The ring buffer to enable writes
4937 void ring_buffer_record_enable(struct trace_buffer *buffer) in ring_buffer_record_enable() argument
4939 atomic_dec(&buffer->record_disabled); in ring_buffer_record_enable()
4944 * ring_buffer_record_off - stop all writes into the buffer
4945 * @buffer: The ring buffer to stop writes to.
4947 * This prevents all writes to the buffer. Any attempt to write
4948 * to the buffer after this will fail and return NULL.
4954 void ring_buffer_record_off(struct trace_buffer *buffer) in ring_buffer_record_off() argument
4959 rd = atomic_read(&buffer->record_disabled); in ring_buffer_record_off()
4962 } while (!atomic_try_cmpxchg(&buffer->record_disabled, &rd, new_rd)); in ring_buffer_record_off()
4967 * ring_buffer_record_on - restart writes into the buffer
4968 * @buffer: The ring buffer to start writes to.
4970 * This enables all writes to the buffer that was disabled by
4977 void ring_buffer_record_on(struct trace_buffer *buffer) in ring_buffer_record_on() argument
4982 rd = atomic_read(&buffer->record_disabled); in ring_buffer_record_on()
4985 } while (!atomic_try_cmpxchg(&buffer->record_disabled, &rd, new_rd)); in ring_buffer_record_on()
4990 * ring_buffer_record_is_on - return true if the ring buffer can write
4991 * @buffer: The ring buffer to see if write is enabled
4993 * Returns true if the ring buffer is in a state that it accepts writes.
4995 bool ring_buffer_record_is_on(struct trace_buffer *buffer) in ring_buffer_record_is_on() argument
4997 return !atomic_read(&buffer->record_disabled); in ring_buffer_record_is_on()
5001 * ring_buffer_record_is_set_on - return true if the ring buffer is set writable
5002 * @buffer: The ring buffer to see if write is set enabled
5004 * Returns true if the ring buffer is set writable by ring_buffer_record_on().
5007 * It may return true when the ring buffer has been disabled by
5009 * the ring buffer.
5011 bool ring_buffer_record_is_set_on(struct trace_buffer *buffer) in ring_buffer_record_is_set_on() argument
5013 return !(atomic_read(&buffer->record_disabled) & RB_BUFFER_OFF); in ring_buffer_record_is_set_on()
5017 * ring_buffer_record_is_on_cpu - return true if the ring buffer can write
5018 * @buffer: The ring buffer to see if write is enabled
5019 * @cpu: The CPU to test if the ring buffer can write too
5021 * Returns true if the ring buffer is in a state that it accepts writes
5024 bool ring_buffer_record_is_on_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_record_is_on_cpu() argument
5028 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_record_is_on_cpu()
5030 return ring_buffer_record_is_set_on(buffer) && in ring_buffer_record_is_on_cpu()
5036 * @buffer: The ring buffer to stop writes to.
5037 * @cpu: The CPU buffer to stop
5039 * This prevents all writes to the buffer. Any attempt to write
5040 * to the buffer after this will fail and return NULL.
5044 void ring_buffer_record_disable_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_record_disable_cpu() argument
5048 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_record_disable_cpu()
5051 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_record_disable_cpu()
5057 * ring_buffer_record_enable_cpu - enable writes to the buffer
5058 * @buffer: The ring buffer to enable writes
5064 void ring_buffer_record_enable_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_record_enable_cpu() argument
5068 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_record_enable_cpu()
5071 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_record_enable_cpu()
5077 * ring_buffer_oldest_event_ts - get the oldest event timestamp from the buffer
5078 * @buffer: The ring buffer
5079 * @cpu: The per CPU buffer to read from.
5081 u64 ring_buffer_oldest_event_ts(struct trace_buffer *buffer, int cpu) in ring_buffer_oldest_event_ts() argument
5088 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_oldest_event_ts()
5091 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_oldest_event_ts()
5110 * ring_buffer_bytes_cpu - get the number of bytes unconsumed in a cpu buffer
5111 * @buffer: The ring buffer
5112 * @cpu: The per CPU buffer to read from.
5114 unsigned long ring_buffer_bytes_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_bytes_cpu() argument
5119 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_bytes_cpu()
5122 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_bytes_cpu()
5130 * ring_buffer_entries_cpu - get the number of entries in a cpu buffer
5131 * @buffer: The ring buffer
5132 * @cpu: The per CPU buffer to get the entries from.
5134 unsigned long ring_buffer_entries_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_entries_cpu() argument
5138 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_entries_cpu()
5141 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_entries_cpu()
5149 * buffer wrapping around (only if RB_FL_OVERWRITE is on).
5150 * @buffer: The ring buffer
5151 * @cpu: The per CPU buffer to get the number of overruns from
5153 unsigned long ring_buffer_overrun_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_overrun_cpu() argument
5158 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_overrun_cpu()
5161 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_overrun_cpu()
5170 * commits failing due to the buffer wrapping around while there are uncommitted
5172 * @buffer: The ring buffer
5173 * @cpu: The per CPU buffer to get the number of overruns from
5176 ring_buffer_commit_overrun_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_commit_overrun_cpu() argument
5181 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_commit_overrun_cpu()
5184 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_commit_overrun_cpu()
5193 * the ring buffer filling up (only if RB_FL_OVERWRITE is off).
5194 * @buffer: The ring buffer
5195 * @cpu: The per CPU buffer to get the number of overruns from
5198 ring_buffer_dropped_events_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_dropped_events_cpu() argument
5203 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_dropped_events_cpu()
5206 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_dropped_events_cpu()
5215 * @buffer: The ring buffer
5216 * @cpu: The per CPU buffer to get the number of events read
5219 ring_buffer_read_events_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_read_events_cpu() argument
5223 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_read_events_cpu()
5226 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_read_events_cpu()
5232 * ring_buffer_entries - get the number of entries in a buffer
5233 * @buffer: The ring buffer
5235 * Returns the total number of entries in the ring buffer
5238 unsigned long ring_buffer_entries(struct trace_buffer *buffer) in ring_buffer_entries() argument
5244 /* if you care about this being correct, lock the buffer */ in ring_buffer_entries()
5245 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_entries()
5246 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_entries()
5255 * ring_buffer_overruns - get the number of overruns in buffer
5256 * @buffer: The ring buffer
5258 * Returns the total number of overruns in the ring buffer
5261 unsigned long ring_buffer_overruns(struct trace_buffer *buffer) in ring_buffer_overruns() argument
5267 /* if you care about this being correct, lock the buffer */ in ring_buffer_overruns()
5268 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_overruns()
5269 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_overruns()
5434 unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size); in rb_get_reader_page()
5471 /* Don't bother swapping if the ring buffer is empty */ in rb_get_reader_page()
5493 * cpu_buffer->pages just needs to point to the buffer, it in rb_get_reader_page()
5494 * has no specific buffer page to point to. Lets move it out in rb_get_reader_page()
5609 /* This function should not be called when buffer is empty */ in rb_advance_reader()
5641 * Check if we are at the end of the buffer. in rb_advance_iter()
5708 ring_buffer_normalize_time_stamp(cpu_buffer->buffer, in rb_buffer_peek()
5718 ring_buffer_normalize_time_stamp(cpu_buffer->buffer, in rb_buffer_peek()
5736 struct trace_buffer *buffer; in rb_iter_peek() local
5745 buffer = cpu_buffer->buffer; in rb_iter_peek()
5748 * Check if someone performed a consuming read to the buffer in rb_iter_peek()
5749 * or removed some pages from the buffer. In these cases, in rb_iter_peek()
5765 * the ring buffer with an active write as the consumer is. in rb_iter_peek()
5801 ring_buffer_normalize_time_stamp(cpu_buffer->buffer, in rb_iter_peek()
5811 ring_buffer_normalize_time_stamp(buffer, in rb_iter_peek()
5832 * If an NMI die dumps out the content of the ring buffer in rb_reader_lock()
5834 * preempted a task that holds the ring buffer locks. If in rb_reader_lock()
5836 * to do the read, but this can corrupt the ring buffer, in rb_reader_lock()
5843 /* Continue without locking, but disable the ring buffer */ in rb_reader_lock()
5857 * @buffer: The ring buffer to read
5866 ring_buffer_peek(struct trace_buffer *buffer, int cpu, u64 *ts, in ring_buffer_peek() argument
5869 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; in ring_buffer_peek()
5874 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_peek()
5893 * @iter: The ring buffer iterator
5908 * @iter: The ring buffer iterator
5934 * @buffer: The ring buffer to get the next event from
5935 * @cpu: the cpu to read the buffer from
5939 * Returns the next event in the ring buffer, and that event is consumed.
5941 * and eventually empty the ring buffer if the producer is slower.
5944 ring_buffer_consume(struct trace_buffer *buffer, int cpu, u64 *ts, in ring_buffer_consume() argument
5956 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_consume()
5959 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_consume()
5983 * ring_buffer_read_start - start a non consuming read of the buffer
5984 * @buffer: The ring buffer to read from
5985 * @cpu: The cpu buffer to iterate over
5989 * the buffer. If the buffer is disabled for writing, it will produce
5990 * the same information each time, but if the buffer is still writing
5996 ring_buffer_read_start(struct trace_buffer *buffer, int cpu, gfp_t flags) in ring_buffer_read_start() argument
6001 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_read_start()
6009 iter->event_size = buffer->subbuf_size; in ring_buffer_read_start()
6016 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_read_start()
6032 * ring_buffer_read_finish - finish reading the iterator of the buffer
6035 * This re-enables resizing of the buffer, and frees the iterator.
6042 /* Use this opportunity to check the integrity of the ring buffer. */ in ring_buffer_read_finish()
6053 * @iter: The ring buffer iterator
6072 * ring_buffer_size - return the size of the ring buffer (in bytes)
6073 * @buffer: The ring buffer.
6074 * @cpu: The CPU to get ring buffer size from.
6076 unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu) in ring_buffer_size() argument
6078 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_size()
6081 return buffer->subbuf_size * buffer->buffers[cpu]->nr_pages; in ring_buffer_size()
6087 * @buffer: The ring buffer.
6091 unsigned long ring_buffer_max_event_size(struct trace_buffer *buffer) in ring_buffer_max_event_size() argument
6094 if (ring_buffer_time_stamp_abs(buffer)) in ring_buffer_max_event_size()
6095 return buffer->max_data_size - RB_LEN_TIME_EXTEND; in ring_buffer_max_event_size()
6096 return buffer->max_data_size; in ring_buffer_max_event_size()
6109 * When the buffer is memory mapped to user space, each sub buffer
6113 * For a normal allocated ring buffer, the id is saved in the buffer page
6116 * But for a fixed memory mapped buffer, the id is already assigned for
6120 * For the normal pages, set the buffer page id with the passed in @id
6131 * otherwise, set the buffer page with this id in rb_page_id()
6218 /* Must have disabled the cpu buffer then done a synchronize_rcu */
6234 * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
6235 * @buffer: The ring buffer to reset a per cpu buffer of
6236 * @cpu: The CPU buffer to be reset
6238 void ring_buffer_reset_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_reset_cpu() argument
6240 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; in ring_buffer_reset_cpu()
6242 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_reset_cpu()
6245 /* prevent another thread from changing buffer sizes */ in ring_buffer_reset_cpu()
6246 mutex_lock(&buffer->mutex); in ring_buffer_reset_cpu()
6259 mutex_unlock(&buffer->mutex); in ring_buffer_reset_cpu()
6267 * ring_buffer_reset_online_cpus - reset a ring buffer per CPU buffer
6268 * @buffer: The ring buffer to reset a per cpu buffer of
6270 void ring_buffer_reset_online_cpus(struct trace_buffer *buffer) in ring_buffer_reset_online_cpus() argument
6275 /* prevent another thread from changing buffer sizes */ in ring_buffer_reset_online_cpus()
6276 mutex_lock(&buffer->mutex); in ring_buffer_reset_online_cpus()
6278 for_each_online_buffer_cpu(buffer, cpu) { in ring_buffer_reset_online_cpus()
6279 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_reset_online_cpus()
6288 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_reset_online_cpus()
6289 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_reset_online_cpus()
6304 mutex_unlock(&buffer->mutex); in ring_buffer_reset_online_cpus()
6308 * ring_buffer_reset - reset a ring buffer
6309 * @buffer: The ring buffer to reset all cpu buffers
6311 void ring_buffer_reset(struct trace_buffer *buffer) in ring_buffer_reset() argument
6316 /* prevent another thread from changing buffer sizes */ in ring_buffer_reset()
6317 mutex_lock(&buffer->mutex); in ring_buffer_reset()
6319 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_reset()
6320 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_reset()
6329 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_reset()
6330 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_reset()
6338 mutex_unlock(&buffer->mutex); in ring_buffer_reset()
6343 * ring_buffer_empty - is the ring buffer empty?
6344 * @buffer: The ring buffer to test
6346 bool ring_buffer_empty(struct trace_buffer *buffer) in ring_buffer_empty() argument
6354 /* yes this is racy, but if you don't like the race, lock the buffer */ in ring_buffer_empty()
6355 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_empty()
6356 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_empty()
6372 * ring_buffer_empty_cpu - is a cpu buffer of a ring buffer empty?
6373 * @buffer: The ring buffer
6374 * @cpu: The CPU buffer to test
6376 bool ring_buffer_empty_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_empty_cpu() argument
6383 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_empty_cpu()
6386 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_empty_cpu()
6399 * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers
6400 * @buffer_a: One buffer to swap with
6401 * @buffer_b: The other buffer to swap with
6405 * of a CPU buffer and has another back up buffer lying around.
6406 * it is expected that the tracer handles the cpu buffer not being
6463 * it will mess the state of the cpu buffer. in ring_buffer_swap_cpu()
6473 cpu_buffer_b->buffer = buffer_a; in ring_buffer_swap_cpu()
6474 cpu_buffer_a->buffer = buffer_b; in ring_buffer_swap_cpu()
6487 * ring_buffer_alloc_read_page - allocate a page to read from buffer
6488 * @buffer: the buffer to allocate for.
6489 * @cpu: the cpu buffer to allocate.
6492 * When reading a full page from the ring buffer, these functions
6495 * needs to get pages from the ring buffer, it passes the result
6497 * the page that was allocated, with the read page of the buffer.
6503 ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu) in ring_buffer_alloc_read_page() argument
6509 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_alloc_read_page()
6516 bpage->order = buffer->subbuf_order; in ring_buffer_alloc_read_page()
6517 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_alloc_read_page()
6532 bpage->data = alloc_cpu_data(cpu, cpu_buffer->buffer->subbuf_order); in ring_buffer_alloc_read_page()
6545 * @buffer: the buffer the page was allocate for
6546 * @cpu: the cpu buffer the page came from
6551 void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, in ring_buffer_free_read_page() argument
6559 if (!buffer || !buffer->buffers || !buffer->buffers[cpu]) in ring_buffer_free_read_page()
6562 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_free_read_page()
6566 * is different from the subbuffer order of the buffer - in ring_buffer_free_read_page()
6569 if (page_ref_count(page) > 1 || data_page->order != buffer->subbuf_order) in ring_buffer_free_read_page()
6590 * ring_buffer_read_page - extract a page from the ring buffer
6591 * @buffer: buffer to extract from
6594 * @cpu: the cpu of the buffer to extract
6597 * This function will pull out a page from the ring buffer and consume it.
6600 * to swap with a page in the ring buffer.
6603 * rpage = ring_buffer_alloc_read_page(buffer, cpu);
6606 * ret = ring_buffer_read_page(buffer, rpage, len, cpu, 0);
6609 * ring_buffer_free_read_page(buffer, cpu, rpage);
6615 * The ring buffer can be used anywhere in the kernel and can not
6616 * blindly call wake_up. The layer that uses the ring buffer must be
6623 int ring_buffer_read_page(struct trace_buffer *buffer, in ring_buffer_read_page() argument
6627 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; in ring_buffer_read_page()
6636 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_read_page()
6651 if (data_page->order != buffer->subbuf_order) in ring_buffer_read_page()
6676 * we must copy the data from the page to the buffer. in ring_buffer_read_page()
6774 if (buffer->subbuf_size - commit >= sizeof(missed_events)) { in ring_buffer_read_page()
6786 if (commit < buffer->subbuf_size) in ring_buffer_read_page()
6787 memset(&bpage->data[commit], 0, buffer->subbuf_size - commit); in ring_buffer_read_page()
6806 * ring_buffer_subbuf_size_get - get size of the sub buffer.
6807 * @buffer: the buffer to get the sub buffer size from
6809 * Returns size of the sub buffer, in bytes.
6811 int ring_buffer_subbuf_size_get(struct trace_buffer *buffer) in ring_buffer_subbuf_size_get() argument
6813 return buffer->subbuf_size + BUF_PAGE_HDR_SIZE; in ring_buffer_subbuf_size_get()
6818 * ring_buffer_subbuf_order_get - get order of system sub pages in one buffer page.
6819 * @buffer: The ring_buffer to get the system sub page order from
6821 * By default, one ring buffer sub page equals to one system page. This parameter
6822 * is configurable, per ring buffer. The size of the ring buffer sub page can be
6825 * Returns the order of buffer sub page size, in system pages:
6826 * 0 means the sub buffer size is 1 system page and so forth.
6829 int ring_buffer_subbuf_order_get(struct trace_buffer *buffer) in ring_buffer_subbuf_order_get() argument
6831 if (!buffer) in ring_buffer_subbuf_order_get()
6834 return buffer->subbuf_order; in ring_buffer_subbuf_order_get()
6839 * ring_buffer_subbuf_order_set - set the size of ring buffer sub page.
6840 * @buffer: The ring_buffer to set the new page size.
6841 * @order: Order of the system pages in one sub buffer page
6843 * By default, one ring buffer pages equals to one system page. This API can be
6844 * used to set new size of the ring buffer page. The size must be order of
6846 * system pages that are allocated for one ring buffer page:
6854 int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) in ring_buffer_subbuf_order_set() argument
6864 if (!buffer || order < 0) in ring_buffer_subbuf_order_set()
6867 if (buffer->subbuf_order == order) in ring_buffer_subbuf_order_set()
6878 old_order = buffer->subbuf_order; in ring_buffer_subbuf_order_set()
6879 old_size = buffer->subbuf_size; in ring_buffer_subbuf_order_set()
6881 /* prevent another thread from changing buffer sizes */ in ring_buffer_subbuf_order_set()
6882 guard(mutex)(&buffer->mutex); in ring_buffer_subbuf_order_set()
6883 atomic_inc(&buffer->record_disabled); in ring_buffer_subbuf_order_set()
6888 buffer->subbuf_order = order; in ring_buffer_subbuf_order_set()
6889 buffer->subbuf_size = psize - BUF_PAGE_HDR_SIZE; in ring_buffer_subbuf_order_set()
6892 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_subbuf_order_set()
6894 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_subbuf_order_set()
6897 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_subbuf_order_set()
6905 nr_pages = old_size * buffer->buffers[cpu]->nr_pages; in ring_buffer_subbuf_order_set()
6906 nr_pages = DIV_ROUND_UP(nr_pages, buffer->subbuf_size); in ring_buffer_subbuf_order_set()
6917 /* Allocate the new size buffer */ in ring_buffer_subbuf_order_set()
6927 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_subbuf_order_set()
6932 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_subbuf_order_set()
6935 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_subbuf_order_set()
6986 atomic_dec(&buffer->record_disabled); in ring_buffer_subbuf_order_set()
6991 buffer->subbuf_order = old_order; in ring_buffer_subbuf_order_set()
6992 buffer->subbuf_size = old_size; in ring_buffer_subbuf_order_set()
6994 atomic_dec(&buffer->record_disabled); in ring_buffer_subbuf_order_set()
6996 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_subbuf_order_set()
6997 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_subbuf_order_set()
7070 meta->subbuf_size = cpu_buffer->buffer->subbuf_size + BUF_PAGE_HDR_SIZE; in rb_setup_ids_meta_page()
7077 rb_get_mapped_buffer(struct trace_buffer *buffer, int cpu) in rb_get_mapped_buffer() argument
7081 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in rb_get_mapped_buffer()
7084 cpu_buffer = buffer->buffers[cpu]; in rb_get_mapped_buffer()
7122 mutex_lock(&cpu_buffer->buffer->mutex); in __rb_inc_dec_mapped()
7134 mutex_unlock(&cpu_buffer->buffer->mutex); in __rb_inc_dec_mapped()
7165 subbuf_order = cpu_buffer->buffer->subbuf_order; in __rb_map_vma()
7250 int ring_buffer_map(struct trace_buffer *buffer, int cpu, in ring_buffer_map() argument
7257 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_map()
7260 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_map()
7271 /* prevent another thread from changing buffer/sub-buffer sizes */ in ring_buffer_map()
7272 guard(mutex)(&buffer->mutex); in ring_buffer_map()
7317 void ring_buffer_map_dup(struct trace_buffer *buffer, int cpu) in ring_buffer_map_dup() argument
7321 if (WARN_ON(!cpumask_test_cpu(cpu, buffer->cpumask))) in ring_buffer_map_dup()
7324 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_map_dup()
7331 WARN(1, "Unexpected buffer stat, it should be mapped"); in ring_buffer_map_dup()
7334 int ring_buffer_unmap(struct trace_buffer *buffer, int cpu) in ring_buffer_unmap() argument
7339 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_unmap()
7342 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_unmap()
7353 guard(mutex)(&buffer->mutex); in ring_buffer_unmap()
7371 int ring_buffer_map_get_reader(struct trace_buffer *buffer, int cpu) in ring_buffer_map_get_reader() argument
7379 cpu_buffer = rb_get_mapped_buffer(buffer, cpu); in ring_buffer_map_get_reader()
7429 if (buffer->subbuf_size - commit >= sizeof(missed_events)) { in ring_buffer_map_get_reader()
7443 * addition of an event and then filled the buffer in ring_buffer_map_get_reader()
7449 pr_info("Ring buffer [%d] commit overrun lost %ld events at timestamp:%lld\n", in ring_buffer_map_get_reader()
7461 buffer->subbuf_size + BUF_PAGE_HDR_SIZE); in ring_buffer_map_get_reader()
7473 * If we were to free the buffer, then the user would lose any trace that was in
7474 * the buffer.
7478 struct trace_buffer *buffer; in trace_rb_cpu_prepare() local
7483 buffer = container_of(node, struct trace_buffer, node); in trace_rb_cpu_prepare()
7484 if (cpumask_test_cpu(cpu, buffer->cpumask)) in trace_rb_cpu_prepare()
7490 for_each_buffer_cpu(buffer, cpu_i) { in trace_rb_cpu_prepare()
7493 nr_pages = buffer->buffers[cpu_i]->nr_pages; in trace_rb_cpu_prepare()
7494 if (nr_pages != buffer->buffers[cpu_i]->nr_pages) { in trace_rb_cpu_prepare()
7502 buffer->buffers[cpu] = in trace_rb_cpu_prepare()
7503 rb_allocate_cpu_buffer(buffer, nr_pages, cpu); in trace_rb_cpu_prepare()
7504 if (!buffer->buffers[cpu]) { in trace_rb_cpu_prepare()
7505 WARN(1, "failed to allocate ring buffer on CPU %u\n", in trace_rb_cpu_prepare()
7510 cpumask_set_cpu(cpu, buffer->cpumask); in trace_rb_cpu_prepare()
7516 * This is a basic integrity check of the ring buffer.
7519 * writing to the per cpu ring buffer various sizes of data.
7523 * IPIs to the other CPUs to also write into the ring buffer.
7524 * this is to test the nesting ability of the buffer.
7527 * ring buffer should happen that's not expected, a big warning
7533 struct trace_buffer *buffer; member
7586 /* read rb_test_started before checking buffer enabled */ in rb_write_something()
7589 event = ring_buffer_lock_reserve(data->buffer, len); in rb_write_something()
7603 if (RB_WARN_ON(data->buffer, event_len < len)) in rb_write_something()
7629 ring_buffer_unlock_commit(data->buffer); in rb_write_something()
7675 struct trace_buffer *buffer; in test_ringbuffer() local
7680 pr_warn("Lockdown is enabled, skipping ring buffer tests\n"); in test_ringbuffer()
7684 pr_info("Running ring buffer tests...\n"); in test_ringbuffer()
7686 buffer = ring_buffer_alloc(RB_TEST_BUFFER_SIZE, RB_FL_OVERWRITE); in test_ringbuffer()
7687 if (WARN_ON(!buffer)) in test_ringbuffer()
7690 /* Disable buffer so that threads can't write to it yet */ in test_ringbuffer()
7691 ring_buffer_record_off(buffer); in test_ringbuffer()
7694 rb_data[cpu].buffer = buffer; in test_ringbuffer()
7714 ring_buffer_record_on(buffer); in test_ringbuffer()
7716 * Show buffer is enabled before setting rb_test_started. in test_ringbuffer()
7719 * buffer gets enabled, there will always be some kind of in test_ringbuffer()
7722 * the threads see that the buffer is active. in test_ringbuffer()
7740 ring_buffer_free(buffer); in test_ringbuffer()
7780 if (RB_WARN_ON(buffer, total_dropped)) in test_ringbuffer()
7785 while ((event = ring_buffer_consume(buffer, cpu, NULL, &lost))) { in test_ringbuffer()
7792 pr_info("buffer had: %.*s\n", item->size, item->str); in test_ringbuffer()
7794 RB_WARN_ON(buffer, 1); in test_ringbuffer()
7814 if (RB_WARN_ON(buffer, total_len != total_alloc || in test_ringbuffer()
7818 if (RB_WARN_ON(buffer, total_lost + total_read != total_events)) in test_ringbuffer()
7824 pr_info("Ring buffer PASSED!\n"); in test_ringbuffer()
7826 ring_buffer_free(buffer); in test_ringbuffer()