1 /* SPDX-License-Identifier: GPL-2.0 */ 2 3 #ifndef _LINUX_TRACE_EVENT_H 4 #define _LINUX_TRACE_EVENT_H 5 6 #include <linux/ring_buffer.h> 7 #include <linux/trace_seq.h> 8 #include <linux/percpu.h> 9 #include <linux/hardirq.h> 10 #include <linux/perf_event.h> 11 #include <linux/tracepoint.h> 12 13 struct trace_array; 14 struct array_buffer; 15 struct tracer; 16 struct dentry; 17 struct bpf_prog; 18 union bpf_attr; 19 20 /* Used for event string fields when they are NULL */ 21 #define EVENT_NULL_STR "(null)" 22 23 const char *trace_print_flags_seq(struct trace_seq *p, const char *delim, 24 unsigned long flags, 25 const struct trace_print_flags *flag_array, 26 size_t flag_array_size); 27 28 const char *trace_print_symbols_seq(struct trace_seq *p, unsigned long val, 29 const struct trace_print_flags *symbol_array, 30 size_t symbol_array_size); 31 32 #if BITS_PER_LONG == 32 33 const char *trace_print_flags_seq_u64(struct trace_seq *p, const char *delim, 34 unsigned long long flags, 35 const struct trace_print_flags_u64 *flag_array, 36 size_t flag_array_size); 37 38 const char *trace_print_symbols_seq_u64(struct trace_seq *p, 39 unsigned long long val, 40 const struct trace_print_flags_u64 *symbol_array, 41 size_t symbol_array_size); 42 #endif 43 44 struct trace_iterator; 45 struct trace_event; 46 47 const char *trace_print_bitmask_seq(struct trace_iterator *iter, void *bitmask_ptr, 48 unsigned int bitmask_size); 49 50 const char *trace_print_hex_seq(struct trace_seq *p, 51 const unsigned char *buf, int len, 52 bool concatenate); 53 54 const char *trace_print_array_seq(struct trace_seq *p, 55 const void *buf, int count, 56 size_t el_size); 57 58 const char * 59 trace_print_hex_dump_seq(struct trace_seq *p, const char *prefix_str, 60 int prefix_type, int rowsize, int groupsize, 61 const void *buf, size_t len, bool ascii); 62 63 int trace_raw_output_prep(struct trace_iterator *iter, 64 struct trace_event *event); 65 extern __printf(2, 3) 66 void trace_event_printf(struct trace_iterator *iter, const char *fmt, ...); 67 68 /* Used to find the offset and length of dynamic fields in trace events */ 69 struct trace_dynamic_info { 70 #ifdef CONFIG_CPU_BIG_ENDIAN 71 u16 len; 72 u16 offset; 73 #else 74 u16 offset; 75 u16 len; 76 #endif 77 } __packed; 78 79 /* 80 * The trace entry - the most basic unit of tracing. This is what 81 * is printed in the end as a single line in the trace output, such as: 82 * 83 * bash-15816 [01] 235.197585: idle_cpu <- irq_enter 84 */ 85 struct trace_entry { 86 unsigned short type; 87 unsigned char flags; 88 unsigned char preempt_count; 89 int pid; 90 }; 91 92 #define TRACE_EVENT_TYPE_MAX \ 93 ((1 << (sizeof(((struct trace_entry *)0)->type) * 8)) - 1) 94 95 /* 96 * Trace iterator - used by printout routines who present trace 97 * results to users and which routines might sleep, etc: 98 */ 99 struct trace_iterator { 100 struct trace_array *tr; 101 struct tracer *trace; 102 struct array_buffer *array_buffer; 103 void *private; 104 int cpu_file; 105 struct mutex mutex; 106 struct ring_buffer_iter **buffer_iter; 107 unsigned long iter_flags; 108 void *temp; /* temp holder */ 109 unsigned int temp_size; 110 char *fmt; /* modified format holder */ 111 unsigned int fmt_size; 112 atomic_t wait_index; 113 114 /* trace_seq for __print_flags() and __print_symbolic() etc. */ 115 struct trace_seq tmp_seq; 116 117 cpumask_var_t started; 118 119 /* Set when the file is closed to prevent new waiters */ 120 bool closed; 121 122 /* it's true when current open file is snapshot */ 123 bool snapshot; 124 125 /* The below is zeroed out in pipe_read */ 126 struct trace_seq seq; 127 struct trace_entry *ent; 128 unsigned long lost_events; 129 int leftover; 130 int ent_size; 131 int cpu; 132 u64 ts; 133 134 loff_t pos; 135 long idx; 136 137 /* All new field here will be zeroed out in pipe_read */ 138 }; 139 140 enum trace_iter_flags { 141 TRACE_FILE_LAT_FMT = 1, 142 TRACE_FILE_ANNOTATE = 2, 143 TRACE_FILE_TIME_IN_NS = 4, 144 TRACE_FILE_PAUSE = 8, 145 }; 146 147 148 typedef enum print_line_t (*trace_print_func)(struct trace_iterator *iter, 149 int flags, struct trace_event *event); 150 151 struct trace_event_functions { 152 trace_print_func trace; 153 trace_print_func raw; 154 trace_print_func hex; 155 trace_print_func binary; 156 }; 157 158 struct trace_event { 159 struct hlist_node node; 160 int type; 161 struct trace_event_functions *funcs; 162 }; 163 164 extern int register_trace_event(struct trace_event *event); 165 extern int unregister_trace_event(struct trace_event *event); 166 167 /* Return values for print_line callback */ 168 enum print_line_t { 169 TRACE_TYPE_PARTIAL_LINE = 0, /* Retry after flushing the seq */ 170 TRACE_TYPE_HANDLED = 1, 171 TRACE_TYPE_UNHANDLED = 2, /* Relay to other output functions */ 172 TRACE_TYPE_NO_CONSUME = 3 /* Handled but ask to not consume */ 173 }; 174 175 enum print_line_t trace_handle_return(struct trace_seq *s); 176 177 static inline void tracing_generic_entry_update(struct trace_entry *entry, 178 unsigned short type, 179 unsigned int trace_ctx) 180 { 181 entry->preempt_count = trace_ctx & 0xff; 182 entry->pid = current->pid; 183 entry->type = type; 184 entry->flags = trace_ctx >> 16; 185 } 186 187 unsigned int tracing_gen_ctx_irq_test(unsigned int irqs_status); 188 189 enum trace_flag_type { 190 TRACE_FLAG_IRQS_OFF = 0x01, 191 TRACE_FLAG_NEED_RESCHED_LAZY = 0x02, 192 TRACE_FLAG_NEED_RESCHED = 0x04, 193 TRACE_FLAG_HARDIRQ = 0x08, 194 TRACE_FLAG_SOFTIRQ = 0x10, 195 TRACE_FLAG_PREEMPT_RESCHED = 0x20, 196 TRACE_FLAG_NMI = 0x40, 197 TRACE_FLAG_BH_OFF = 0x80, 198 }; 199 200 static inline unsigned int tracing_gen_ctx_flags(unsigned long irqflags) 201 { 202 unsigned int irq_status = irqs_disabled_flags(irqflags) ? 203 TRACE_FLAG_IRQS_OFF : 0; 204 return tracing_gen_ctx_irq_test(irq_status); 205 } 206 static inline unsigned int tracing_gen_ctx(void) 207 { 208 unsigned long irqflags; 209 210 local_save_flags(irqflags); 211 return tracing_gen_ctx_flags(irqflags); 212 } 213 214 static inline unsigned int tracing_gen_ctx_dec(void) 215 { 216 unsigned int trace_ctx; 217 218 trace_ctx = tracing_gen_ctx(); 219 /* 220 * Subtract one from the preemption counter if preemption is enabled, 221 * see trace_event_buffer_reserve()for details. 222 */ 223 if (IS_ENABLED(CONFIG_PREEMPTION)) 224 trace_ctx--; 225 return trace_ctx; 226 } 227 228 struct trace_event_file; 229 230 struct ring_buffer_event * 231 trace_event_buffer_lock_reserve(struct trace_buffer **current_buffer, 232 struct trace_event_file *trace_file, 233 int type, unsigned long len, 234 unsigned int trace_ctx); 235 236 #define TRACE_RECORD_CMDLINE BIT(0) 237 #define TRACE_RECORD_TGID BIT(1) 238 239 void tracing_record_taskinfo(struct task_struct *task, int flags); 240 void tracing_record_taskinfo_sched_switch(struct task_struct *prev, 241 struct task_struct *next, int flags); 242 243 void tracing_record_cmdline(struct task_struct *task); 244 void tracing_record_tgid(struct task_struct *task); 245 246 int trace_output_call(struct trace_iterator *iter, char *name, char *fmt, ...) 247 __printf(3, 4); 248 249 struct event_filter; 250 251 enum trace_reg { 252 TRACE_REG_REGISTER, 253 TRACE_REG_UNREGISTER, 254 #ifdef CONFIG_PERF_EVENTS 255 TRACE_REG_PERF_REGISTER, 256 TRACE_REG_PERF_UNREGISTER, 257 TRACE_REG_PERF_OPEN, 258 TRACE_REG_PERF_CLOSE, 259 /* 260 * These (ADD/DEL) use a 'boolean' return value, where 1 (true) means a 261 * custom action was taken and the default action is not to be 262 * performed. 263 */ 264 TRACE_REG_PERF_ADD, 265 TRACE_REG_PERF_DEL, 266 #endif 267 }; 268 269 struct trace_event_call; 270 271 #define TRACE_FUNCTION_TYPE ((const char *)~0UL) 272 273 struct trace_event_fields { 274 const char *type; 275 union { 276 struct { 277 const char *name; 278 const int size; 279 const int align; 280 const unsigned int is_signed:1; 281 unsigned int needs_test:1; 282 const int filter_type; 283 const int len; 284 }; 285 int (*define_fields)(struct trace_event_call *); 286 }; 287 }; 288 289 struct trace_event_class { 290 const char *system; 291 void *probe; 292 #ifdef CONFIG_PERF_EVENTS 293 void *perf_probe; 294 #endif 295 int (*reg)(struct trace_event_call *event, 296 enum trace_reg type, void *data); 297 struct trace_event_fields *fields_array; 298 struct list_head *(*get_fields)(struct trace_event_call *); 299 struct list_head fields; 300 int (*raw_init)(struct trace_event_call *); 301 }; 302 303 extern int trace_event_reg(struct trace_event_call *event, 304 enum trace_reg type, void *data); 305 306 struct trace_event_buffer { 307 struct trace_buffer *buffer; 308 struct ring_buffer_event *event; 309 struct trace_event_file *trace_file; 310 void *entry; 311 unsigned int trace_ctx; 312 struct pt_regs *regs; 313 }; 314 315 void *trace_event_buffer_reserve(struct trace_event_buffer *fbuffer, 316 struct trace_event_file *trace_file, 317 unsigned long len); 318 319 void trace_event_buffer_commit(struct trace_event_buffer *fbuffer); 320 321 enum { 322 TRACE_EVENT_FL_CAP_ANY_BIT, 323 TRACE_EVENT_FL_NO_SET_FILTER_BIT, 324 TRACE_EVENT_FL_IGNORE_ENABLE_BIT, 325 TRACE_EVENT_FL_TRACEPOINT_BIT, 326 TRACE_EVENT_FL_DYNAMIC_BIT, 327 TRACE_EVENT_FL_KPROBE_BIT, 328 TRACE_EVENT_FL_UPROBE_BIT, 329 TRACE_EVENT_FL_EPROBE_BIT, 330 TRACE_EVENT_FL_FPROBE_BIT, 331 TRACE_EVENT_FL_CUSTOM_BIT, 332 TRACE_EVENT_FL_TEST_STR_BIT, 333 }; 334 335 /* 336 * Event flags: 337 * CAP_ANY - Any user can enable for perf 338 * NO_SET_FILTER - Set when filter has error and is to be ignored 339 * IGNORE_ENABLE - For trace internal events, do not enable with debugfs file 340 * TRACEPOINT - Event is a tracepoint 341 * DYNAMIC - Event is a dynamic event (created at run time) 342 * KPROBE - Event is a kprobe 343 * UPROBE - Event is a uprobe 344 * EPROBE - Event is an event probe 345 * FPROBE - Event is an function probe 346 * CUSTOM - Event is a custom event (to be attached to an exsiting tracepoint) 347 * This is set when the custom event has not been attached 348 * to a tracepoint yet, then it is cleared when it is. 349 * TEST_STR - The event has a "%s" that points to a string outside the event 350 */ 351 enum { 352 TRACE_EVENT_FL_CAP_ANY = (1 << TRACE_EVENT_FL_CAP_ANY_BIT), 353 TRACE_EVENT_FL_NO_SET_FILTER = (1 << TRACE_EVENT_FL_NO_SET_FILTER_BIT), 354 TRACE_EVENT_FL_IGNORE_ENABLE = (1 << TRACE_EVENT_FL_IGNORE_ENABLE_BIT), 355 TRACE_EVENT_FL_TRACEPOINT = (1 << TRACE_EVENT_FL_TRACEPOINT_BIT), 356 TRACE_EVENT_FL_DYNAMIC = (1 << TRACE_EVENT_FL_DYNAMIC_BIT), 357 TRACE_EVENT_FL_KPROBE = (1 << TRACE_EVENT_FL_KPROBE_BIT), 358 TRACE_EVENT_FL_UPROBE = (1 << TRACE_EVENT_FL_UPROBE_BIT), 359 TRACE_EVENT_FL_EPROBE = (1 << TRACE_EVENT_FL_EPROBE_BIT), 360 TRACE_EVENT_FL_FPROBE = (1 << TRACE_EVENT_FL_FPROBE_BIT), 361 TRACE_EVENT_FL_CUSTOM = (1 << TRACE_EVENT_FL_CUSTOM_BIT), 362 TRACE_EVENT_FL_TEST_STR = (1 << TRACE_EVENT_FL_TEST_STR_BIT), 363 }; 364 365 #define TRACE_EVENT_FL_UKPROBE (TRACE_EVENT_FL_KPROBE | TRACE_EVENT_FL_UPROBE) 366 367 struct trace_event_call { 368 struct list_head list; 369 struct trace_event_class *class; 370 union { 371 const char *name; 372 /* Set TRACE_EVENT_FL_TRACEPOINT flag when using "tp" */ 373 struct tracepoint *tp; 374 }; 375 struct trace_event event; 376 char *print_fmt; 377 /* 378 * Static events can disappear with modules, 379 * where as dynamic ones need their own ref count. 380 */ 381 union { 382 void *module; 383 atomic_t refcnt; 384 }; 385 void *data; 386 387 /* See the TRACE_EVENT_FL_* flags above */ 388 int flags; /* static flags of different events */ 389 390 #ifdef CONFIG_PERF_EVENTS 391 int perf_refcount; 392 struct hlist_head __percpu *perf_events; 393 struct bpf_prog_array __rcu *prog_array; 394 395 int (*perf_perm)(struct trace_event_call *, 396 struct perf_event *); 397 #endif 398 }; 399 400 #ifdef CONFIG_DYNAMIC_EVENTS 401 bool trace_event_dyn_try_get_ref(struct trace_event_call *call); 402 void trace_event_dyn_put_ref(struct trace_event_call *call); 403 bool trace_event_dyn_busy(struct trace_event_call *call); 404 #else 405 static inline bool trace_event_dyn_try_get_ref(struct trace_event_call *call) 406 { 407 /* Without DYNAMIC_EVENTS configured, nothing should be calling this */ 408 return false; 409 } 410 static inline void trace_event_dyn_put_ref(struct trace_event_call *call) 411 { 412 } 413 static inline bool trace_event_dyn_busy(struct trace_event_call *call) 414 { 415 /* Nothing should call this without DYNAIMIC_EVENTS configured. */ 416 return true; 417 } 418 #endif 419 420 static inline bool trace_event_try_get_ref(struct trace_event_call *call) 421 { 422 if (call->flags & TRACE_EVENT_FL_DYNAMIC) 423 return trace_event_dyn_try_get_ref(call); 424 else 425 return try_module_get(call->module); 426 } 427 428 static inline void trace_event_put_ref(struct trace_event_call *call) 429 { 430 if (call->flags & TRACE_EVENT_FL_DYNAMIC) 431 trace_event_dyn_put_ref(call); 432 else 433 module_put(call->module); 434 } 435 436 #ifdef CONFIG_PERF_EVENTS 437 static inline bool bpf_prog_array_valid(struct trace_event_call *call) 438 { 439 /* 440 * This inline function checks whether call->prog_array 441 * is valid or not. The function is called in various places, 442 * outside rcu_read_lock/unlock, as a heuristic to speed up execution. 443 * 444 * If this function returns true, and later call->prog_array 445 * becomes false inside rcu_read_lock/unlock region, 446 * we bail out then. If this function return false, 447 * there is a risk that we might miss a few events if the checking 448 * were delayed until inside rcu_read_lock/unlock region and 449 * call->prog_array happened to become non-NULL then. 450 * 451 * Here, READ_ONCE() is used instead of rcu_access_pointer(). 452 * rcu_access_pointer() requires the actual definition of 453 * "struct bpf_prog_array" while READ_ONCE() only needs 454 * a declaration of the same type. 455 */ 456 return !!READ_ONCE(call->prog_array); 457 } 458 #endif 459 460 static inline const char * 461 trace_event_name(struct trace_event_call *call) 462 { 463 if (call->flags & TRACE_EVENT_FL_CUSTOM) 464 return call->name; 465 else if (call->flags & TRACE_EVENT_FL_TRACEPOINT) 466 return call->tp ? call->tp->name : NULL; 467 else 468 return call->name; 469 } 470 471 static inline struct list_head * 472 trace_get_fields(struct trace_event_call *event_call) 473 { 474 if (!event_call->class->get_fields) 475 return &event_call->class->fields; 476 return event_call->class->get_fields(event_call); 477 } 478 479 struct trace_subsystem_dir; 480 481 enum { 482 EVENT_FILE_FL_ENABLED_BIT, 483 EVENT_FILE_FL_RECORDED_CMD_BIT, 484 EVENT_FILE_FL_RECORDED_TGID_BIT, 485 EVENT_FILE_FL_FILTERED_BIT, 486 EVENT_FILE_FL_NO_SET_FILTER_BIT, 487 EVENT_FILE_FL_SOFT_DISABLED_BIT, 488 EVENT_FILE_FL_TRIGGER_MODE_BIT, 489 EVENT_FILE_FL_TRIGGER_COND_BIT, 490 EVENT_FILE_FL_PID_FILTER_BIT, 491 EVENT_FILE_FL_WAS_ENABLED_BIT, 492 EVENT_FILE_FL_FREED_BIT, 493 }; 494 495 extern struct trace_event_file *trace_get_event_file(const char *instance, 496 const char *system, 497 const char *event); 498 extern void trace_put_event_file(struct trace_event_file *file); 499 500 #define MAX_DYNEVENT_CMD_LEN (2048) 501 502 enum dynevent_type { 503 DYNEVENT_TYPE_SYNTH = 1, 504 DYNEVENT_TYPE_KPROBE, 505 DYNEVENT_TYPE_NONE, 506 }; 507 508 struct dynevent_cmd; 509 510 typedef int (*dynevent_create_fn_t)(struct dynevent_cmd *cmd); 511 512 struct dynevent_cmd { 513 struct seq_buf seq; 514 const char *event_name; 515 unsigned int n_fields; 516 enum dynevent_type type; 517 dynevent_create_fn_t run_command; 518 void *private_data; 519 }; 520 521 extern int dynevent_create(struct dynevent_cmd *cmd); 522 523 extern int synth_event_delete(const char *name); 524 525 extern void synth_event_cmd_init(struct dynevent_cmd *cmd, 526 char *buf, int maxlen); 527 528 extern int __synth_event_gen_cmd_start(struct dynevent_cmd *cmd, 529 const char *name, 530 struct module *mod, ...); 531 532 #define synth_event_gen_cmd_start(cmd, name, mod, ...) \ 533 __synth_event_gen_cmd_start(cmd, name, mod, ## __VA_ARGS__, NULL) 534 535 struct synth_field_desc { 536 const char *type; 537 const char *name; 538 }; 539 540 extern int synth_event_gen_cmd_array_start(struct dynevent_cmd *cmd, 541 const char *name, 542 struct module *mod, 543 struct synth_field_desc *fields, 544 unsigned int n_fields); 545 extern int synth_event_create(const char *name, 546 struct synth_field_desc *fields, 547 unsigned int n_fields, struct module *mod); 548 549 extern int synth_event_add_field(struct dynevent_cmd *cmd, 550 const char *type, 551 const char *name); 552 extern int synth_event_add_field_str(struct dynevent_cmd *cmd, 553 const char *type_name); 554 extern int synth_event_add_fields(struct dynevent_cmd *cmd, 555 struct synth_field_desc *fields, 556 unsigned int n_fields); 557 558 #define synth_event_gen_cmd_end(cmd) \ 559 dynevent_create(cmd) 560 561 struct synth_event; 562 563 struct synth_event_trace_state { 564 struct trace_event_buffer fbuffer; 565 struct synth_trace_event *entry; 566 struct trace_buffer *buffer; 567 struct synth_event *event; 568 unsigned int cur_field; 569 unsigned int n_u64; 570 bool disabled; 571 bool add_next; 572 bool add_name; 573 }; 574 575 extern int synth_event_trace(struct trace_event_file *file, 576 unsigned int n_vals, ...); 577 extern int synth_event_trace_array(struct trace_event_file *file, u64 *vals, 578 unsigned int n_vals); 579 extern int synth_event_trace_start(struct trace_event_file *file, 580 struct synth_event_trace_state *trace_state); 581 extern int synth_event_add_next_val(u64 val, 582 struct synth_event_trace_state *trace_state); 583 extern int synth_event_add_val(const char *field_name, u64 val, 584 struct synth_event_trace_state *trace_state); 585 extern int synth_event_trace_end(struct synth_event_trace_state *trace_state); 586 587 extern int kprobe_event_delete(const char *name); 588 589 extern void kprobe_event_cmd_init(struct dynevent_cmd *cmd, 590 char *buf, int maxlen); 591 592 #define kprobe_event_gen_cmd_start(cmd, name, loc, ...) \ 593 __kprobe_event_gen_cmd_start(cmd, false, name, loc, ## __VA_ARGS__, NULL) 594 595 #define kretprobe_event_gen_cmd_start(cmd, name, loc, ...) \ 596 __kprobe_event_gen_cmd_start(cmd, true, name, loc, ## __VA_ARGS__, NULL) 597 598 extern int __kprobe_event_gen_cmd_start(struct dynevent_cmd *cmd, 599 bool kretprobe, 600 const char *name, 601 const char *loc, ...); 602 603 #define kprobe_event_add_fields(cmd, ...) \ 604 __kprobe_event_add_fields(cmd, ## __VA_ARGS__, NULL) 605 606 #define kprobe_event_add_field(cmd, field) \ 607 __kprobe_event_add_fields(cmd, field, NULL) 608 609 extern int __kprobe_event_add_fields(struct dynevent_cmd *cmd, ...); 610 611 #define kprobe_event_gen_cmd_end(cmd) \ 612 dynevent_create(cmd) 613 614 #define kretprobe_event_gen_cmd_end(cmd) \ 615 dynevent_create(cmd) 616 617 /* 618 * Event file flags: 619 * ENABLED - The event is enabled 620 * RECORDED_CMD - The comms should be recorded at sched_switch 621 * RECORDED_TGID - The tgids should be recorded at sched_switch 622 * FILTERED - The event has a filter attached 623 * NO_SET_FILTER - Set when filter has error and is to be ignored 624 * SOFT_DISABLED - When set, do not trace the event (even though its 625 * tracepoint may be enabled) 626 * TRIGGER_MODE - When set, invoke the triggers associated with the event 627 * TRIGGER_COND - When set, one or more triggers has an associated filter 628 * PID_FILTER - When set, the event is filtered based on pid 629 * WAS_ENABLED - Set when enabled to know to clear trace on module removal 630 * FREED - File descriptor is freed, all fields should be considered invalid 631 */ 632 enum { 633 EVENT_FILE_FL_ENABLED = (1 << EVENT_FILE_FL_ENABLED_BIT), 634 EVENT_FILE_FL_RECORDED_CMD = (1 << EVENT_FILE_FL_RECORDED_CMD_BIT), 635 EVENT_FILE_FL_RECORDED_TGID = (1 << EVENT_FILE_FL_RECORDED_TGID_BIT), 636 EVENT_FILE_FL_FILTERED = (1 << EVENT_FILE_FL_FILTERED_BIT), 637 EVENT_FILE_FL_NO_SET_FILTER = (1 << EVENT_FILE_FL_NO_SET_FILTER_BIT), 638 EVENT_FILE_FL_SOFT_DISABLED = (1 << EVENT_FILE_FL_SOFT_DISABLED_BIT), 639 EVENT_FILE_FL_TRIGGER_MODE = (1 << EVENT_FILE_FL_TRIGGER_MODE_BIT), 640 EVENT_FILE_FL_TRIGGER_COND = (1 << EVENT_FILE_FL_TRIGGER_COND_BIT), 641 EVENT_FILE_FL_PID_FILTER = (1 << EVENT_FILE_FL_PID_FILTER_BIT), 642 EVENT_FILE_FL_WAS_ENABLED = (1 << EVENT_FILE_FL_WAS_ENABLED_BIT), 643 EVENT_FILE_FL_FREED = (1 << EVENT_FILE_FL_FREED_BIT), 644 }; 645 646 struct trace_event_file { 647 struct list_head list; 648 struct trace_event_call *event_call; 649 struct event_filter __rcu *filter; 650 struct eventfs_inode *ei; 651 struct trace_array *tr; 652 struct trace_subsystem_dir *system; 653 struct list_head triggers; 654 655 /* 656 * 32 bit flags: 657 * bit 0: enabled 658 * bit 1: enabled cmd record 659 * bit 2: enable/disable with the soft disable bit 660 * bit 3: soft disabled 661 * bit 4: trigger enabled 662 * 663 * Note: The bits must be set atomically to prevent races 664 * from other writers. Reads of flags do not need to be in 665 * sync as they occur in critical sections. But the way flags 666 * is currently used, these changes do not affect the code 667 * except that when a change is made, it may have a slight 668 * delay in propagating the changes to other CPUs due to 669 * caching and such. Which is mostly OK ;-) 670 */ 671 unsigned long flags; 672 refcount_t ref; /* ref count for opened files */ 673 atomic_t sm_ref; /* soft-mode reference counter */ 674 atomic_t tm_ref; /* trigger-mode reference counter */ 675 }; 676 677 #ifdef CONFIG_HIST_TRIGGERS 678 extern struct irq_work hist_poll_work; 679 extern wait_queue_head_t hist_poll_wq; 680 681 static inline void hist_poll_wakeup(void) 682 { 683 if (wq_has_sleeper(&hist_poll_wq)) 684 irq_work_queue(&hist_poll_work); 685 } 686 687 #define hist_poll_wait(file, wait) \ 688 poll_wait(file, &hist_poll_wq, wait) 689 690 #else 691 static inline void hist_poll_wakeup(void) 692 { 693 } 694 #endif 695 696 #define __TRACE_EVENT_FLAGS(name, value) \ 697 static int __init trace_init_flags_##name(void) \ 698 { \ 699 event_##name.flags |= value; \ 700 return 0; \ 701 } \ 702 early_initcall(trace_init_flags_##name); 703 704 #define __TRACE_EVENT_PERF_PERM(name, expr...) \ 705 static int perf_perm_##name(struct trace_event_call *tp_event, \ 706 struct perf_event *p_event) \ 707 { \ 708 return ({ expr; }); \ 709 } \ 710 static int __init trace_init_perf_perm_##name(void) \ 711 { \ 712 event_##name.perf_perm = &perf_perm_##name; \ 713 return 0; \ 714 } \ 715 early_initcall(trace_init_perf_perm_##name); 716 717 #define PERF_MAX_TRACE_SIZE 8192 718 719 #define MAX_FILTER_STR_VAL 256U /* Should handle KSYM_SYMBOL_LEN */ 720 721 enum event_trigger_type { 722 ETT_NONE = (0), 723 ETT_TRACE_ONOFF = (1 << 0), 724 ETT_SNAPSHOT = (1 << 1), 725 ETT_STACKTRACE = (1 << 2), 726 ETT_EVENT_ENABLE = (1 << 3), 727 ETT_EVENT_HIST = (1 << 4), 728 ETT_HIST_ENABLE = (1 << 5), 729 ETT_EVENT_EPROBE = (1 << 6), 730 }; 731 732 extern int filter_match_preds(struct event_filter *filter, void *rec); 733 734 extern enum event_trigger_type 735 event_triggers_call(struct trace_event_file *file, 736 struct trace_buffer *buffer, void *rec, 737 struct ring_buffer_event *event); 738 extern void 739 event_triggers_post_call(struct trace_event_file *file, 740 enum event_trigger_type tt); 741 742 bool trace_event_ignore_this_pid(struct trace_event_file *trace_file); 743 744 bool __trace_trigger_soft_disabled(struct trace_event_file *file); 745 746 /** 747 * trace_trigger_soft_disabled - do triggers and test if soft disabled 748 * @file: The file pointer of the event to test 749 * 750 * If any triggers without filters are attached to this event, they 751 * will be called here. If the event is soft disabled and has no 752 * triggers that require testing the fields, it will return true, 753 * otherwise false. 754 */ 755 static __always_inline bool 756 trace_trigger_soft_disabled(struct trace_event_file *file) 757 { 758 unsigned long eflags = file->flags; 759 760 if (likely(!(eflags & (EVENT_FILE_FL_TRIGGER_MODE | 761 EVENT_FILE_FL_SOFT_DISABLED | 762 EVENT_FILE_FL_PID_FILTER)))) 763 return false; 764 765 if (likely(eflags & EVENT_FILE_FL_TRIGGER_COND)) 766 return false; 767 768 return __trace_trigger_soft_disabled(file); 769 } 770 771 #ifdef CONFIG_BPF_EVENTS 772 unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx); 773 int perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie); 774 void perf_event_detach_bpf_prog(struct perf_event *event); 775 int perf_event_query_prog_array(struct perf_event *event, void __user *info); 776 777 struct bpf_raw_tp_link; 778 int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link); 779 int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link); 780 781 struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name); 782 void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp); 783 int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, 784 u32 *fd_type, const char **buf, 785 u64 *probe_offset, u64 *probe_addr, 786 unsigned long *missed); 787 int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog); 788 int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog); 789 #else 790 static inline unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx) 791 { 792 return 1; 793 } 794 795 static inline int 796 perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie) 797 { 798 return -EOPNOTSUPP; 799 } 800 801 static inline void perf_event_detach_bpf_prog(struct perf_event *event) { } 802 803 static inline int 804 perf_event_query_prog_array(struct perf_event *event, void __user *info) 805 { 806 return -EOPNOTSUPP; 807 } 808 struct bpf_raw_tp_link; 809 static inline int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link) 810 { 811 return -EOPNOTSUPP; 812 } 813 static inline int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link) 814 { 815 return -EOPNOTSUPP; 816 } 817 static inline struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name) 818 { 819 return NULL; 820 } 821 static inline void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp) 822 { 823 } 824 static inline int bpf_get_perf_event_info(const struct perf_event *event, 825 u32 *prog_id, u32 *fd_type, 826 const char **buf, u64 *probe_offset, 827 u64 *probe_addr, unsigned long *missed) 828 { 829 return -EOPNOTSUPP; 830 } 831 static inline int 832 bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 833 { 834 return -EOPNOTSUPP; 835 } 836 static inline int 837 bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 838 { 839 return -EOPNOTSUPP; 840 } 841 #endif 842 843 enum { 844 FILTER_OTHER = 0, 845 FILTER_STATIC_STRING, 846 FILTER_DYN_STRING, 847 FILTER_RDYN_STRING, 848 FILTER_PTR_STRING, 849 FILTER_TRACE_FN, 850 FILTER_CPUMASK, 851 FILTER_COMM, 852 FILTER_CPU, 853 FILTER_STACKTRACE, 854 }; 855 856 extern int trace_event_raw_init(struct trace_event_call *call); 857 extern int trace_define_field(struct trace_event_call *call, const char *type, 858 const char *name, int offset, int size, 859 int is_signed, int filter_type); 860 extern int trace_add_event_call(struct trace_event_call *call); 861 extern int trace_remove_event_call(struct trace_event_call *call); 862 extern int trace_event_get_offsets(struct trace_event_call *call); 863 864 int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set); 865 int trace_set_clr_event(const char *system, const char *event, int set); 866 int trace_array_set_clr_event(struct trace_array *tr, const char *system, 867 const char *event, bool enable); 868 869 #ifdef CONFIG_PERF_EVENTS 870 struct perf_event; 871 872 DECLARE_PER_CPU(struct pt_regs, perf_trace_regs); 873 874 extern int perf_trace_init(struct perf_event *event); 875 extern void perf_trace_destroy(struct perf_event *event); 876 extern int perf_trace_add(struct perf_event *event, int flags); 877 extern void perf_trace_del(struct perf_event *event, int flags); 878 #ifdef CONFIG_KPROBE_EVENTS 879 extern int perf_kprobe_init(struct perf_event *event, bool is_retprobe); 880 extern void perf_kprobe_destroy(struct perf_event *event); 881 extern int bpf_get_kprobe_info(const struct perf_event *event, 882 u32 *fd_type, const char **symbol, 883 u64 *probe_offset, u64 *probe_addr, 884 unsigned long *missed, 885 bool perf_type_tracepoint); 886 #endif 887 #ifdef CONFIG_UPROBE_EVENTS 888 extern int perf_uprobe_init(struct perf_event *event, 889 unsigned long ref_ctr_offset, bool is_retprobe); 890 extern void perf_uprobe_destroy(struct perf_event *event); 891 extern int bpf_get_uprobe_info(const struct perf_event *event, 892 u32 *fd_type, const char **filename, 893 u64 *probe_offset, u64 *probe_addr, 894 bool perf_type_tracepoint); 895 #endif 896 extern int ftrace_profile_set_filter(struct perf_event *event, int event_id, 897 char *filter_str); 898 extern void ftrace_profile_free_filter(struct perf_event *event); 899 void perf_trace_buf_update(void *record, u16 type); 900 void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp); 901 902 int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie); 903 void perf_event_free_bpf_prog(struct perf_event *event); 904 905 void bpf_trace_run1(struct bpf_raw_tp_link *link, u64 arg1); 906 void bpf_trace_run2(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2); 907 void bpf_trace_run3(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2, 908 u64 arg3); 909 void bpf_trace_run4(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2, 910 u64 arg3, u64 arg4); 911 void bpf_trace_run5(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2, 912 u64 arg3, u64 arg4, u64 arg5); 913 void bpf_trace_run6(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2, 914 u64 arg3, u64 arg4, u64 arg5, u64 arg6); 915 void bpf_trace_run7(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2, 916 u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7); 917 void bpf_trace_run8(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2, 918 u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, 919 u64 arg8); 920 void bpf_trace_run9(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2, 921 u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, 922 u64 arg8, u64 arg9); 923 void bpf_trace_run10(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2, 924 u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, 925 u64 arg8, u64 arg9, u64 arg10); 926 void bpf_trace_run11(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2, 927 u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, 928 u64 arg8, u64 arg9, u64 arg10, u64 arg11); 929 void bpf_trace_run12(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2, 930 u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7, 931 u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12); 932 void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx, 933 struct trace_event_call *call, u64 count, 934 struct pt_regs *regs, struct hlist_head *head, 935 struct task_struct *task); 936 937 static inline void 938 perf_trace_buf_submit(void *raw_data, int size, int rctx, u16 type, 939 u64 count, struct pt_regs *regs, void *head, 940 struct task_struct *task) 941 { 942 perf_tp_event(type, count, raw_data, size, regs, head, rctx, task); 943 } 944 945 #endif 946 947 #define TRACE_EVENT_STR_MAX 512 948 949 /* 950 * gcc warns that you can not use a va_list in an inlined 951 * function. But lets me make it into a macro :-/ 952 */ 953 #define __trace_event_vstr_len(fmt, va) \ 954 ({ \ 955 va_list __ap; \ 956 int __ret; \ 957 \ 958 va_copy(__ap, *(va)); \ 959 __ret = vsnprintf(NULL, 0, fmt, __ap) + 1; \ 960 va_end(__ap); \ 961 \ 962 min(__ret, TRACE_EVENT_STR_MAX); \ 963 }) 964 965 #endif /* _LINUX_TRACE_EVENT_H */ 966 967 /* 968 * Note: we keep the TRACE_CUSTOM_EVENT outside the include file ifdef protection. 969 * This is due to the way trace custom events work. If a file includes two 970 * trace event headers under one "CREATE_CUSTOM_TRACE_EVENTS" the first include 971 * will override the TRACE_CUSTOM_EVENT and break the second include. 972 */ 973 974 #ifndef TRACE_CUSTOM_EVENT 975 976 #define DECLARE_CUSTOM_EVENT_CLASS(name, proto, args, tstruct, assign, print) 977 #define DEFINE_CUSTOM_EVENT(template, name, proto, args) 978 #define TRACE_CUSTOM_EVENT(name, proto, args, struct, assign, print) 979 980 #endif /* ifdef TRACE_CUSTOM_EVENT (see note above) */ 981