69 static void trace_note(struct blk_trace *bt, pid_t pid, int action,  in trace_note()  argument
92 	if (!bt->rchan)  in trace_note()
95 	t = relay_reserve(bt->rchan, sizeof(*t) + len + cgid_len);  in trace_note()
100 		t->device = bt->dev;  in trace_note()
121 	struct blk_trace *bt;  in trace_note_tsk()  local
125 	list_for_each_entry(bt, &running_trace_list, running_list) {  in trace_note_tsk()
126 		trace_note(bt, tsk->pid, BLK_TN_PROCESS, tsk->comm,  in trace_note_tsk()
132 static void trace_note_time(struct blk_trace *bt)  in trace_note_time()  argument
144 	trace_note(bt, 0, BLK_TN_TIMESTAMP, words, sizeof(words), 0);  in trace_note_time()
148 void __trace_note_message(struct blk_trace *bt, struct blkcg *blkcg,  in __trace_note_message()  argument
156 	if (unlikely(bt->trace_state != Blktrace_running &&  in __trace_note_message()
164 	if (!(bt->act_mask & BLK_TC_NOTIFY))  in __trace_note_message()
168 	buf = this_cpu_ptr(bt->msg_data);  in __trace_note_message()
176 	trace_note(bt, current->pid, BLK_TN_MESSAGE, buf, n,  in __trace_note_message()
179 	trace_note(bt, current->pid, BLK_TN_MESSAGE, buf, n, 0);  in __trace_note_message()
185 static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector,  in act_log_check()  argument
188 	if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0)  in act_log_check()
190 	if (sector && (sector < bt->start_lba || sector > bt->end_lba))  in act_log_check()
192 	if (bt->pid && pid != bt->pid)  in act_log_check()
215 static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,  in __blk_add_trace()  argument
230 	if (unlikely(bt->trace_state != Blktrace_running && !blk_tracer))  in __blk_add_trace()
247 	if (act_log_check(bt, what, sector, pid))  in __blk_add_trace()
274 	t = relay_reserve(bt->rchan, sizeof(*t) + pdu_len + cgid_len);  in __blk_add_trace()
276 		sequence = per_cpu_ptr(bt->sequence, cpu);  in __blk_add_trace()
294 		t->device = bt->dev;  in __blk_add_trace()
312 static void blk_trace_free(struct blk_trace *bt)  in blk_trace_free()  argument
314 	debugfs_remove(bt->msg_file);  in blk_trace_free()
315 	debugfs_remove(bt->dropped_file);  in blk_trace_free()
316 	relay_close(bt->rchan);  in blk_trace_free()
317 	debugfs_remove(bt->dir);  in blk_trace_free()
318 	free_percpu(bt->sequence);  in blk_trace_free()
319 	free_percpu(bt->msg_data);  in blk_trace_free()
320 	kfree(bt);  in blk_trace_free()
339 static void blk_trace_cleanup(struct blk_trace *bt)  in blk_trace_cleanup()  argument
342 	blk_trace_free(bt);  in blk_trace_cleanup()
348 	struct blk_trace *bt;  in __blk_trace_remove()  local
350 	bt = rcu_replace_pointer(q->blk_trace, NULL,  in __blk_trace_remove()
352 	if (!bt)  in __blk_trace_remove()
355 	if (bt->trace_state != Blktrace_running)  in __blk_trace_remove()
356 		blk_trace_cleanup(bt);  in __blk_trace_remove()
376 	struct blk_trace *bt = filp->private_data;  in blk_dropped_read()  local
379 	snprintf(buf, sizeof(buf), "%u\n", atomic_read(&bt->dropped));  in blk_dropped_read()
395 	struct blk_trace *bt;  in blk_msg_write()  local
404 	bt = filp->private_data;  in blk_msg_write()
405 	__trace_note_message(bt, NULL, "%s", msg);  in blk_msg_write()
425 	struct blk_trace *bt;  in blk_subbuf_start_callback()  local
430 	bt = buf->chan->private_data;  in blk_subbuf_start_callback()
431 	atomic_inc(&bt->dropped);  in blk_subbuf_start_callback()
458 static void blk_trace_setup_lba(struct blk_trace *bt,  in blk_trace_setup_lba()  argument
467 		bt->start_lba = part->start_sect;  in blk_trace_setup_lba()
468 		bt->end_lba = part->start_sect + part->nr_sects;  in blk_trace_setup_lba()
470 		bt->start_lba = 0;  in blk_trace_setup_lba()
471 		bt->end_lba = -1ULL;  in blk_trace_setup_lba()
482 	struct blk_trace *bt = NULL;  in do_blk_trace_setup()  local
511 	bt = kzalloc(sizeof(*bt), GFP_KERNEL);  in do_blk_trace_setup()
512 	if (!bt)  in do_blk_trace_setup()
516 	bt->sequence = alloc_percpu(unsigned long);  in do_blk_trace_setup()
517 	if (!bt->sequence)  in do_blk_trace_setup()
520 	bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char));  in do_blk_trace_setup()
521 	if (!bt->msg_data)  in do_blk_trace_setup()
533 		bt->dir = dir = debugfs_create_dir(buts->name, blk_debugfs_root);  in do_blk_trace_setup()
547 	bt->dev = dev;  in do_blk_trace_setup()
548 	atomic_set(&bt->dropped, 0);  in do_blk_trace_setup()
549 	INIT_LIST_HEAD(&bt->running_list);  in do_blk_trace_setup()
552 	bt->dropped_file = debugfs_create_file("dropped", 0444, dir, bt,  in do_blk_trace_setup()
555 	bt->msg_file = debugfs_create_file("msg", 0222, dir, bt, &blk_msg_fops);  in do_blk_trace_setup()
557 	bt->rchan = relay_open("trace", dir, buts->buf_size,  in do_blk_trace_setup()
558 				buts->buf_nr, &blk_relay_callbacks, bt);  in do_blk_trace_setup()
559 	if (!bt->rchan)  in do_blk_trace_setup()
562 	bt->act_mask = buts->act_mask;  in do_blk_trace_setup()
563 	if (!bt->act_mask)  in do_blk_trace_setup()
564 		bt->act_mask = (u16) -1;  in do_blk_trace_setup()
566 	blk_trace_setup_lba(bt, bdev);  in do_blk_trace_setup()
570 		bt->start_lba = buts->start_lba;  in do_blk_trace_setup()
572 		bt->end_lba = buts->end_lba;  in do_blk_trace_setup()
574 	bt->pid = buts->pid;  in do_blk_trace_setup()
575 	bt->trace_state = Blktrace_setup;  in do_blk_trace_setup()
577 	rcu_assign_pointer(q->blk_trace, bt);  in do_blk_trace_setup()
583 		blk_trace_free(bt);  in do_blk_trace_setup()
659 	struct blk_trace *bt;  in __blk_trace_startstop()  local
661 	bt = rcu_dereference_protected(q->blk_trace,  in __blk_trace_startstop()
663 	if (bt == NULL)  in __blk_trace_startstop()
672 		if (bt->trace_state == Blktrace_setup ||  in __blk_trace_startstop()
673 		    bt->trace_state == Blktrace_stopped) {  in __blk_trace_startstop()
676 			bt->trace_state = Blktrace_running;  in __blk_trace_startstop()
678 			list_add(&bt->running_list, &running_trace_list);  in __blk_trace_startstop()
681 			trace_note_time(bt);  in __blk_trace_startstop()
685 		if (bt->trace_state == Blktrace_running) {  in __blk_trace_startstop()
686 			bt->trace_state = Blktrace_stopped;  in __blk_trace_startstop()
688 			list_del_init(&bt->running_list);  in __blk_trace_startstop()
690 			relay_flush(bt->rchan);  in __blk_trace_startstop()
784 	struct blk_trace *bt;  in blk_trace_bio_get_cgid()  local
786 	/* We don't use the 'bt' value here except as an optimization... */  in blk_trace_bio_get_cgid()
787 	bt = rcu_dereference_protected(q->blk_trace, 1);  in blk_trace_bio_get_cgid()
788 	if (!bt || !(blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP))  in blk_trace_bio_get_cgid()
830 	struct blk_trace *bt;  in blk_add_trace_rq()  local
833 	bt = rcu_dereference(rq->q->blk_trace);  in blk_add_trace_rq()
834 	if (likely(!bt)) {  in blk_add_trace_rq()
844 	__blk_add_trace(bt, blk_rq_trace_sector(rq), nr_bytes, req_op(rq),  in blk_add_trace_rq()
899 	struct blk_trace *bt;  in blk_add_trace_bio()  local
902 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_bio()
903 	if (likely(!bt)) {  in blk_add_trace_bio()
908 	__blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,  in blk_add_trace_bio()
956 		struct blk_trace *bt;  in blk_add_trace_getrq()  local
959 		bt = rcu_dereference(q->blk_trace);  in blk_add_trace_getrq()
960 		if (bt)  in blk_add_trace_getrq()
961 			__blk_add_trace(bt, 0, 0, rw, 0, BLK_TA_GETRQ, 0, 0,  in blk_add_trace_getrq()
975 		struct blk_trace *bt;  in blk_add_trace_sleeprq()  local
978 		bt = rcu_dereference(q->blk_trace);  in blk_add_trace_sleeprq()
979 		if (bt)  in blk_add_trace_sleeprq()
980 			__blk_add_trace(bt, 0, 0, rw, 0, BLK_TA_SLEEPRQ,  in blk_add_trace_sleeprq()
988 	struct blk_trace *bt;  in blk_add_trace_plug()  local
991 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_plug()
992 	if (bt)  in blk_add_trace_plug()
993 		__blk_add_trace(bt, 0, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL, 0);  in blk_add_trace_plug()
1000 	struct blk_trace *bt;  in blk_add_trace_unplug()  local
1003 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_unplug()
1004 	if (bt) {  in blk_add_trace_unplug()
1013 		__blk_add_trace(bt, 0, 0, 0, 0, what, 0, sizeof(rpdu), &rpdu, 0);  in blk_add_trace_unplug()
1022 	struct blk_trace *bt;  in blk_add_trace_split()  local
1025 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_split()
1026 	if (bt) {  in blk_add_trace_split()
1029 		__blk_add_trace(bt, bio->bi_iter.bi_sector,  in blk_add_trace_split()
1056 	struct blk_trace *bt;  in blk_add_trace_bio_remap()  local
1060 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_bio_remap()
1061 	if (likely(!bt)) {  in blk_add_trace_bio_remap()
1070 	__blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,  in blk_add_trace_bio_remap()
1095 	struct blk_trace *bt;  in blk_add_trace_rq_remap()  local
1099 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_rq_remap()
1100 	if (likely(!bt)) {  in blk_add_trace_rq_remap()
1109 	__blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq),  in blk_add_trace_rq_remap()
1130 	struct blk_trace *bt;  in blk_add_driver_data()  local
1133 	bt = rcu_dereference(q->blk_trace);  in blk_add_driver_data()
1134 	if (likely(!bt)) {  in blk_add_driver_data()
1139 	__blk_add_trace(bt, blk_rq_trace_sector(rq), blk_rq_bytes(rq), 0, 0,  in blk_add_driver_data()
1666 	struct blk_trace *bt;  in blk_trace_remove_queue()  local
1668 	bt = rcu_replace_pointer(q->blk_trace, NULL,  in blk_trace_remove_queue()
1670 	if (bt == NULL)  in blk_trace_remove_queue()
1675 	blk_trace_free(bt);  in blk_trace_remove_queue()
1685 	struct blk_trace *bt = NULL;  in blk_trace_setup_queue()  local
1688 	bt = kzalloc(sizeof(*bt), GFP_KERNEL);  in blk_trace_setup_queue()
1689 	if (!bt)  in blk_trace_setup_queue()
1692 	bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char));  in blk_trace_setup_queue()
1693 	if (!bt->msg_data)  in blk_trace_setup_queue()
1696 	bt->dev = bdev->bd_dev;  in blk_trace_setup_queue()
1697 	bt->act_mask = (u16)-1;  in blk_trace_setup_queue()
1699 	blk_trace_setup_lba(bt, bdev);  in blk_trace_setup_queue()
1701 	rcu_assign_pointer(q->blk_trace, bt);  in blk_trace_setup_queue()
1706 	blk_trace_free(bt);  in blk_trace_setup_queue()
1832 	struct blk_trace *bt;  in sysfs_blk_trace_attr_show()  local
1844 	bt = rcu_dereference_protected(q->blk_trace,  in sysfs_blk_trace_attr_show()
1847 		ret = sprintf(buf, "%u\n", !!bt);  in sysfs_blk_trace_attr_show()
1851 	if (bt == NULL)  in sysfs_blk_trace_attr_show()
1854 		ret = blk_trace_mask2str(buf, bt->act_mask);  in sysfs_blk_trace_attr_show()
1856 		ret = sprintf(buf, "%u\n", bt->pid);  in sysfs_blk_trace_attr_show()
1858 		ret = sprintf(buf, "%llu\n", bt->start_lba);  in sysfs_blk_trace_attr_show()
1860 		ret = sprintf(buf, "%llu\n", bt->end_lba);  in sysfs_blk_trace_attr_show()
1876 	struct blk_trace *bt;  in sysfs_blk_trace_attr_store()  local
1905 	bt = rcu_dereference_protected(q->blk_trace,  in sysfs_blk_trace_attr_store()
1908 		if (!!value == !!bt) {  in sysfs_blk_trace_attr_store()
1920 	if (bt == NULL) {  in sysfs_blk_trace_attr_store()
1922 		bt = rcu_dereference_protected(q->blk_trace,  in sysfs_blk_trace_attr_store()
1928 			bt->act_mask = value;  in sysfs_blk_trace_attr_store()
1930 			bt->pid = value;  in sysfs_blk_trace_attr_store()
1932 			bt->start_lba = value;  in sysfs_blk_trace_attr_store()
1934 			bt->end_lba = value;  in sysfs_blk_trace_attr_store()