16 static bool rcu_rdp_is_offloaded(struct rcu_data *rdp)  in rcu_rdp_is_offloaded()  argument
30 		  lockdep_is_held(&rdp->nocb_lock) ||  in rcu_rdp_is_offloaded()
33 		   rdp == this_cpu_ptr(&rcu_data)) ||  in rcu_rdp_is_offloaded()
34 		  rcu_current_is_nocb_kthread(rdp)),  in rcu_rdp_is_offloaded()
38 	return rcu_segcblist_is_offloaded(&rdp->cblist);  in rcu_rdp_is_offloaded()
162 static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp)  in rcu_preempt_ctxt_queue()  argument
167 			 (rnp->qsmask & rdp->grpmask ? RCU_GP_BLKD : 0) +  in rcu_preempt_ctxt_queue()
168 			 (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0);  in rcu_preempt_ctxt_queue()
172 	WARN_ON_ONCE(rdp->mynode != rnp);  in rcu_preempt_ctxt_queue()
176 		     rdp->grpmask);  in rcu_preempt_ctxt_queue()
261 		     !(rnp->qsmask & rdp->grpmask));  in rcu_preempt_ctxt_queue()
263 		     !(rnp->expmask & rdp->grpmask));  in rcu_preempt_ctxt_queue()
274 	if (blkd_state & RCU_EXP_BLKD && rdp->cpu_no_qs.b.exp)  in rcu_preempt_ctxt_queue()
275 		rcu_report_exp_rdp(rdp);  in rcu_preempt_ctxt_queue()
277 		WARN_ON_ONCE(rdp->cpu_no_qs.b.exp);  in rcu_preempt_ctxt_queue()
278 	ASSERT_EXCLUSIVE_WRITER_SCOPED(rdp->cpu_no_qs.b.exp);  in rcu_preempt_ctxt_queue()
327 	struct rcu_data *rdp = this_cpu_ptr(&rcu_data);  in rcu_note_context_switch()  local
337 		rnp = rdp->mynode;  in rcu_note_context_switch()
347 		WARN_ON_ONCE(!rcu_rdp_cpu_online(rdp));  in rcu_note_context_switch()
351 				       (rnp->qsmask & rdp->grpmask)  in rcu_note_context_switch()
354 		rcu_preempt_ctxt_queue(rnp, rdp);  in rcu_note_context_switch()
369 	if (rdp->cpu_no_qs.b.exp)  in rcu_note_context_switch()
370 		rcu_report_exp_rdp(rdp);  in rcu_note_context_switch()
485 	struct rcu_data *rdp;  in rcu_preempt_deferred_qs_irqrestore()  local
489 	rdp = this_cpu_ptr(&rcu_data);  in rcu_preempt_deferred_qs_irqrestore()
490 	if (rdp->defer_qs_pending == DEFER_QS_PENDING)  in rcu_preempt_deferred_qs_irqrestore()
491 		rdp->defer_qs_pending = DEFER_QS_IDLE;  in rcu_preempt_deferred_qs_irqrestore()
499 	if (!special.s && !rdp->cpu_no_qs.b.exp) {  in rcu_preempt_deferred_qs_irqrestore()
506 			rdp->cpu_no_qs.b.norm = false;  in rcu_preempt_deferred_qs_irqrestore()
507 			rcu_report_qs_rdp(rdp);  in rcu_preempt_deferred_qs_irqrestore()
520 	if (rdp->cpu_no_qs.b.exp)  in rcu_preempt_deferred_qs_irqrestore()
521 		rcu_report_exp_rdp(rdp);  in rcu_preempt_deferred_qs_irqrestore()
629 	struct rcu_data *rdp;  in rcu_preempt_deferred_qs_handler()  local
632 	rdp = container_of(iwp, struct rcu_data, defer_qs_iw);  in rcu_preempt_deferred_qs_handler()
648 		WRITE_ONCE(rdp->defer_qs_pending, DEFER_QS_IDLE);  in rcu_preempt_deferred_qs_handler()
669 				      struct rcu_data *rdp,  in rcu_unlock_needs_exp_handling()  argument
690 	if (rdp->grpmask & READ_ONCE(rnp->expmask))  in rcu_unlock_needs_exp_handling()
702 	    ((rdp->grpmask & READ_ONCE(rnp->qsmask)) || t->rcu_blocked_node))  in rcu_unlock_needs_exp_handling()
740 		struct rcu_data *rdp = this_cpu_ptr(&rcu_data);  in rcu_read_unlock_special()  local
741 		struct rcu_node *rnp = rdp->mynode;  in rcu_read_unlock_special()
743 		needs_exp = rcu_unlock_needs_exp_handling(t, rdp, rnp, irqs_were_disabled);  in rcu_read_unlock_special()
750 			if (rdp->defer_qs_pending != DEFER_QS_PENDING) {  in rcu_read_unlock_special()
751 				rdp->defer_qs_pending = DEFER_QS_PENDING;  in rcu_read_unlock_special()
761 			    needs_exp && rdp->defer_qs_pending != DEFER_QS_PENDING &&  in rcu_read_unlock_special()
762 			    cpu_online(rdp->cpu)) {  in rcu_read_unlock_special()
765 				rdp->defer_qs_pending = DEFER_QS_PENDING;  in rcu_read_unlock_special()
766 				irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu);  in rcu_read_unlock_special()
872 	struct rcu_data *rdp;  in dump_blkd_tasks()  local
894 		rdp = per_cpu_ptr(&rcu_data, cpu);  in dump_blkd_tasks()
896 			cpu, ".o"[rcu_rdp_cpu_online(rdp)],  in dump_blkd_tasks()
897 			(long)rdp->rcu_onl_gp_seq, rdp->rcu_onl_gp_state,  in dump_blkd_tasks()
898 			(long)rdp->rcu_ofl_gp_seq, rdp->rcu_ofl_gp_state);  in dump_blkd_tasks()
902 static void rcu_preempt_deferred_qs_init(struct rcu_data *rdp)  in rcu_preempt_deferred_qs_init()  argument
904 	rdp->defer_qs_iw = IRQ_WORK_INIT_HARD(rcu_preempt_deferred_qs_handler);  in rcu_preempt_deferred_qs_init()
915 	struct rcu_data *rdp;  in rcu_read_unlock_strict()  local
928 	rdp = this_cpu_ptr(&rcu_data);  in rcu_read_unlock_strict()
929 	rdp->cpu_no_qs.b.norm = false;  in rcu_read_unlock_strict()
930 	rcu_report_qs_rdp(rdp);  in rcu_read_unlock_strict()
1046 	struct rcu_data *rdp = this_cpu_ptr(&rcu_data);  in rcu_preempt_deferred_qs()  local
1048 	if (READ_ONCE(rdp->cpu_no_qs.b.exp))  in rcu_preempt_deferred_qs()
1049 		rcu_report_exp_rdp(rdp);  in rcu_preempt_deferred_qs()
1105 static void rcu_preempt_deferred_qs_init(struct rcu_data *rdp) { }  in rcu_preempt_deferred_qs_init()  argument
1114 	struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);  in rcu_cpu_kthread_setup()  local
1122 	WRITE_ONCE(rdp->rcuc_activity, jiffies);  in rcu_cpu_kthread_setup()
1125 static bool rcu_is_callbacks_nocb_kthread(struct rcu_data *rdp)  in rcu_is_callbacks_nocb_kthread()  argument
1128 	return rdp->nocb_cb_kthread == current;  in rcu_is_callbacks_nocb_kthread()
1138 static bool rcu_is_callbacks_kthread(struct rcu_data *rdp)  in rcu_is_callbacks_kthread()  argument
1140 	return rdp->rcu_cpu_kthread_task == current ||  in rcu_is_callbacks_kthread()
1141 			rcu_is_callbacks_nocb_kthread(rdp);  in rcu_is_callbacks_kthread()