Lines Matching refs:rcu_data
16 static bool rcu_rdp_is_offloaded(struct rcu_data *rdp) in rcu_rdp_is_offloaded()
33 rdp == this_cpu_ptr(&rcu_data)) || in rcu_rdp_is_offloaded()
162 static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_preempt_ctxt_queue()
301 if (__this_cpu_read(rcu_data.cpu_no_qs.b.norm)) { in rcu_qs()
303 __this_cpu_read(rcu_data.gp_seq), in rcu_qs()
305 __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); in rcu_qs()
327 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_note_context_switch()
485 struct rcu_data *rdp; in rcu_preempt_deferred_qs_irqrestore()
489 rdp = this_cpu_ptr(&rcu_data); in rcu_preempt_deferred_qs_irqrestore()
602 return (__this_cpu_read(rcu_data.cpu_no_qs.b.exp) || in rcu_preempt_need_deferred_qs()
629 struct rcu_data *rdp; in rcu_preempt_deferred_qs_handler()
632 rdp = container_of(iwp, struct rcu_data, defer_qs_iw); in rcu_preempt_deferred_qs_handler()
669 struct rcu_data *rdp, in rcu_unlock_needs_exp_handling()
740 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_read_unlock_special()
830 __this_cpu_read(rcu_data.core_needs_qs) && in rcu_flavor_sched_clock_irq()
831 __this_cpu_read(rcu_data.cpu_no_qs.b.norm) && in rcu_flavor_sched_clock_irq()
872 struct rcu_data *rdp; in dump_blkd_tasks()
894 rdp = per_cpu_ptr(&rcu_data, cpu); in dump_blkd_tasks()
902 static void rcu_preempt_deferred_qs_init(struct rcu_data *rdp) in rcu_preempt_deferred_qs_init()
915 struct rcu_data *rdp; in rcu_read_unlock_strict()
928 rdp = this_cpu_ptr(&rcu_data); in rcu_read_unlock_strict()
953 if (!__this_cpu_read(rcu_data.cpu_no_qs.s)) in rcu_qs()
956 __this_cpu_read(rcu_data.gp_seq), TPS("cpuqs")); in rcu_qs()
957 __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); in rcu_qs()
958 if (__this_cpu_read(rcu_data.cpu_no_qs.b.exp)) in rcu_qs()
959 rcu_report_exp_rdp(this_cpu_ptr(&rcu_data)); in rcu_qs()
973 if (!raw_cpu_read(rcu_data.rcu_urgent_qs)) in rcu_all_qs()
977 if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) { in rcu_all_qs()
981 this_cpu_write(rcu_data.rcu_urgent_qs, false); in rcu_all_qs()
982 if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) { in rcu_all_qs()
1000 if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) in rcu_note_context_switch()
1002 this_cpu_write(rcu_data.rcu_urgent_qs, false); in rcu_note_context_switch()
1003 if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) in rcu_note_context_switch()
1046 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_preempt_deferred_qs()
1105 static void rcu_preempt_deferred_qs_init(struct rcu_data *rdp) { } in rcu_preempt_deferred_qs_init()
1114 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_cpu_kthread_setup()
1125 static bool rcu_is_callbacks_nocb_kthread(struct rcu_data *rdp) in rcu_is_callbacks_nocb_kthread()
1138 static bool rcu_is_callbacks_kthread(struct rcu_data *rdp) in rcu_is_callbacks_kthread()