1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Read-Copy Update module-based torture test facility 4 * 5 * Copyright (C) IBM Corporation, 2005, 2006 6 * 7 * Authors: Paul E. McKenney <paulmck@linux.ibm.com> 8 * Josh Triplett <josh@joshtriplett.org> 9 * 10 * See also: Documentation/RCU/torture.rst 11 */ 12 13 #define pr_fmt(fmt) fmt 14 15 #include <linux/types.h> 16 #include <linux/kernel.h> 17 #include <linux/init.h> 18 #include <linux/module.h> 19 #include <linux/kthread.h> 20 #include <linux/err.h> 21 #include <linux/spinlock.h> 22 #include <linux/smp.h> 23 #include <linux/rcupdate_wait.h> 24 #include <linux/rcu_notifier.h> 25 #include <linux/interrupt.h> 26 #include <linux/sched/signal.h> 27 #include <uapi/linux/sched/types.h> 28 #include <linux/atomic.h> 29 #include <linux/bitops.h> 30 #include <linux/completion.h> 31 #include <linux/moduleparam.h> 32 #include <linux/percpu.h> 33 #include <linux/notifier.h> 34 #include <linux/reboot.h> 35 #include <linux/freezer.h> 36 #include <linux/cpu.h> 37 #include <linux/delay.h> 38 #include <linux/stat.h> 39 #include <linux/srcu.h> 40 #include <linux/slab.h> 41 #include <linux/trace_clock.h> 42 #include <asm/byteorder.h> 43 #include <linux/torture.h> 44 #include <linux/vmalloc.h> 45 #include <linux/sched/debug.h> 46 #include <linux/sched/sysctl.h> 47 #include <linux/oom.h> 48 #include <linux/tick.h> 49 #include <linux/rcupdate_trace.h> 50 #include <linux/nmi.h> 51 52 #include "rcu.h" 53 54 MODULE_DESCRIPTION("Read-Copy Update module-based torture test facility"); 55 MODULE_LICENSE("GPL"); 56 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com> and Josh Triplett <josh@joshtriplett.org>"); 57 58 /* Bits for ->extendables field, extendables param, and related definitions. */ 59 #define RCUTORTURE_RDR_SHIFT_1 8 /* Put SRCU index in upper bits. */ 60 #define RCUTORTURE_RDR_MASK_1 (0xff << RCUTORTURE_RDR_SHIFT_1) 61 #define RCUTORTURE_RDR_SHIFT_2 16 /* Put SRCU index in upper bits. */ 62 #define RCUTORTURE_RDR_MASK_2 (0xff << RCUTORTURE_RDR_SHIFT_2) 63 #define RCUTORTURE_RDR_BH 0x01 /* Extend readers by disabling bh. */ 64 #define RCUTORTURE_RDR_IRQ 0x02 /* ... disabling interrupts. */ 65 #define RCUTORTURE_RDR_PREEMPT 0x04 /* ... disabling preemption. */ 66 #define RCUTORTURE_RDR_RBH 0x08 /* ... rcu_read_lock_bh(). */ 67 #define RCUTORTURE_RDR_SCHED 0x10 /* ... rcu_read_lock_sched(). */ 68 #define RCUTORTURE_RDR_RCU_1 0x20 /* ... entering another RCU reader. */ 69 #define RCUTORTURE_RDR_RCU_2 0x40 /* ... entering another RCU reader. */ 70 #define RCUTORTURE_RDR_NBITS 7 /* Number of bits defined above. */ 71 #define RCUTORTURE_MAX_EXTEND \ 72 (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_IRQ | RCUTORTURE_RDR_PREEMPT | \ 73 RCUTORTURE_RDR_RBH | RCUTORTURE_RDR_SCHED) 74 #define RCUTORTURE_RDR_ALLBITS \ 75 (RCUTORTURE_MAX_EXTEND | RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2 | \ 76 RCUTORTURE_RDR_MASK_1 | RCUTORTURE_RDR_MASK_2) 77 #define RCUTORTURE_RDR_MAX_LOOPS 0x7 /* Maximum reader extensions. */ 78 /* Must be power of two minus one. */ 79 #define RCUTORTURE_RDR_MAX_SEGS (RCUTORTURE_RDR_MAX_LOOPS + 3) 80 81 torture_param(int, extendables, RCUTORTURE_MAX_EXTEND, 82 "Extend readers by disabling bh (1), irqs (2), or preempt (4)"); 83 torture_param(int, fqs_duration, 0, "Duration of fqs bursts (us), 0 to disable"); 84 torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)"); 85 torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)"); 86 torture_param(int, fwd_progress, 1, "Number of grace-period forward progress tasks (0 to disable)"); 87 torture_param(int, fwd_progress_div, 4, "Fraction of CPU stall to wait"); 88 torture_param(int, fwd_progress_holdoff, 60, "Time between forward-progress tests (s)"); 89 torture_param(bool, fwd_progress_need_resched, 1, "Hide cond_resched() behind need_resched()"); 90 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); 91 torture_param(bool, gp_cond_exp, false, "Use conditional/async expedited GP wait primitives"); 92 torture_param(bool, gp_cond_full, false, "Use conditional/async full-state GP wait primitives"); 93 torture_param(bool, gp_cond_exp_full, false, 94 "Use conditional/async full-stateexpedited GP wait primitives"); 95 torture_param(int, gp_cond_wi, 16 * USEC_PER_SEC / HZ, 96 "Wait interval for normal conditional grace periods, us (default 16 jiffies)"); 97 torture_param(int, gp_cond_wi_exp, 128, 98 "Wait interval for expedited conditional grace periods, us (default 128 us)"); 99 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); 100 torture_param(bool, gp_normal, false, "Use normal (non-expedited) GP wait primitives"); 101 torture_param(bool, gp_poll, false, "Use polling GP wait primitives"); 102 torture_param(bool, gp_poll_exp, false, "Use polling expedited GP wait primitives"); 103 torture_param(bool, gp_poll_full, false, "Use polling full-state GP wait primitives"); 104 torture_param(bool, gp_poll_exp_full, false, "Use polling full-state expedited GP wait primitives"); 105 torture_param(int, gp_poll_wi, 16 * USEC_PER_SEC / HZ, 106 "Wait interval for normal polled grace periods, us (default 16 jiffies)"); 107 torture_param(int, gp_poll_wi_exp, 128, 108 "Wait interval for expedited polled grace periods, us (default 128 us)"); 109 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); 110 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); 111 torture_param(int, leakpointer, 0, "Leak pointer dereferences from readers"); 112 torture_param(int, n_barrier_cbs, 0, "# of callbacks/kthreads for barrier testing"); 113 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); 114 torture_param(int, nreaders, -1, "Number of RCU reader threads"); 115 torture_param(int, object_debug, 0, "Enable debug-object double call_rcu() testing"); 116 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); 117 torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (jiffies), 0=disable"); 118 torture_param(bool, gpwrap_lag, true, "Enable grace-period wrap lag testing"); 119 torture_param(int, gpwrap_lag_gps, 8, "Value to set for set_gpwrap_lag during an active testing period."); 120 torture_param(int, gpwrap_lag_cycle_mins, 30, "Total cycle duration for gpwrap lag testing (in minutes)"); 121 torture_param(int, gpwrap_lag_active_mins, 5, "Duration for which gpwrap lag is active within each cycle (in minutes)"); 122 torture_param(int, nocbs_nthreads, 0, "Number of NOCB toggle threads, 0 to disable"); 123 torture_param(int, nocbs_toggle, 1000, "Time between toggling nocb state (ms)"); 124 torture_param(int, preempt_duration, 0, "Preemption duration (ms), zero to disable"); 125 torture_param(int, preempt_interval, MSEC_PER_SEC, "Interval between preemptions (ms)"); 126 torture_param(int, read_exit_delay, 13, "Delay between read-then-exit episodes (s)"); 127 torture_param(int, read_exit_burst, 16, "# of read-then-exit bursts per episode, zero to disable"); 128 torture_param(int, reader_flavor, SRCU_READ_FLAVOR_NORMAL, "Reader flavors to use, one per bit."); 129 torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); 130 torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); 131 torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable."); 132 torture_param(int, stall_cpu_holdoff, 10, "Time to wait before starting stall (s)."); 133 torture_param(bool, stall_no_softlockup, false, "Avoid softlockup warning during cpu stall."); 134 torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling."); 135 torture_param(int, stall_cpu_block, 0, "Sleep while stalling."); 136 torture_param(int, stall_cpu_repeat, 0, "Number of additional stalls after the first one."); 137 torture_param(int, stall_gp_kthread, 0, "Grace-period kthread stall duration (s)."); 138 torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s"); 139 torture_param(int, stutter, 5, "Number of seconds to run/halt test"); 140 torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); 141 torture_param(int, test_boost_duration, 4, "Duration of each boost test, seconds."); 142 torture_param(int, test_boost_holdoff, 0, "Holdoff time from rcutorture start, seconds."); 143 torture_param(int, test_boost_interval, 7, "Interval between boost tests, seconds."); 144 torture_param(int, test_nmis, 0, "End-test NMI tests, 0 to disable."); 145 torture_param(bool, test_no_idle_hz, true, "Test support for tickless idle CPUs"); 146 torture_param(int, test_srcu_lockdep, 0, "Test specified SRCU deadlock scenario."); 147 torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); 148 149 static char *torture_type = "rcu"; 150 module_param(torture_type, charp, 0444); 151 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, srcu, ...)"); 152 153 static int nrealnocbers; 154 static int nrealreaders; 155 static int nrealfakewriters; 156 static struct task_struct *writer_task; 157 static struct task_struct **fakewriter_tasks; 158 static struct task_struct **reader_tasks; 159 static struct task_struct **nocb_tasks; 160 static struct task_struct *stats_task; 161 static struct task_struct *fqs_task; 162 static struct task_struct *boost_tasks[NR_CPUS]; 163 static struct task_struct *stall_task; 164 static struct task_struct **fwd_prog_tasks; 165 static struct task_struct **barrier_cbs_tasks; 166 static struct task_struct *barrier_task; 167 static struct task_struct *read_exit_task; 168 static struct task_struct *preempt_task; 169 170 #define RCU_TORTURE_PIPE_LEN 10 171 172 // Mailbox-like structure to check RCU global memory ordering. 173 struct rcu_torture_reader_check { 174 unsigned long rtc_myloops; 175 int rtc_chkrdr; 176 unsigned long rtc_chkloops; 177 int rtc_ready; 178 struct rcu_torture_reader_check *rtc_assigner; 179 } ____cacheline_internodealigned_in_smp; 180 181 // Update-side data structure used to check RCU readers. 182 struct rcu_torture { 183 struct rcu_head rtort_rcu; 184 int rtort_pipe_count; 185 struct list_head rtort_free; 186 int rtort_mbtest; 187 struct rcu_torture_reader_check *rtort_chkp; 188 }; 189 190 static LIST_HEAD(rcu_torture_freelist); 191 static struct rcu_torture __rcu *rcu_torture_current; 192 static unsigned long rcu_torture_current_version; 193 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; 194 static DEFINE_SPINLOCK(rcu_torture_lock); 195 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count); 196 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch); 197 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; 198 static struct rcu_torture_reader_check *rcu_torture_reader_mbchk; 199 static atomic_t n_rcu_torture_alloc; 200 static atomic_t n_rcu_torture_alloc_fail; 201 static atomic_t n_rcu_torture_free; 202 static atomic_t n_rcu_torture_mberror; 203 static atomic_t n_rcu_torture_mbchk_fail; 204 static atomic_t n_rcu_torture_mbchk_tries; 205 static atomic_t n_rcu_torture_error; 206 static long n_rcu_torture_barrier_error; 207 static long n_rcu_torture_boost_ktrerror; 208 static long n_rcu_torture_boost_failure; 209 static long n_rcu_torture_boosts; 210 static atomic_long_t n_rcu_torture_timers; 211 static long n_barrier_attempts; 212 static long n_barrier_successes; /* did rcu_barrier test succeed? */ 213 static unsigned long n_read_exits; 214 static struct list_head rcu_torture_removed; 215 static unsigned long shutdown_jiffies; 216 static unsigned long start_gp_seq; 217 static atomic_long_t n_nocb_offload; 218 static atomic_long_t n_nocb_deoffload; 219 220 static int rcu_torture_writer_state; 221 #define RTWS_FIXED_DELAY 0 222 #define RTWS_DELAY 1 223 #define RTWS_REPLACE 2 224 #define RTWS_DEF_FREE 3 225 #define RTWS_EXP_SYNC 4 226 #define RTWS_COND_GET 5 227 #define RTWS_COND_GET_FULL 6 228 #define RTWS_COND_GET_EXP 7 229 #define RTWS_COND_GET_EXP_FULL 8 230 #define RTWS_COND_SYNC 9 231 #define RTWS_COND_SYNC_FULL 10 232 #define RTWS_COND_SYNC_EXP 11 233 #define RTWS_COND_SYNC_EXP_FULL 12 234 #define RTWS_POLL_GET 13 235 #define RTWS_POLL_GET_FULL 14 236 #define RTWS_POLL_GET_EXP 15 237 #define RTWS_POLL_GET_EXP_FULL 16 238 #define RTWS_POLL_WAIT 17 239 #define RTWS_POLL_WAIT_FULL 18 240 #define RTWS_POLL_WAIT_EXP 19 241 #define RTWS_POLL_WAIT_EXP_FULL 20 242 #define RTWS_SYNC 21 243 #define RTWS_STUTTER 22 244 #define RTWS_STOPPING 23 245 static const char * const rcu_torture_writer_state_names[] = { 246 "RTWS_FIXED_DELAY", 247 "RTWS_DELAY", 248 "RTWS_REPLACE", 249 "RTWS_DEF_FREE", 250 "RTWS_EXP_SYNC", 251 "RTWS_COND_GET", 252 "RTWS_COND_GET_FULL", 253 "RTWS_COND_GET_EXP", 254 "RTWS_COND_GET_EXP_FULL", 255 "RTWS_COND_SYNC", 256 "RTWS_COND_SYNC_FULL", 257 "RTWS_COND_SYNC_EXP", 258 "RTWS_COND_SYNC_EXP_FULL", 259 "RTWS_POLL_GET", 260 "RTWS_POLL_GET_FULL", 261 "RTWS_POLL_GET_EXP", 262 "RTWS_POLL_GET_EXP_FULL", 263 "RTWS_POLL_WAIT", 264 "RTWS_POLL_WAIT_FULL", 265 "RTWS_POLL_WAIT_EXP", 266 "RTWS_POLL_WAIT_EXP_FULL", 267 "RTWS_SYNC", 268 "RTWS_STUTTER", 269 "RTWS_STOPPING", 270 }; 271 272 /* Record reader segment types and duration for first failing read. */ 273 struct rt_read_seg { 274 int rt_readstate; 275 unsigned long rt_delay_jiffies; 276 unsigned long rt_delay_ms; 277 unsigned long rt_delay_us; 278 bool rt_preempted; 279 int rt_cpu; 280 int rt_end_cpu; 281 unsigned long long rt_gp_seq; 282 unsigned long long rt_gp_seq_end; 283 u64 rt_ts; 284 }; 285 static int err_segs_recorded; 286 static struct rt_read_seg err_segs[RCUTORTURE_RDR_MAX_SEGS]; 287 static int rt_read_nsegs; 288 static int rt_read_preempted; 289 290 static const char *rcu_torture_writer_state_getname(void) 291 { 292 unsigned int i = READ_ONCE(rcu_torture_writer_state); 293 294 if (i >= ARRAY_SIZE(rcu_torture_writer_state_names)) 295 return "???"; 296 return rcu_torture_writer_state_names[i]; 297 } 298 299 #ifdef CONFIG_RCU_TRACE 300 static u64 notrace rcu_trace_clock_local(void) 301 { 302 u64 ts = trace_clock_local(); 303 304 (void)do_div(ts, NSEC_PER_USEC); 305 return ts; 306 } 307 #else /* #ifdef CONFIG_RCU_TRACE */ 308 static u64 notrace rcu_trace_clock_local(void) 309 { 310 return 0ULL; 311 } 312 #endif /* #else #ifdef CONFIG_RCU_TRACE */ 313 314 /* 315 * Stop aggressive CPU-hog tests a bit before the end of the test in order 316 * to avoid interfering with test shutdown. 317 */ 318 static bool shutdown_time_arrived(void) 319 { 320 return shutdown_secs && time_after(jiffies, shutdown_jiffies - 30 * HZ); 321 } 322 323 static unsigned long boost_starttime; /* jiffies of next boost test start. */ 324 static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ 325 /* and boost task create/destroy. */ 326 static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ 327 static bool barrier_phase; /* Test phase. */ 328 static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ 329 static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ 330 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); 331 332 static atomic_t rcu_fwd_cb_nodelay; /* Short rcu_torture_delay() delays. */ 333 334 /* 335 * Allocate an element from the rcu_tortures pool. 336 */ 337 static struct rcu_torture * 338 rcu_torture_alloc(void) 339 { 340 struct list_head *p; 341 342 spin_lock_bh(&rcu_torture_lock); 343 if (list_empty(&rcu_torture_freelist)) { 344 atomic_inc(&n_rcu_torture_alloc_fail); 345 spin_unlock_bh(&rcu_torture_lock); 346 return NULL; 347 } 348 atomic_inc(&n_rcu_torture_alloc); 349 p = rcu_torture_freelist.next; 350 list_del_init(p); 351 spin_unlock_bh(&rcu_torture_lock); 352 return container_of(p, struct rcu_torture, rtort_free); 353 } 354 355 /* 356 * Free an element to the rcu_tortures pool. 357 */ 358 static void 359 rcu_torture_free(struct rcu_torture *p) 360 { 361 atomic_inc(&n_rcu_torture_free); 362 spin_lock_bh(&rcu_torture_lock); 363 list_add_tail(&p->rtort_free, &rcu_torture_freelist); 364 spin_unlock_bh(&rcu_torture_lock); 365 } 366 367 /* 368 * Operations vector for selecting different types of tests. 369 */ 370 371 struct rcu_torture_ops { 372 int ttype; 373 void (*init)(void); 374 void (*cleanup)(void); 375 int (*readlock)(void); 376 void (*read_delay)(struct torture_random_state *rrsp, 377 struct rt_read_seg *rtrsp); 378 void (*readunlock)(int idx); 379 int (*readlock_held)(void); // lockdep. 380 int (*readlock_nesting)(void); // actual nesting, if available, -1 if not. 381 unsigned long (*get_gp_seq)(void); 382 unsigned long (*gp_diff)(unsigned long new, unsigned long old); 383 void (*deferred_free)(struct rcu_torture *p); 384 void (*sync)(void); 385 void (*exp_sync)(void); 386 unsigned long (*get_gp_state_exp)(void); 387 unsigned long (*start_gp_poll_exp)(void); 388 void (*start_gp_poll_exp_full)(struct rcu_gp_oldstate *rgosp); 389 bool (*poll_gp_state_exp)(unsigned long oldstate); 390 void (*cond_sync_exp)(unsigned long oldstate); 391 void (*cond_sync_exp_full)(struct rcu_gp_oldstate *rgosp); 392 unsigned long (*get_comp_state)(void); 393 void (*get_comp_state_full)(struct rcu_gp_oldstate *rgosp); 394 bool (*same_gp_state)(unsigned long oldstate1, unsigned long oldstate2); 395 bool (*same_gp_state_full)(struct rcu_gp_oldstate *rgosp1, struct rcu_gp_oldstate *rgosp2); 396 unsigned long (*get_gp_state)(void); 397 void (*get_gp_state_full)(struct rcu_gp_oldstate *rgosp); 398 unsigned long (*start_gp_poll)(void); 399 void (*start_gp_poll_full)(struct rcu_gp_oldstate *rgosp); 400 bool (*poll_gp_state)(unsigned long oldstate); 401 bool (*poll_gp_state_full)(struct rcu_gp_oldstate *rgosp); 402 bool (*poll_need_2gp)(bool poll, bool poll_full); 403 void (*cond_sync)(unsigned long oldstate); 404 void (*cond_sync_full)(struct rcu_gp_oldstate *rgosp); 405 int poll_active; 406 int poll_active_full; 407 call_rcu_func_t call; 408 void (*cb_barrier)(void); 409 void (*fqs)(void); 410 void (*stats)(void); 411 void (*gp_kthread_dbg)(void); 412 bool (*check_boost_failed)(unsigned long gp_state, int *cpup); 413 int (*stall_dur)(void); 414 void (*get_gp_data)(int *flags, unsigned long *gp_seq); 415 void (*gp_slow_register)(atomic_t *rgssp); 416 void (*gp_slow_unregister)(atomic_t *rgssp); 417 bool (*reader_blocked)(void); 418 unsigned long long (*gather_gp_seqs)(void); 419 void (*format_gp_seqs)(unsigned long long seqs, char *cp, size_t len); 420 void (*set_gpwrap_lag)(unsigned long lag); 421 int (*get_gpwrap_count)(int cpu); 422 long cbflood_max; 423 int irq_capable; 424 int can_boost; 425 int extendables; 426 int slow_gps; 427 int no_pi_lock; 428 int debug_objects; 429 int start_poll_irqsoff; 430 const char *name; 431 }; 432 433 static struct rcu_torture_ops *cur_ops; 434 435 /* 436 * Definitions for rcu torture testing. 437 */ 438 439 static int torture_readlock_not_held(void) 440 { 441 return rcu_read_lock_bh_held() || rcu_read_lock_sched_held(); 442 } 443 444 static int rcu_torture_read_lock(void) 445 { 446 rcu_read_lock(); 447 return 0; 448 } 449 450 static void 451 rcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) 452 { 453 unsigned long started; 454 unsigned long completed; 455 const unsigned long shortdelay_us = 200; 456 unsigned long longdelay_ms = 300; 457 unsigned long long ts; 458 459 /* We want a short delay sometimes to make a reader delay the grace 460 * period, and we want a long delay occasionally to trigger 461 * force_quiescent_state. */ 462 463 if (!atomic_read(&rcu_fwd_cb_nodelay) && 464 !(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) { 465 started = cur_ops->get_gp_seq(); 466 ts = rcu_trace_clock_local(); 467 if (preempt_count() & (SOFTIRQ_MASK | HARDIRQ_MASK)) 468 longdelay_ms = 5; /* Avoid triggering BH limits. */ 469 mdelay(longdelay_ms); 470 rtrsp->rt_delay_ms = longdelay_ms; 471 completed = cur_ops->get_gp_seq(); 472 do_trace_rcu_torture_read(cur_ops->name, NULL, ts, 473 started, completed); 474 } 475 if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) { 476 udelay(shortdelay_us); 477 rtrsp->rt_delay_us = shortdelay_us; 478 } 479 if (!preempt_count() && 480 !(torture_random(rrsp) % (nrealreaders * 500))) 481 torture_preempt_schedule(); /* QS only if preemptible. */ 482 } 483 484 static void rcu_torture_read_unlock(int idx) 485 { 486 rcu_read_unlock(); 487 } 488 489 static int rcu_torture_readlock_nesting(void) 490 { 491 if (IS_ENABLED(CONFIG_PREEMPT_RCU)) 492 return rcu_preempt_depth(); 493 if (IS_ENABLED(CONFIG_PREEMPT_COUNT)) 494 return (preempt_count() & PREEMPT_MASK); 495 return -1; 496 } 497 498 /* 499 * Update callback in the pipe. This should be invoked after a grace period. 500 */ 501 static bool 502 rcu_torture_pipe_update_one(struct rcu_torture *rp) 503 { 504 int i; 505 struct rcu_torture_reader_check *rtrcp = READ_ONCE(rp->rtort_chkp); 506 507 if (rtrcp) { 508 WRITE_ONCE(rp->rtort_chkp, NULL); 509 smp_store_release(&rtrcp->rtc_ready, 1); // Pair with smp_load_acquire(). 510 } 511 i = rp->rtort_pipe_count; 512 if (i > RCU_TORTURE_PIPE_LEN) 513 i = RCU_TORTURE_PIPE_LEN; 514 atomic_inc(&rcu_torture_wcount[i]); 515 WRITE_ONCE(rp->rtort_pipe_count, i + 1); 516 ASSERT_EXCLUSIVE_WRITER(rp->rtort_pipe_count); 517 if (i + 1 >= RCU_TORTURE_PIPE_LEN) { 518 rp->rtort_mbtest = 0; 519 return true; 520 } 521 return false; 522 } 523 524 /* 525 * Update all callbacks in the pipe. Suitable for synchronous grace-period 526 * primitives. 527 */ 528 static void 529 rcu_torture_pipe_update(struct rcu_torture *old_rp) 530 { 531 struct rcu_torture *rp; 532 struct rcu_torture *rp1; 533 534 if (old_rp) 535 list_add(&old_rp->rtort_free, &rcu_torture_removed); 536 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) { 537 if (rcu_torture_pipe_update_one(rp)) { 538 list_del(&rp->rtort_free); 539 rcu_torture_free(rp); 540 } 541 } 542 } 543 544 static void 545 rcu_torture_cb(struct rcu_head *p) 546 { 547 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 548 549 if (torture_must_stop_irq()) { 550 /* Test is ending, just drop callbacks on the floor. */ 551 /* The next initialization will pick up the pieces. */ 552 return; 553 } 554 if (rcu_torture_pipe_update_one(rp)) 555 rcu_torture_free(rp); 556 else 557 cur_ops->deferred_free(rp); 558 } 559 560 static unsigned long rcu_no_completed(void) 561 { 562 return 0; 563 } 564 565 static void rcu_torture_deferred_free(struct rcu_torture *p) 566 { 567 call_rcu_hurry(&p->rtort_rcu, rcu_torture_cb); 568 } 569 570 static void rcu_sync_torture_init(void) 571 { 572 INIT_LIST_HEAD(&rcu_torture_removed); 573 } 574 575 static bool rcu_poll_need_2gp(bool poll, bool poll_full) 576 { 577 return poll; 578 } 579 580 static struct rcu_torture_ops rcu_ops = { 581 .ttype = RCU_FLAVOR, 582 .init = rcu_sync_torture_init, 583 .readlock = rcu_torture_read_lock, 584 .read_delay = rcu_read_delay, 585 .readunlock = rcu_torture_read_unlock, 586 .readlock_held = torture_readlock_not_held, 587 .readlock_nesting = rcu_torture_readlock_nesting, 588 .get_gp_seq = rcu_get_gp_seq, 589 .gp_diff = rcu_seq_diff, 590 .deferred_free = rcu_torture_deferred_free, 591 .sync = synchronize_rcu, 592 .exp_sync = synchronize_rcu_expedited, 593 .same_gp_state = same_state_synchronize_rcu, 594 .same_gp_state_full = same_state_synchronize_rcu_full, 595 .get_comp_state = get_completed_synchronize_rcu, 596 .get_comp_state_full = get_completed_synchronize_rcu_full, 597 .get_gp_state = get_state_synchronize_rcu, 598 .get_gp_state_full = get_state_synchronize_rcu_full, 599 .start_gp_poll = start_poll_synchronize_rcu, 600 .start_gp_poll_full = start_poll_synchronize_rcu_full, 601 .poll_gp_state = poll_state_synchronize_rcu, 602 .poll_gp_state_full = poll_state_synchronize_rcu_full, 603 .poll_need_2gp = rcu_poll_need_2gp, 604 .cond_sync = cond_synchronize_rcu, 605 .cond_sync_full = cond_synchronize_rcu_full, 606 .poll_active = NUM_ACTIVE_RCU_POLL_OLDSTATE, 607 .poll_active_full = NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE, 608 .get_gp_state_exp = get_state_synchronize_rcu, 609 .start_gp_poll_exp = start_poll_synchronize_rcu_expedited, 610 .start_gp_poll_exp_full = start_poll_synchronize_rcu_expedited_full, 611 .poll_gp_state_exp = poll_state_synchronize_rcu, 612 .cond_sync_exp = cond_synchronize_rcu_expedited, 613 .cond_sync_exp_full = cond_synchronize_rcu_expedited_full, 614 .call = call_rcu_hurry, 615 .cb_barrier = rcu_barrier, 616 .fqs = rcu_force_quiescent_state, 617 .gp_kthread_dbg = show_rcu_gp_kthreads, 618 .check_boost_failed = rcu_check_boost_fail, 619 .stall_dur = rcu_jiffies_till_stall_check, 620 .get_gp_data = rcutorture_get_gp_data, 621 .gp_slow_register = rcu_gp_slow_register, 622 .gp_slow_unregister = rcu_gp_slow_unregister, 623 .reader_blocked = IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU) 624 ? has_rcu_reader_blocked 625 : NULL, 626 .gather_gp_seqs = rcutorture_gather_gp_seqs, 627 .format_gp_seqs = rcutorture_format_gp_seqs, 628 .set_gpwrap_lag = rcu_set_gpwrap_lag, 629 .get_gpwrap_count = rcu_get_gpwrap_count, 630 .irq_capable = 1, 631 .can_boost = IS_ENABLED(CONFIG_RCU_BOOST), 632 .extendables = RCUTORTURE_MAX_EXTEND, 633 .debug_objects = 1, 634 .start_poll_irqsoff = 1, 635 .name = "rcu" 636 }; 637 638 /* 639 * Don't even think about trying any of these in real life!!! 640 * The names includes "busted", and they really means it! 641 * The only purpose of these functions is to provide a buggy RCU 642 * implementation to make sure that rcutorture correctly emits 643 * buggy-RCU error messages. 644 */ 645 static void rcu_busted_torture_deferred_free(struct rcu_torture *p) 646 { 647 /* This is a deliberate bug for testing purposes only! */ 648 rcu_torture_cb(&p->rtort_rcu); 649 } 650 651 static void synchronize_rcu_busted(void) 652 { 653 /* This is a deliberate bug for testing purposes only! */ 654 } 655 656 static void 657 call_rcu_busted(struct rcu_head *head, rcu_callback_t func) 658 { 659 /* This is a deliberate bug for testing purposes only! */ 660 func(head); 661 } 662 663 static struct rcu_torture_ops rcu_busted_ops = { 664 .ttype = INVALID_RCU_FLAVOR, 665 .init = rcu_sync_torture_init, 666 .readlock = rcu_torture_read_lock, 667 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 668 .readunlock = rcu_torture_read_unlock, 669 .readlock_held = torture_readlock_not_held, 670 .get_gp_seq = rcu_no_completed, 671 .deferred_free = rcu_busted_torture_deferred_free, 672 .sync = synchronize_rcu_busted, 673 .exp_sync = synchronize_rcu_busted, 674 .call = call_rcu_busted, 675 .gather_gp_seqs = rcutorture_gather_gp_seqs, 676 .format_gp_seqs = rcutorture_format_gp_seqs, 677 .irq_capable = 1, 678 .extendables = RCUTORTURE_MAX_EXTEND, 679 .name = "busted" 680 }; 681 682 /* 683 * Definitions for srcu torture testing. 684 */ 685 686 DEFINE_STATIC_SRCU(srcu_ctl); 687 static struct srcu_struct srcu_ctld; 688 static struct srcu_struct *srcu_ctlp = &srcu_ctl; 689 static struct rcu_torture_ops srcud_ops; 690 691 static void srcu_get_gp_data(int *flags, unsigned long *gp_seq) 692 { 693 srcutorture_get_gp_data(srcu_ctlp, flags, gp_seq); 694 } 695 696 static int srcu_torture_read_lock(void) 697 { 698 int idx; 699 struct srcu_ctr __percpu *scp; 700 int ret = 0; 701 702 WARN_ON_ONCE(reader_flavor & ~SRCU_READ_FLAVOR_ALL); 703 704 if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) || !(reader_flavor & SRCU_READ_FLAVOR_ALL)) { 705 idx = srcu_read_lock(srcu_ctlp); 706 WARN_ON_ONCE(idx & ~0x1); 707 ret += idx; 708 } 709 if (reader_flavor & SRCU_READ_FLAVOR_NMI) { 710 idx = srcu_read_lock_nmisafe(srcu_ctlp); 711 WARN_ON_ONCE(idx & ~0x1); 712 ret += idx << 1; 713 } 714 if (reader_flavor & SRCU_READ_FLAVOR_LITE) { 715 idx = srcu_read_lock_lite(srcu_ctlp); 716 WARN_ON_ONCE(idx & ~0x1); 717 ret += idx << 2; 718 } 719 if (reader_flavor & SRCU_READ_FLAVOR_FAST) { 720 scp = srcu_read_lock_fast(srcu_ctlp); 721 idx = __srcu_ptr_to_ctr(srcu_ctlp, scp); 722 WARN_ON_ONCE(idx & ~0x1); 723 ret += idx << 3; 724 } 725 return ret; 726 } 727 728 static void 729 srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) 730 { 731 long delay; 732 const long uspertick = 1000000 / HZ; 733 const long longdelay = 10; 734 735 /* We want there to be long-running readers, but not all the time. */ 736 737 delay = torture_random(rrsp) % 738 (nrealreaders * 2 * longdelay * uspertick); 739 if (!delay && in_task()) { 740 schedule_timeout_interruptible(longdelay); 741 rtrsp->rt_delay_jiffies = longdelay; 742 } else { 743 rcu_read_delay(rrsp, rtrsp); 744 } 745 } 746 747 static void srcu_torture_read_unlock(int idx) 748 { 749 WARN_ON_ONCE((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1))); 750 if (reader_flavor & SRCU_READ_FLAVOR_FAST) 751 srcu_read_unlock_fast(srcu_ctlp, __srcu_ctr_to_ptr(srcu_ctlp, (idx & 0x8) >> 3)); 752 if (reader_flavor & SRCU_READ_FLAVOR_LITE) 753 srcu_read_unlock_lite(srcu_ctlp, (idx & 0x4) >> 2); 754 if (reader_flavor & SRCU_READ_FLAVOR_NMI) 755 srcu_read_unlock_nmisafe(srcu_ctlp, (idx & 0x2) >> 1); 756 if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) || !(reader_flavor & SRCU_READ_FLAVOR_ALL)) 757 srcu_read_unlock(srcu_ctlp, idx & 0x1); 758 } 759 760 static int torture_srcu_read_lock_held(void) 761 { 762 return srcu_read_lock_held(srcu_ctlp); 763 } 764 765 static unsigned long srcu_torture_completed(void) 766 { 767 return srcu_batches_completed(srcu_ctlp); 768 } 769 770 static void srcu_torture_deferred_free(struct rcu_torture *rp) 771 { 772 call_srcu(srcu_ctlp, &rp->rtort_rcu, rcu_torture_cb); 773 } 774 775 static void srcu_torture_synchronize(void) 776 { 777 synchronize_srcu(srcu_ctlp); 778 } 779 780 static unsigned long srcu_torture_get_gp_state(void) 781 { 782 return get_state_synchronize_srcu(srcu_ctlp); 783 } 784 785 static unsigned long srcu_torture_start_gp_poll(void) 786 { 787 return start_poll_synchronize_srcu(srcu_ctlp); 788 } 789 790 static bool srcu_torture_poll_gp_state(unsigned long oldstate) 791 { 792 return poll_state_synchronize_srcu(srcu_ctlp, oldstate); 793 } 794 795 static void srcu_torture_call(struct rcu_head *head, 796 rcu_callback_t func) 797 { 798 call_srcu(srcu_ctlp, head, func); 799 } 800 801 static void srcu_torture_barrier(void) 802 { 803 srcu_barrier(srcu_ctlp); 804 } 805 806 static void srcu_torture_stats(void) 807 { 808 srcu_torture_stats_print(srcu_ctlp, torture_type, TORTURE_FLAG); 809 } 810 811 static void srcu_torture_synchronize_expedited(void) 812 { 813 synchronize_srcu_expedited(srcu_ctlp); 814 } 815 816 static struct rcu_torture_ops srcu_ops = { 817 .ttype = SRCU_FLAVOR, 818 .init = rcu_sync_torture_init, 819 .readlock = srcu_torture_read_lock, 820 .read_delay = srcu_read_delay, 821 .readunlock = srcu_torture_read_unlock, 822 .readlock_held = torture_srcu_read_lock_held, 823 .get_gp_seq = srcu_torture_completed, 824 .gp_diff = rcu_seq_diff, 825 .deferred_free = srcu_torture_deferred_free, 826 .sync = srcu_torture_synchronize, 827 .exp_sync = srcu_torture_synchronize_expedited, 828 .same_gp_state = same_state_synchronize_srcu, 829 .get_comp_state = get_completed_synchronize_srcu, 830 .get_gp_state = srcu_torture_get_gp_state, 831 .start_gp_poll = srcu_torture_start_gp_poll, 832 .poll_gp_state = srcu_torture_poll_gp_state, 833 .poll_active = NUM_ACTIVE_SRCU_POLL_OLDSTATE, 834 .call = srcu_torture_call, 835 .cb_barrier = srcu_torture_barrier, 836 .stats = srcu_torture_stats, 837 .get_gp_data = srcu_get_gp_data, 838 .cbflood_max = 50000, 839 .irq_capable = 1, 840 .no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU), 841 .debug_objects = 1, 842 .name = "srcu" 843 }; 844 845 static void srcu_torture_init(void) 846 { 847 rcu_sync_torture_init(); 848 WARN_ON(init_srcu_struct(&srcu_ctld)); 849 srcu_ctlp = &srcu_ctld; 850 } 851 852 static void srcu_torture_cleanup(void) 853 { 854 cleanup_srcu_struct(&srcu_ctld); 855 srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */ 856 } 857 858 /* As above, but dynamically allocated. */ 859 static struct rcu_torture_ops srcud_ops = { 860 .ttype = SRCU_FLAVOR, 861 .init = srcu_torture_init, 862 .cleanup = srcu_torture_cleanup, 863 .readlock = srcu_torture_read_lock, 864 .read_delay = srcu_read_delay, 865 .readunlock = srcu_torture_read_unlock, 866 .readlock_held = torture_srcu_read_lock_held, 867 .get_gp_seq = srcu_torture_completed, 868 .gp_diff = rcu_seq_diff, 869 .deferred_free = srcu_torture_deferred_free, 870 .sync = srcu_torture_synchronize, 871 .exp_sync = srcu_torture_synchronize_expedited, 872 .same_gp_state = same_state_synchronize_srcu, 873 .get_comp_state = get_completed_synchronize_srcu, 874 .get_gp_state = srcu_torture_get_gp_state, 875 .start_gp_poll = srcu_torture_start_gp_poll, 876 .poll_gp_state = srcu_torture_poll_gp_state, 877 .poll_active = NUM_ACTIVE_SRCU_POLL_OLDSTATE, 878 .call = srcu_torture_call, 879 .cb_barrier = srcu_torture_barrier, 880 .stats = srcu_torture_stats, 881 .get_gp_data = srcu_get_gp_data, 882 .cbflood_max = 50000, 883 .irq_capable = 1, 884 .no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU), 885 .debug_objects = 1, 886 .name = "srcud" 887 }; 888 889 /* As above, but broken due to inappropriate reader extension. */ 890 static struct rcu_torture_ops busted_srcud_ops = { 891 .ttype = SRCU_FLAVOR, 892 .init = srcu_torture_init, 893 .cleanup = srcu_torture_cleanup, 894 .readlock = srcu_torture_read_lock, 895 .read_delay = rcu_read_delay, 896 .readunlock = srcu_torture_read_unlock, 897 .readlock_held = torture_srcu_read_lock_held, 898 .get_gp_seq = srcu_torture_completed, 899 .deferred_free = srcu_torture_deferred_free, 900 .sync = srcu_torture_synchronize, 901 .exp_sync = srcu_torture_synchronize_expedited, 902 .call = srcu_torture_call, 903 .cb_barrier = srcu_torture_barrier, 904 .stats = srcu_torture_stats, 905 .irq_capable = 1, 906 .no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU), 907 .extendables = RCUTORTURE_MAX_EXTEND, 908 .name = "busted_srcud" 909 }; 910 911 /* 912 * Definitions for trivial CONFIG_PREEMPT=n-only torture testing. 913 * This implementation does not necessarily work well with CPU hotplug. 914 */ 915 916 static void synchronize_rcu_trivial(void) 917 { 918 int cpu; 919 920 for_each_online_cpu(cpu) { 921 torture_sched_setaffinity(current->pid, cpumask_of(cpu), true); 922 WARN_ON_ONCE(raw_smp_processor_id() != cpu); 923 } 924 } 925 926 static int rcu_torture_read_lock_trivial(void) 927 { 928 preempt_disable(); 929 return 0; 930 } 931 932 static void rcu_torture_read_unlock_trivial(int idx) 933 { 934 preempt_enable(); 935 } 936 937 static struct rcu_torture_ops trivial_ops = { 938 .ttype = RCU_TRIVIAL_FLAVOR, 939 .init = rcu_sync_torture_init, 940 .readlock = rcu_torture_read_lock_trivial, 941 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 942 .readunlock = rcu_torture_read_unlock_trivial, 943 .readlock_held = torture_readlock_not_held, 944 .get_gp_seq = rcu_no_completed, 945 .sync = synchronize_rcu_trivial, 946 .exp_sync = synchronize_rcu_trivial, 947 .irq_capable = 1, 948 .name = "trivial" 949 }; 950 951 #ifdef CONFIG_TASKS_RCU 952 953 /* 954 * Definitions for RCU-tasks torture testing. 955 */ 956 957 static int tasks_torture_read_lock(void) 958 { 959 return 0; 960 } 961 962 static void tasks_torture_read_unlock(int idx) 963 { 964 } 965 966 static void rcu_tasks_torture_deferred_free(struct rcu_torture *p) 967 { 968 call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb); 969 } 970 971 static void synchronize_rcu_mult_test(void) 972 { 973 synchronize_rcu_mult(call_rcu_tasks, call_rcu_hurry); 974 } 975 976 static struct rcu_torture_ops tasks_ops = { 977 .ttype = RCU_TASKS_FLAVOR, 978 .init = rcu_sync_torture_init, 979 .readlock = tasks_torture_read_lock, 980 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 981 .readunlock = tasks_torture_read_unlock, 982 .get_gp_seq = rcu_no_completed, 983 .deferred_free = rcu_tasks_torture_deferred_free, 984 .sync = synchronize_rcu_tasks, 985 .exp_sync = synchronize_rcu_mult_test, 986 .call = call_rcu_tasks, 987 .cb_barrier = rcu_barrier_tasks, 988 .gp_kthread_dbg = show_rcu_tasks_classic_gp_kthread, 989 .get_gp_data = rcu_tasks_get_gp_data, 990 .irq_capable = 1, 991 .slow_gps = 1, 992 .name = "tasks" 993 }; 994 995 #define TASKS_OPS &tasks_ops, 996 997 #else // #ifdef CONFIG_TASKS_RCU 998 999 #define TASKS_OPS 1000 1001 #endif // #else #ifdef CONFIG_TASKS_RCU 1002 1003 1004 #ifdef CONFIG_TASKS_RUDE_RCU 1005 1006 /* 1007 * Definitions for rude RCU-tasks torture testing. 1008 */ 1009 1010 static struct rcu_torture_ops tasks_rude_ops = { 1011 .ttype = RCU_TASKS_RUDE_FLAVOR, 1012 .init = rcu_sync_torture_init, 1013 .readlock = rcu_torture_read_lock_trivial, 1014 .read_delay = rcu_read_delay, /* just reuse rcu's version. */ 1015 .readunlock = rcu_torture_read_unlock_trivial, 1016 .get_gp_seq = rcu_no_completed, 1017 .sync = synchronize_rcu_tasks_rude, 1018 .exp_sync = synchronize_rcu_tasks_rude, 1019 .gp_kthread_dbg = show_rcu_tasks_rude_gp_kthread, 1020 .get_gp_data = rcu_tasks_rude_get_gp_data, 1021 .cbflood_max = 50000, 1022 .irq_capable = 1, 1023 .name = "tasks-rude" 1024 }; 1025 1026 #define TASKS_RUDE_OPS &tasks_rude_ops, 1027 1028 #else // #ifdef CONFIG_TASKS_RUDE_RCU 1029 1030 #define TASKS_RUDE_OPS 1031 1032 #endif // #else #ifdef CONFIG_TASKS_RUDE_RCU 1033 1034 1035 #ifdef CONFIG_TASKS_TRACE_RCU 1036 1037 /* 1038 * Definitions for tracing RCU-tasks torture testing. 1039 */ 1040 1041 static int tasks_tracing_torture_read_lock(void) 1042 { 1043 rcu_read_lock_trace(); 1044 return 0; 1045 } 1046 1047 static void tasks_tracing_torture_read_unlock(int idx) 1048 { 1049 rcu_read_unlock_trace(); 1050 } 1051 1052 static void rcu_tasks_tracing_torture_deferred_free(struct rcu_torture *p) 1053 { 1054 call_rcu_tasks_trace(&p->rtort_rcu, rcu_torture_cb); 1055 } 1056 1057 static struct rcu_torture_ops tasks_tracing_ops = { 1058 .ttype = RCU_TASKS_TRACING_FLAVOR, 1059 .init = rcu_sync_torture_init, 1060 .readlock = tasks_tracing_torture_read_lock, 1061 .read_delay = srcu_read_delay, /* just reuse srcu's version. */ 1062 .readunlock = tasks_tracing_torture_read_unlock, 1063 .readlock_held = rcu_read_lock_trace_held, 1064 .get_gp_seq = rcu_no_completed, 1065 .deferred_free = rcu_tasks_tracing_torture_deferred_free, 1066 .sync = synchronize_rcu_tasks_trace, 1067 .exp_sync = synchronize_rcu_tasks_trace, 1068 .call = call_rcu_tasks_trace, 1069 .cb_barrier = rcu_barrier_tasks_trace, 1070 .gp_kthread_dbg = show_rcu_tasks_trace_gp_kthread, 1071 .get_gp_data = rcu_tasks_trace_get_gp_data, 1072 .cbflood_max = 50000, 1073 .irq_capable = 1, 1074 .slow_gps = 1, 1075 .name = "tasks-tracing" 1076 }; 1077 1078 #define TASKS_TRACING_OPS &tasks_tracing_ops, 1079 1080 #else // #ifdef CONFIG_TASKS_TRACE_RCU 1081 1082 #define TASKS_TRACING_OPS 1083 1084 #endif // #else #ifdef CONFIG_TASKS_TRACE_RCU 1085 1086 1087 static unsigned long rcutorture_seq_diff(unsigned long new, unsigned long old) 1088 { 1089 if (!cur_ops->gp_diff) 1090 return new - old; 1091 return cur_ops->gp_diff(new, old); 1092 } 1093 1094 /* 1095 * RCU torture priority-boost testing. Runs one real-time thread per 1096 * CPU for moderate bursts, repeatedly starting grace periods and waiting 1097 * for them to complete. If a given grace period takes too long, we assume 1098 * that priority inversion has occurred. 1099 */ 1100 1101 static int old_rt_runtime = -1; 1102 1103 static void rcu_torture_disable_rt_throttle(void) 1104 { 1105 /* 1106 * Disable RT throttling so that rcutorture's boost threads don't get 1107 * throttled. Only possible if rcutorture is built-in otherwise the 1108 * user should manually do this by setting the sched_rt_period_us and 1109 * sched_rt_runtime sysctls. 1110 */ 1111 if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime != -1) 1112 return; 1113 1114 old_rt_runtime = sysctl_sched_rt_runtime; 1115 sysctl_sched_rt_runtime = -1; 1116 } 1117 1118 static void rcu_torture_enable_rt_throttle(void) 1119 { 1120 if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime == -1) 1121 return; 1122 1123 sysctl_sched_rt_runtime = old_rt_runtime; 1124 old_rt_runtime = -1; 1125 } 1126 1127 static bool rcu_torture_boost_failed(unsigned long gp_state, unsigned long *start) 1128 { 1129 int cpu; 1130 static int dbg_done; 1131 unsigned long end = jiffies; 1132 bool gp_done; 1133 unsigned long j; 1134 static unsigned long last_persist; 1135 unsigned long lp; 1136 unsigned long mininterval = test_boost_duration * HZ - HZ / 2; 1137 1138 if (end - *start > mininterval) { 1139 // Recheck after checking time to avoid false positives. 1140 smp_mb(); // Time check before grace-period check. 1141 if (cur_ops->poll_gp_state(gp_state)) 1142 return false; // passed, though perhaps just barely 1143 if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, &cpu)) { 1144 // At most one persisted message per boost test. 1145 j = jiffies; 1146 lp = READ_ONCE(last_persist); 1147 if (time_after(j, lp + mininterval) && 1148 cmpxchg(&last_persist, lp, j) == lp) { 1149 if (cpu < 0) 1150 pr_info("Boost inversion persisted: QS from all CPUs\n"); 1151 else 1152 pr_info("Boost inversion persisted: No QS from CPU %d\n", cpu); 1153 } 1154 return false; // passed on a technicality 1155 } 1156 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed"); 1157 n_rcu_torture_boost_failure++; 1158 if (!xchg(&dbg_done, 1) && cur_ops->gp_kthread_dbg) { 1159 pr_info("Boost inversion thread ->rt_priority %u gp_state %lu jiffies %lu\n", 1160 current->rt_priority, gp_state, end - *start); 1161 cur_ops->gp_kthread_dbg(); 1162 // Recheck after print to flag grace period ending during splat. 1163 gp_done = cur_ops->poll_gp_state(gp_state); 1164 pr_info("Boost inversion: GP %lu %s.\n", gp_state, 1165 gp_done ? "ended already" : "still pending"); 1166 1167 } 1168 1169 return true; // failed 1170 } else if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, NULL)) { 1171 *start = jiffies; 1172 } 1173 1174 return false; // passed 1175 } 1176 1177 static int rcu_torture_boost(void *arg) 1178 { 1179 unsigned long endtime; 1180 unsigned long gp_state; 1181 unsigned long gp_state_time; 1182 unsigned long oldstarttime; 1183 unsigned long booststarttime = get_torture_init_jiffies() + test_boost_holdoff * HZ; 1184 1185 if (test_boost_holdoff <= 0 || time_after(jiffies, booststarttime)) { 1186 VERBOSE_TOROUT_STRING("rcu_torture_boost started"); 1187 } else { 1188 VERBOSE_TOROUT_STRING("rcu_torture_boost started holdoff period"); 1189 while (time_before(jiffies, booststarttime)) { 1190 schedule_timeout_idle(HZ); 1191 if (kthread_should_stop()) 1192 goto cleanup; 1193 } 1194 VERBOSE_TOROUT_STRING("rcu_torture_boost finished holdoff period"); 1195 } 1196 1197 /* Set real-time priority. */ 1198 sched_set_fifo_low(current); 1199 1200 /* Each pass through the following loop does one boost-test cycle. */ 1201 do { 1202 bool failed = false; // Test failed already in this test interval 1203 bool gp_initiated = false; 1204 1205 if (kthread_should_stop()) 1206 goto checkwait; 1207 1208 /* Wait for the next test interval. */ 1209 oldstarttime = READ_ONCE(boost_starttime); 1210 while (time_before(jiffies, oldstarttime)) { 1211 schedule_timeout_interruptible(oldstarttime - jiffies); 1212 if (stutter_wait("rcu_torture_boost")) 1213 sched_set_fifo_low(current); 1214 if (torture_must_stop()) 1215 goto checkwait; 1216 } 1217 1218 // Do one boost-test interval. 1219 endtime = oldstarttime + test_boost_duration * HZ; 1220 while (time_before(jiffies, endtime)) { 1221 // Has current GP gone too long? 1222 if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state)) 1223 failed = rcu_torture_boost_failed(gp_state, &gp_state_time); 1224 // If we don't have a grace period in flight, start one. 1225 if (!gp_initiated || cur_ops->poll_gp_state(gp_state)) { 1226 gp_state = cur_ops->start_gp_poll(); 1227 gp_initiated = true; 1228 gp_state_time = jiffies; 1229 } 1230 if (stutter_wait("rcu_torture_boost")) { 1231 sched_set_fifo_low(current); 1232 // If the grace period already ended, 1233 // we don't know when that happened, so 1234 // start over. 1235 if (cur_ops->poll_gp_state(gp_state)) 1236 gp_initiated = false; 1237 } 1238 if (torture_must_stop()) 1239 goto checkwait; 1240 } 1241 1242 // In case the grace period extended beyond the end of the loop. 1243 if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state)) 1244 rcu_torture_boost_failed(gp_state, &gp_state_time); 1245 1246 /* 1247 * Set the start time of the next test interval. 1248 * Yes, this is vulnerable to long delays, but such 1249 * delays simply cause a false negative for the next 1250 * interval. Besides, we are running at RT priority, 1251 * so delays should be relatively rare. 1252 */ 1253 while (oldstarttime == READ_ONCE(boost_starttime) && !kthread_should_stop()) { 1254 if (mutex_trylock(&boost_mutex)) { 1255 if (oldstarttime == boost_starttime) { 1256 WRITE_ONCE(boost_starttime, 1257 jiffies + test_boost_interval * HZ); 1258 n_rcu_torture_boosts++; 1259 } 1260 mutex_unlock(&boost_mutex); 1261 break; 1262 } 1263 schedule_timeout_uninterruptible(HZ / 20); 1264 } 1265 1266 /* Go do the stutter. */ 1267 checkwait: if (stutter_wait("rcu_torture_boost")) 1268 sched_set_fifo_low(current); 1269 } while (!torture_must_stop()); 1270 1271 cleanup: 1272 /* Clean up and exit. */ 1273 while (!kthread_should_stop()) { 1274 torture_shutdown_absorb("rcu_torture_boost"); 1275 schedule_timeout_uninterruptible(HZ / 20); 1276 } 1277 torture_kthread_stopping("rcu_torture_boost"); 1278 return 0; 1279 } 1280 1281 /* 1282 * RCU torture force-quiescent-state kthread. Repeatedly induces 1283 * bursts of calls to force_quiescent_state(), increasing the probability 1284 * of occurrence of some important types of race conditions. 1285 */ 1286 static int 1287 rcu_torture_fqs(void *arg) 1288 { 1289 unsigned long fqs_resume_time; 1290 int fqs_burst_remaining; 1291 int oldnice = task_nice(current); 1292 1293 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started"); 1294 do { 1295 fqs_resume_time = jiffies + fqs_stutter * HZ; 1296 while (time_before(jiffies, fqs_resume_time) && 1297 !kthread_should_stop()) { 1298 schedule_timeout_interruptible(HZ / 20); 1299 } 1300 fqs_burst_remaining = fqs_duration; 1301 while (fqs_burst_remaining > 0 && 1302 !kthread_should_stop()) { 1303 cur_ops->fqs(); 1304 udelay(fqs_holdoff); 1305 fqs_burst_remaining -= fqs_holdoff; 1306 } 1307 if (stutter_wait("rcu_torture_fqs")) 1308 sched_set_normal(current, oldnice); 1309 } while (!torture_must_stop()); 1310 torture_kthread_stopping("rcu_torture_fqs"); 1311 return 0; 1312 } 1313 1314 // Used by writers to randomly choose from the available grace-period primitives. 1315 static int synctype[ARRAY_SIZE(rcu_torture_writer_state_names)] = { }; 1316 static int nsynctypes; 1317 1318 /* 1319 * Determine which grace-period primitives are available. 1320 */ 1321 static void rcu_torture_write_types(void) 1322 { 1323 bool gp_cond1 = gp_cond, gp_cond_exp1 = gp_cond_exp, gp_cond_full1 = gp_cond_full; 1324 bool gp_cond_exp_full1 = gp_cond_exp_full, gp_exp1 = gp_exp, gp_poll_exp1 = gp_poll_exp; 1325 bool gp_poll_exp_full1 = gp_poll_exp_full, gp_normal1 = gp_normal, gp_poll1 = gp_poll; 1326 bool gp_poll_full1 = gp_poll_full, gp_sync1 = gp_sync; 1327 1328 /* Initialize synctype[] array. If none set, take default. */ 1329 if (!gp_cond1 && 1330 !gp_cond_exp1 && 1331 !gp_cond_full1 && 1332 !gp_cond_exp_full1 && 1333 !gp_exp1 && 1334 !gp_poll_exp1 && 1335 !gp_poll_exp_full1 && 1336 !gp_normal1 && 1337 !gp_poll1 && 1338 !gp_poll_full1 && 1339 !gp_sync1) { 1340 gp_cond1 = true; 1341 gp_cond_exp1 = true; 1342 gp_cond_full1 = true; 1343 gp_cond_exp_full1 = true; 1344 gp_exp1 = true; 1345 gp_poll_exp1 = true; 1346 gp_poll_exp_full1 = true; 1347 gp_normal1 = true; 1348 gp_poll1 = true; 1349 gp_poll_full1 = true; 1350 gp_sync1 = true; 1351 } 1352 if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) { 1353 synctype[nsynctypes++] = RTWS_COND_GET; 1354 pr_info("%s: Testing conditional GPs.\n", __func__); 1355 } else if (gp_cond && (!cur_ops->get_gp_state || !cur_ops->cond_sync)) { 1356 pr_alert("%s: gp_cond without primitives.\n", __func__); 1357 } 1358 if (gp_cond_exp1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp) { 1359 synctype[nsynctypes++] = RTWS_COND_GET_EXP; 1360 pr_info("%s: Testing conditional expedited GPs.\n", __func__); 1361 } else if (gp_cond_exp && (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp)) { 1362 pr_alert("%s: gp_cond_exp without primitives.\n", __func__); 1363 } 1364 if (gp_cond_full1 && cur_ops->get_gp_state && cur_ops->cond_sync_full) { 1365 synctype[nsynctypes++] = RTWS_COND_GET_FULL; 1366 pr_info("%s: Testing conditional full-state GPs.\n", __func__); 1367 } else if (gp_cond_full && (!cur_ops->get_gp_state || !cur_ops->cond_sync_full)) { 1368 pr_alert("%s: gp_cond_full without primitives.\n", __func__); 1369 } 1370 if (gp_cond_exp_full1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp_full) { 1371 synctype[nsynctypes++] = RTWS_COND_GET_EXP_FULL; 1372 pr_info("%s: Testing conditional full-state expedited GPs.\n", __func__); 1373 } else if (gp_cond_exp_full && 1374 (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp_full)) { 1375 pr_alert("%s: gp_cond_exp_full without primitives.\n", __func__); 1376 } 1377 if (gp_exp1 && cur_ops->exp_sync) { 1378 synctype[nsynctypes++] = RTWS_EXP_SYNC; 1379 pr_info("%s: Testing expedited GPs.\n", __func__); 1380 } else if (gp_exp && !cur_ops->exp_sync) { 1381 pr_alert("%s: gp_exp without primitives.\n", __func__); 1382 } 1383 if (gp_normal1 && cur_ops->deferred_free) { 1384 synctype[nsynctypes++] = RTWS_DEF_FREE; 1385 pr_info("%s: Testing asynchronous GPs.\n", __func__); 1386 } else if (gp_normal && !cur_ops->deferred_free) { 1387 pr_alert("%s: gp_normal without primitives.\n", __func__); 1388 } 1389 if (gp_poll1 && cur_ops->get_comp_state && cur_ops->same_gp_state && 1390 cur_ops->start_gp_poll && cur_ops->poll_gp_state) { 1391 synctype[nsynctypes++] = RTWS_POLL_GET; 1392 pr_info("%s: Testing polling GPs.\n", __func__); 1393 } else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) { 1394 pr_alert("%s: gp_poll without primitives.\n", __func__); 1395 } 1396 if (gp_poll_full1 && cur_ops->get_comp_state_full && cur_ops->same_gp_state_full 1397 && cur_ops->start_gp_poll_full && cur_ops->poll_gp_state_full) { 1398 synctype[nsynctypes++] = RTWS_POLL_GET_FULL; 1399 pr_info("%s: Testing polling full-state GPs.\n", __func__); 1400 } else if (gp_poll_full && (!cur_ops->start_gp_poll_full || !cur_ops->poll_gp_state_full)) { 1401 pr_alert("%s: gp_poll_full without primitives.\n", __func__); 1402 } 1403 if (gp_poll_exp1 && cur_ops->start_gp_poll_exp && cur_ops->poll_gp_state_exp) { 1404 synctype[nsynctypes++] = RTWS_POLL_GET_EXP; 1405 pr_info("%s: Testing polling expedited GPs.\n", __func__); 1406 } else if (gp_poll_exp && (!cur_ops->start_gp_poll_exp || !cur_ops->poll_gp_state_exp)) { 1407 pr_alert("%s: gp_poll_exp without primitives.\n", __func__); 1408 } 1409 if (gp_poll_exp_full1 && cur_ops->start_gp_poll_exp_full && cur_ops->poll_gp_state_full) { 1410 synctype[nsynctypes++] = RTWS_POLL_GET_EXP_FULL; 1411 pr_info("%s: Testing polling full-state expedited GPs.\n", __func__); 1412 } else if (gp_poll_exp_full && 1413 (!cur_ops->start_gp_poll_exp_full || !cur_ops->poll_gp_state_full)) { 1414 pr_alert("%s: gp_poll_exp_full without primitives.\n", __func__); 1415 } 1416 if (gp_sync1 && cur_ops->sync) { 1417 synctype[nsynctypes++] = RTWS_SYNC; 1418 pr_info("%s: Testing normal GPs.\n", __func__); 1419 } else if (gp_sync && !cur_ops->sync) { 1420 pr_alert("%s: gp_sync without primitives.\n", __func__); 1421 } 1422 pr_alert("%s: Testing %d update types.\n", __func__, nsynctypes); 1423 pr_info("%s: gp_cond_wi %d gp_cond_wi_exp %d gp_poll_wi %d gp_poll_wi_exp %d\n", __func__, gp_cond_wi, gp_cond_wi_exp, gp_poll_wi, gp_poll_wi_exp); 1424 } 1425 1426 /* 1427 * Do the specified rcu_torture_writer() synchronous grace period, 1428 * while also testing out the polled APIs. Note well that the single-CPU 1429 * grace-period optimizations must be accounted for. 1430 */ 1431 static void do_rtws_sync(struct torture_random_state *trsp, void (*sync)(void)) 1432 { 1433 unsigned long cookie; 1434 struct rcu_gp_oldstate cookie_full; 1435 bool dopoll; 1436 bool dopoll_full; 1437 unsigned long r = torture_random(trsp); 1438 1439 dopoll = cur_ops->get_gp_state && cur_ops->poll_gp_state && !(r & 0x300); 1440 dopoll_full = cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full && !(r & 0xc00); 1441 if (dopoll || dopoll_full) 1442 cpus_read_lock(); 1443 if (dopoll) 1444 cookie = cur_ops->get_gp_state(); 1445 if (dopoll_full) 1446 cur_ops->get_gp_state_full(&cookie_full); 1447 if (cur_ops->poll_need_2gp && cur_ops->poll_need_2gp(dopoll, dopoll_full)) 1448 sync(); 1449 sync(); 1450 WARN_ONCE(dopoll && !cur_ops->poll_gp_state(cookie), 1451 "%s: Cookie check 3 failed %pS() online %*pbl.", 1452 __func__, sync, cpumask_pr_args(cpu_online_mask)); 1453 WARN_ONCE(dopoll_full && !cur_ops->poll_gp_state_full(&cookie_full), 1454 "%s: Cookie check 4 failed %pS() online %*pbl", 1455 __func__, sync, cpumask_pr_args(cpu_online_mask)); 1456 if (dopoll || dopoll_full) 1457 cpus_read_unlock(); 1458 } 1459 1460 /* 1461 * RCU torture writer kthread. Repeatedly substitutes a new structure 1462 * for that pointed to by rcu_torture_current, freeing the old structure 1463 * after a series of grace periods (the "pipeline"). 1464 */ 1465 static int 1466 rcu_torture_writer(void *arg) 1467 { 1468 bool boot_ended; 1469 bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal(); 1470 unsigned long cookie; 1471 struct rcu_gp_oldstate cookie_full; 1472 int expediting = 0; 1473 unsigned long gp_snap; 1474 unsigned long gp_snap1; 1475 struct rcu_gp_oldstate gp_snap_full; 1476 struct rcu_gp_oldstate gp_snap1_full; 1477 int i; 1478 int idx; 1479 int oldnice = task_nice(current); 1480 struct rcu_gp_oldstate *rgo = NULL; 1481 int rgo_size = 0; 1482 struct rcu_torture *rp; 1483 struct rcu_torture *old_rp; 1484 static DEFINE_TORTURE_RANDOM(rand); 1485 unsigned long stallsdone = jiffies; 1486 bool stutter_waited; 1487 unsigned long *ulo = NULL; 1488 int ulo_size = 0; 1489 1490 // If a new stall test is added, this must be adjusted. 1491 if (stall_cpu_holdoff + stall_gp_kthread + stall_cpu) 1492 stallsdone += (stall_cpu_holdoff + stall_gp_kthread + stall_cpu + 60) * 1493 HZ * (stall_cpu_repeat + 1); 1494 VERBOSE_TOROUT_STRING("rcu_torture_writer task started"); 1495 if (!can_expedite) 1496 pr_alert("%s" TORTURE_FLAG 1497 " GP expediting controlled from boot/sysfs for %s.\n", 1498 torture_type, cur_ops->name); 1499 if (WARN_ONCE(nsynctypes == 0, 1500 "%s: No update-side primitives.\n", __func__)) { 1501 /* 1502 * No updates primitives, so don't try updating. 1503 * The resulting test won't be testing much, hence the 1504 * above WARN_ONCE(). 1505 */ 1506 rcu_torture_writer_state = RTWS_STOPPING; 1507 torture_kthread_stopping("rcu_torture_writer"); 1508 return 0; 1509 } 1510 if (cur_ops->poll_active > 0) { 1511 ulo = kzalloc(cur_ops->poll_active * sizeof(ulo[0]), GFP_KERNEL); 1512 if (!WARN_ON(!ulo)) 1513 ulo_size = cur_ops->poll_active; 1514 } 1515 if (cur_ops->poll_active_full > 0) { 1516 rgo = kzalloc(cur_ops->poll_active_full * sizeof(rgo[0]), GFP_KERNEL); 1517 if (!WARN_ON(!rgo)) 1518 rgo_size = cur_ops->poll_active_full; 1519 } 1520 1521 do { 1522 rcu_torture_writer_state = RTWS_FIXED_DELAY; 1523 torture_hrtimeout_us(500, 1000, &rand); 1524 rp = rcu_torture_alloc(); 1525 if (rp == NULL) 1526 continue; 1527 rp->rtort_pipe_count = 0; 1528 ASSERT_EXCLUSIVE_WRITER(rp->rtort_pipe_count); 1529 rcu_torture_writer_state = RTWS_DELAY; 1530 udelay(torture_random(&rand) & 0x3ff); 1531 rcu_torture_writer_state = RTWS_REPLACE; 1532 old_rp = rcu_dereference_check(rcu_torture_current, 1533 current == writer_task); 1534 rp->rtort_mbtest = 1; 1535 rcu_assign_pointer(rcu_torture_current, rp); 1536 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ 1537 if (old_rp) { 1538 i = old_rp->rtort_pipe_count; 1539 if (i > RCU_TORTURE_PIPE_LEN) 1540 i = RCU_TORTURE_PIPE_LEN; 1541 atomic_inc(&rcu_torture_wcount[i]); 1542 WRITE_ONCE(old_rp->rtort_pipe_count, 1543 old_rp->rtort_pipe_count + 1); 1544 ASSERT_EXCLUSIVE_WRITER(old_rp->rtort_pipe_count); 1545 1546 // Make sure readers block polled grace periods. 1547 if (cur_ops->get_gp_state && cur_ops->poll_gp_state) { 1548 idx = cur_ops->readlock(); 1549 cookie = cur_ops->get_gp_state(); 1550 WARN_ONCE(cur_ops->poll_gp_state(cookie), 1551 "%s: Cookie check 1 failed %s(%d) %lu->%lu\n", 1552 __func__, 1553 rcu_torture_writer_state_getname(), 1554 rcu_torture_writer_state, 1555 cookie, cur_ops->get_gp_state()); 1556 if (cur_ops->get_comp_state) { 1557 cookie = cur_ops->get_comp_state(); 1558 WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie)); 1559 } 1560 cur_ops->readunlock(idx); 1561 } 1562 if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) { 1563 idx = cur_ops->readlock(); 1564 cur_ops->get_gp_state_full(&cookie_full); 1565 WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full), 1566 "%s: Cookie check 5 failed %s(%d) online %*pbl\n", 1567 __func__, 1568 rcu_torture_writer_state_getname(), 1569 rcu_torture_writer_state, 1570 cpumask_pr_args(cpu_online_mask)); 1571 if (cur_ops->get_comp_state_full) { 1572 cur_ops->get_comp_state_full(&cookie_full); 1573 WARN_ON_ONCE(!cur_ops->poll_gp_state_full(&cookie_full)); 1574 } 1575 cur_ops->readunlock(idx); 1576 } 1577 switch (synctype[torture_random(&rand) % nsynctypes]) { 1578 case RTWS_DEF_FREE: 1579 rcu_torture_writer_state = RTWS_DEF_FREE; 1580 cur_ops->deferred_free(old_rp); 1581 break; 1582 case RTWS_EXP_SYNC: 1583 rcu_torture_writer_state = RTWS_EXP_SYNC; 1584 do_rtws_sync(&rand, cur_ops->exp_sync); 1585 rcu_torture_pipe_update(old_rp); 1586 break; 1587 case RTWS_COND_GET: 1588 rcu_torture_writer_state = RTWS_COND_GET; 1589 gp_snap = cur_ops->get_gp_state(); 1590 torture_hrtimeout_us(torture_random(&rand) % gp_cond_wi, 1591 1000, &rand); 1592 rcu_torture_writer_state = RTWS_COND_SYNC; 1593 cur_ops->cond_sync(gp_snap); 1594 rcu_torture_pipe_update(old_rp); 1595 break; 1596 case RTWS_COND_GET_EXP: 1597 rcu_torture_writer_state = RTWS_COND_GET_EXP; 1598 gp_snap = cur_ops->get_gp_state_exp(); 1599 torture_hrtimeout_us(torture_random(&rand) % gp_cond_wi_exp, 1600 1000, &rand); 1601 rcu_torture_writer_state = RTWS_COND_SYNC_EXP; 1602 cur_ops->cond_sync_exp(gp_snap); 1603 rcu_torture_pipe_update(old_rp); 1604 break; 1605 case RTWS_COND_GET_FULL: 1606 rcu_torture_writer_state = RTWS_COND_GET_FULL; 1607 cur_ops->get_gp_state_full(&gp_snap_full); 1608 torture_hrtimeout_us(torture_random(&rand) % gp_cond_wi, 1609 1000, &rand); 1610 rcu_torture_writer_state = RTWS_COND_SYNC_FULL; 1611 cur_ops->cond_sync_full(&gp_snap_full); 1612 rcu_torture_pipe_update(old_rp); 1613 break; 1614 case RTWS_COND_GET_EXP_FULL: 1615 rcu_torture_writer_state = RTWS_COND_GET_EXP_FULL; 1616 cur_ops->get_gp_state_full(&gp_snap_full); 1617 torture_hrtimeout_us(torture_random(&rand) % gp_cond_wi_exp, 1618 1000, &rand); 1619 rcu_torture_writer_state = RTWS_COND_SYNC_EXP_FULL; 1620 cur_ops->cond_sync_exp_full(&gp_snap_full); 1621 rcu_torture_pipe_update(old_rp); 1622 break; 1623 case RTWS_POLL_GET: 1624 rcu_torture_writer_state = RTWS_POLL_GET; 1625 for (i = 0; i < ulo_size; i++) 1626 ulo[i] = cur_ops->get_comp_state(); 1627 gp_snap = cur_ops->start_gp_poll(); 1628 rcu_torture_writer_state = RTWS_POLL_WAIT; 1629 while (!cur_ops->poll_gp_state(gp_snap)) { 1630 gp_snap1 = cur_ops->get_gp_state(); 1631 for (i = 0; i < ulo_size; i++) 1632 if (cur_ops->poll_gp_state(ulo[i]) || 1633 cur_ops->same_gp_state(ulo[i], gp_snap1)) { 1634 ulo[i] = gp_snap1; 1635 break; 1636 } 1637 WARN_ON_ONCE(ulo_size > 0 && i >= ulo_size); 1638 torture_hrtimeout_us(torture_random(&rand) % gp_poll_wi, 1639 1000, &rand); 1640 } 1641 rcu_torture_pipe_update(old_rp); 1642 break; 1643 case RTWS_POLL_GET_FULL: 1644 rcu_torture_writer_state = RTWS_POLL_GET_FULL; 1645 for (i = 0; i < rgo_size; i++) 1646 cur_ops->get_comp_state_full(&rgo[i]); 1647 cur_ops->start_gp_poll_full(&gp_snap_full); 1648 rcu_torture_writer_state = RTWS_POLL_WAIT_FULL; 1649 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { 1650 cur_ops->get_gp_state_full(&gp_snap1_full); 1651 for (i = 0; i < rgo_size; i++) 1652 if (cur_ops->poll_gp_state_full(&rgo[i]) || 1653 cur_ops->same_gp_state_full(&rgo[i], 1654 &gp_snap1_full)) { 1655 rgo[i] = gp_snap1_full; 1656 break; 1657 } 1658 WARN_ON_ONCE(rgo_size > 0 && i >= rgo_size); 1659 torture_hrtimeout_us(torture_random(&rand) % gp_poll_wi, 1660 1000, &rand); 1661 } 1662 rcu_torture_pipe_update(old_rp); 1663 break; 1664 case RTWS_POLL_GET_EXP: 1665 rcu_torture_writer_state = RTWS_POLL_GET_EXP; 1666 gp_snap = cur_ops->start_gp_poll_exp(); 1667 rcu_torture_writer_state = RTWS_POLL_WAIT_EXP; 1668 while (!cur_ops->poll_gp_state_exp(gp_snap)) 1669 torture_hrtimeout_us(torture_random(&rand) % gp_poll_wi_exp, 1670 1000, &rand); 1671 rcu_torture_pipe_update(old_rp); 1672 break; 1673 case RTWS_POLL_GET_EXP_FULL: 1674 rcu_torture_writer_state = RTWS_POLL_GET_EXP_FULL; 1675 cur_ops->start_gp_poll_exp_full(&gp_snap_full); 1676 rcu_torture_writer_state = RTWS_POLL_WAIT_EXP_FULL; 1677 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) 1678 torture_hrtimeout_us(torture_random(&rand) % gp_poll_wi_exp, 1679 1000, &rand); 1680 rcu_torture_pipe_update(old_rp); 1681 break; 1682 case RTWS_SYNC: 1683 rcu_torture_writer_state = RTWS_SYNC; 1684 do_rtws_sync(&rand, cur_ops->sync); 1685 rcu_torture_pipe_update(old_rp); 1686 break; 1687 default: 1688 WARN_ON_ONCE(1); 1689 break; 1690 } 1691 } 1692 WRITE_ONCE(rcu_torture_current_version, 1693 rcu_torture_current_version + 1); 1694 /* Cycle through nesting levels of rcu_expedite_gp() calls. */ 1695 if (can_expedite && 1696 !(torture_random(&rand) & 0xff & (!!expediting - 1))) { 1697 WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited()); 1698 if (expediting >= 0) 1699 rcu_expedite_gp(); 1700 else 1701 rcu_unexpedite_gp(); 1702 if (++expediting > 3) 1703 expediting = -expediting; 1704 } else if (!can_expedite) { /* Disabled during boot, recheck. */ 1705 can_expedite = !rcu_gp_is_expedited() && 1706 !rcu_gp_is_normal(); 1707 } 1708 rcu_torture_writer_state = RTWS_STUTTER; 1709 boot_ended = rcu_inkernel_boot_has_ended(); 1710 stutter_waited = stutter_wait("rcu_torture_writer"); 1711 if (stutter_waited && 1712 !atomic_read(&rcu_fwd_cb_nodelay) && 1713 !cur_ops->slow_gps && 1714 !torture_must_stop() && 1715 boot_ended && 1716 time_after(jiffies, stallsdone)) 1717 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) 1718 if (list_empty(&rcu_tortures[i].rtort_free) && 1719 rcu_access_pointer(rcu_torture_current) != &rcu_tortures[i]) { 1720 tracing_off(); 1721 if (cur_ops->gp_kthread_dbg) 1722 cur_ops->gp_kthread_dbg(); 1723 WARN(1, "%s: rtort_pipe_count: %d\n", __func__, rcu_tortures[i].rtort_pipe_count); 1724 rcu_ftrace_dump(DUMP_ALL); 1725 } 1726 if (stutter_waited) 1727 sched_set_normal(current, oldnice); 1728 } while (!torture_must_stop()); 1729 rcu_torture_current = NULL; // Let stats task know that we are done. 1730 /* Reset expediting back to unexpedited. */ 1731 if (expediting > 0) 1732 expediting = -expediting; 1733 while (can_expedite && expediting++ < 0) 1734 rcu_unexpedite_gp(); 1735 WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited()); 1736 if (!can_expedite) 1737 pr_alert("%s" TORTURE_FLAG 1738 " Dynamic grace-period expediting was disabled.\n", 1739 torture_type); 1740 kfree(ulo); 1741 kfree(rgo); 1742 rcu_torture_writer_state = RTWS_STOPPING; 1743 torture_kthread_stopping("rcu_torture_writer"); 1744 return 0; 1745 } 1746 1747 /* 1748 * RCU torture fake writer kthread. Repeatedly calls sync, with a random 1749 * delay between calls. 1750 */ 1751 static int 1752 rcu_torture_fakewriter(void *arg) 1753 { 1754 unsigned long gp_snap; 1755 struct rcu_gp_oldstate gp_snap_full; 1756 DEFINE_TORTURE_RANDOM(rand); 1757 1758 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); 1759 set_user_nice(current, MAX_NICE); 1760 1761 if (WARN_ONCE(nsynctypes == 0, 1762 "%s: No update-side primitives.\n", __func__)) { 1763 /* 1764 * No updates primitives, so don't try updating. 1765 * The resulting test won't be testing much, hence the 1766 * above WARN_ONCE(). 1767 */ 1768 torture_kthread_stopping("rcu_torture_fakewriter"); 1769 return 0; 1770 } 1771 1772 do { 1773 torture_hrtimeout_jiffies(torture_random(&rand) % 10, &rand); 1774 if (cur_ops->cb_barrier != NULL && 1775 torture_random(&rand) % (nrealfakewriters * 8) == 0) { 1776 cur_ops->cb_barrier(); 1777 } else { 1778 switch (synctype[torture_random(&rand) % nsynctypes]) { 1779 case RTWS_DEF_FREE: 1780 break; 1781 case RTWS_EXP_SYNC: 1782 cur_ops->exp_sync(); 1783 break; 1784 case RTWS_COND_GET: 1785 gp_snap = cur_ops->get_gp_state(); 1786 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1787 cur_ops->cond_sync(gp_snap); 1788 break; 1789 case RTWS_COND_GET_EXP: 1790 gp_snap = cur_ops->get_gp_state_exp(); 1791 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1792 cur_ops->cond_sync_exp(gp_snap); 1793 break; 1794 case RTWS_COND_GET_FULL: 1795 cur_ops->get_gp_state_full(&gp_snap_full); 1796 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1797 cur_ops->cond_sync_full(&gp_snap_full); 1798 break; 1799 case RTWS_COND_GET_EXP_FULL: 1800 cur_ops->get_gp_state_full(&gp_snap_full); 1801 torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); 1802 cur_ops->cond_sync_exp_full(&gp_snap_full); 1803 break; 1804 case RTWS_POLL_GET: 1805 if (cur_ops->start_poll_irqsoff) 1806 local_irq_disable(); 1807 gp_snap = cur_ops->start_gp_poll(); 1808 if (cur_ops->start_poll_irqsoff) 1809 local_irq_enable(); 1810 while (!cur_ops->poll_gp_state(gp_snap)) { 1811 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1812 &rand); 1813 } 1814 break; 1815 case RTWS_POLL_GET_FULL: 1816 if (cur_ops->start_poll_irqsoff) 1817 local_irq_disable(); 1818 cur_ops->start_gp_poll_full(&gp_snap_full); 1819 if (cur_ops->start_poll_irqsoff) 1820 local_irq_enable(); 1821 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { 1822 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1823 &rand); 1824 } 1825 break; 1826 case RTWS_POLL_GET_EXP: 1827 gp_snap = cur_ops->start_gp_poll_exp(); 1828 while (!cur_ops->poll_gp_state_exp(gp_snap)) { 1829 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1830 &rand); 1831 } 1832 break; 1833 case RTWS_POLL_GET_EXP_FULL: 1834 cur_ops->start_gp_poll_exp_full(&gp_snap_full); 1835 while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { 1836 torture_hrtimeout_jiffies(torture_random(&rand) % 16, 1837 &rand); 1838 } 1839 break; 1840 case RTWS_SYNC: 1841 cur_ops->sync(); 1842 break; 1843 default: 1844 WARN_ON_ONCE(1); 1845 break; 1846 } 1847 } 1848 stutter_wait("rcu_torture_fakewriter"); 1849 } while (!torture_must_stop()); 1850 1851 torture_kthread_stopping("rcu_torture_fakewriter"); 1852 return 0; 1853 } 1854 1855 static void rcu_torture_timer_cb(struct rcu_head *rhp) 1856 { 1857 kfree(rhp); 1858 } 1859 1860 // Set up and carry out testing of RCU's global memory ordering 1861 static void rcu_torture_reader_do_mbchk(long myid, struct rcu_torture *rtp, 1862 struct torture_random_state *trsp) 1863 { 1864 unsigned long loops; 1865 int noc = torture_num_online_cpus(); 1866 int rdrchked; 1867 int rdrchker; 1868 struct rcu_torture_reader_check *rtrcp; // Me. 1869 struct rcu_torture_reader_check *rtrcp_assigner; // Assigned us to do checking. 1870 struct rcu_torture_reader_check *rtrcp_chked; // Reader being checked. 1871 struct rcu_torture_reader_check *rtrcp_chker; // Reader doing checking when not me. 1872 1873 if (myid < 0) 1874 return; // Don't try this from timer handlers. 1875 1876 // Increment my counter. 1877 rtrcp = &rcu_torture_reader_mbchk[myid]; 1878 WRITE_ONCE(rtrcp->rtc_myloops, rtrcp->rtc_myloops + 1); 1879 1880 // Attempt to assign someone else some checking work. 1881 rdrchked = torture_random(trsp) % nrealreaders; 1882 rtrcp_chked = &rcu_torture_reader_mbchk[rdrchked]; 1883 rdrchker = torture_random(trsp) % nrealreaders; 1884 rtrcp_chker = &rcu_torture_reader_mbchk[rdrchker]; 1885 if (rdrchked != myid && rdrchked != rdrchker && noc >= rdrchked && noc >= rdrchker && 1886 smp_load_acquire(&rtrcp->rtc_chkrdr) < 0 && // Pairs with smp_store_release below. 1887 !READ_ONCE(rtp->rtort_chkp) && 1888 !smp_load_acquire(&rtrcp_chker->rtc_assigner)) { // Pairs with smp_store_release below. 1889 rtrcp->rtc_chkloops = READ_ONCE(rtrcp_chked->rtc_myloops); 1890 WARN_ON_ONCE(rtrcp->rtc_chkrdr >= 0); 1891 rtrcp->rtc_chkrdr = rdrchked; 1892 WARN_ON_ONCE(rtrcp->rtc_ready); // This gets set after the grace period ends. 1893 if (cmpxchg_relaxed(&rtrcp_chker->rtc_assigner, NULL, rtrcp) || 1894 cmpxchg_relaxed(&rtp->rtort_chkp, NULL, rtrcp)) 1895 (void)cmpxchg_relaxed(&rtrcp_chker->rtc_assigner, rtrcp, NULL); // Back out. 1896 } 1897 1898 // If assigned some completed work, do it! 1899 rtrcp_assigner = READ_ONCE(rtrcp->rtc_assigner); 1900 if (!rtrcp_assigner || !smp_load_acquire(&rtrcp_assigner->rtc_ready)) 1901 return; // No work or work not yet ready. 1902 rdrchked = rtrcp_assigner->rtc_chkrdr; 1903 if (WARN_ON_ONCE(rdrchked < 0)) 1904 return; 1905 rtrcp_chked = &rcu_torture_reader_mbchk[rdrchked]; 1906 loops = READ_ONCE(rtrcp_chked->rtc_myloops); 1907 atomic_inc(&n_rcu_torture_mbchk_tries); 1908 if (ULONG_CMP_LT(loops, rtrcp_assigner->rtc_chkloops)) 1909 atomic_inc(&n_rcu_torture_mbchk_fail); 1910 rtrcp_assigner->rtc_chkloops = loops + ULONG_MAX / 2; 1911 rtrcp_assigner->rtc_ready = 0; 1912 smp_store_release(&rtrcp->rtc_assigner, NULL); // Someone else can assign us work. 1913 smp_store_release(&rtrcp_assigner->rtc_chkrdr, -1); // Assigner can again assign. 1914 } 1915 1916 // Verify the specified RCUTORTURE_RDR* state. 1917 #define ROEC_ARGS "%s %s: Current %#x To add %#x To remove %#x preempt_count() %#x\n", __func__, s, curstate, new, old, preempt_count() 1918 static void rcutorture_one_extend_check(char *s, int curstate, int new, int old, bool insoftirq) 1919 { 1920 int mask; 1921 1922 if (!IS_ENABLED(CONFIG_RCU_TORTURE_TEST_CHK_RDR_STATE)) 1923 return; 1924 1925 WARN_ONCE(!(curstate & RCUTORTURE_RDR_IRQ) && irqs_disabled(), ROEC_ARGS); 1926 WARN_ONCE((curstate & RCUTORTURE_RDR_IRQ) && !irqs_disabled(), ROEC_ARGS); 1927 1928 // If CONFIG_PREEMPT_COUNT=n, further checks are unreliable. 1929 if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) 1930 return; 1931 1932 WARN_ONCE((curstate & (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH)) && 1933 !(preempt_count() & SOFTIRQ_MASK), ROEC_ARGS); 1934 WARN_ONCE((curstate & (RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED)) && 1935 !(preempt_count() & PREEMPT_MASK), ROEC_ARGS); 1936 WARN_ONCE(cur_ops->readlock_nesting && 1937 (curstate & (RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2)) && 1938 cur_ops->readlock_nesting() == 0, ROEC_ARGS); 1939 1940 // Timer handlers have all sorts of stuff disabled, so ignore 1941 // unintended disabling. 1942 if (insoftirq) 1943 return; 1944 1945 WARN_ONCE(cur_ops->extendables && 1946 !(curstate & (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH)) && 1947 (preempt_count() & SOFTIRQ_MASK), ROEC_ARGS); 1948 1949 /* 1950 * non-preemptible RCU in a preemptible kernel uses preempt_disable() 1951 * as rcu_read_lock(). 1952 */ 1953 mask = RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED; 1954 if (!IS_ENABLED(CONFIG_PREEMPT_RCU)) 1955 mask |= RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2; 1956 1957 WARN_ONCE(cur_ops->extendables && !(curstate & mask) && 1958 (preempt_count() & PREEMPT_MASK), ROEC_ARGS); 1959 1960 /* 1961 * non-preemptible RCU in a preemptible kernel uses "preempt_count() & 1962 * PREEMPT_MASK" as ->readlock_nesting(). 1963 */ 1964 mask = RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2; 1965 if (!IS_ENABLED(CONFIG_PREEMPT_RCU)) 1966 mask |= RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED; 1967 1968 WARN_ONCE(cur_ops->readlock_nesting && !(curstate & mask) && 1969 cur_ops->readlock_nesting() > 0, ROEC_ARGS); 1970 } 1971 1972 /* 1973 * Do one extension of an RCU read-side critical section using the 1974 * current reader state in readstate (set to zero for initial entry 1975 * to extended critical section), set the new state as specified by 1976 * newstate (set to zero for final exit from extended critical section), 1977 * and random-number-generator state in trsp. If this is neither the 1978 * beginning or end of the critical section and if there was actually a 1979 * change, do a ->read_delay(). 1980 */ 1981 static void rcutorture_one_extend(int *readstate, int newstate, bool insoftirq, 1982 struct torture_random_state *trsp, 1983 struct rt_read_seg *rtrsp) 1984 { 1985 bool first; 1986 unsigned long flags; 1987 int idxnew1 = -1; 1988 int idxnew2 = -1; 1989 int idxold1 = *readstate; 1990 int idxold2 = idxold1; 1991 int statesnew = ~*readstate & newstate; 1992 int statesold = *readstate & ~newstate; 1993 1994 first = idxold1 == 0; 1995 WARN_ON_ONCE(idxold2 < 0); 1996 WARN_ON_ONCE(idxold2 & ~RCUTORTURE_RDR_ALLBITS); 1997 rcutorture_one_extend_check("before change", idxold1, statesnew, statesold, insoftirq); 1998 rtrsp->rt_readstate = newstate; 1999 2000 /* First, put new protection in place to avoid critical-section gap. */ 2001 if (statesnew & RCUTORTURE_RDR_BH) 2002 local_bh_disable(); 2003 if (statesnew & RCUTORTURE_RDR_RBH) 2004 rcu_read_lock_bh(); 2005 if (statesnew & RCUTORTURE_RDR_IRQ) 2006 local_irq_disable(); 2007 if (statesnew & RCUTORTURE_RDR_PREEMPT) 2008 preempt_disable(); 2009 if (statesnew & RCUTORTURE_RDR_SCHED) 2010 rcu_read_lock_sched(); 2011 if (statesnew & RCUTORTURE_RDR_RCU_1) 2012 idxnew1 = (cur_ops->readlock() << RCUTORTURE_RDR_SHIFT_1) & RCUTORTURE_RDR_MASK_1; 2013 if (statesnew & RCUTORTURE_RDR_RCU_2) 2014 idxnew2 = (cur_ops->readlock() << RCUTORTURE_RDR_SHIFT_2) & RCUTORTURE_RDR_MASK_2; 2015 2016 // Complain unless both the old and the new protection is in place. 2017 rcutorture_one_extend_check("during change", 2018 idxold1 | statesnew, statesnew, statesold, insoftirq); 2019 2020 // Sample CPU under both sets of protections to reduce confusion. 2021 if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)) { 2022 int cpu = raw_smp_processor_id(); 2023 rtrsp->rt_cpu = cpu; 2024 if (!first) { 2025 rtrsp[-1].rt_end_cpu = cpu; 2026 if (cur_ops->reader_blocked) 2027 rtrsp[-1].rt_preempted = cur_ops->reader_blocked(); 2028 } 2029 } 2030 // Sample grace-period sequence number, as good a place as any. 2031 if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP) && cur_ops->gather_gp_seqs) { 2032 rtrsp->rt_gp_seq = cur_ops->gather_gp_seqs(); 2033 rtrsp->rt_ts = ktime_get_mono_fast_ns(); 2034 if (!first) 2035 rtrsp[-1].rt_gp_seq_end = rtrsp->rt_gp_seq; 2036 } 2037 2038 /* 2039 * Next, remove old protection, in decreasing order of strength 2040 * to avoid unlock paths that aren't safe in the stronger 2041 * context. Namely: BH can not be enabled with disabled interrupts. 2042 * Additionally PREEMPT_RT requires that BH is enabled in preemptible 2043 * context. 2044 */ 2045 if (statesold & RCUTORTURE_RDR_IRQ) 2046 local_irq_enable(); 2047 if (statesold & RCUTORTURE_RDR_PREEMPT) 2048 preempt_enable(); 2049 if (statesold & RCUTORTURE_RDR_SCHED) 2050 rcu_read_unlock_sched(); 2051 if (statesold & RCUTORTURE_RDR_BH) 2052 local_bh_enable(); 2053 if (statesold & RCUTORTURE_RDR_RBH) 2054 rcu_read_unlock_bh(); 2055 if (statesold & RCUTORTURE_RDR_RCU_2) { 2056 cur_ops->readunlock((idxold2 & RCUTORTURE_RDR_MASK_2) >> RCUTORTURE_RDR_SHIFT_2); 2057 WARN_ON_ONCE(idxnew2 != -1); 2058 idxold2 = 0; 2059 } 2060 if (statesold & RCUTORTURE_RDR_RCU_1) { 2061 bool lockit; 2062 2063 lockit = !cur_ops->no_pi_lock && !statesnew && !(torture_random(trsp) & 0xffff); 2064 if (lockit) 2065 raw_spin_lock_irqsave(¤t->pi_lock, flags); 2066 cur_ops->readunlock((idxold1 & RCUTORTURE_RDR_MASK_1) >> RCUTORTURE_RDR_SHIFT_1); 2067 WARN_ON_ONCE(idxnew1 != -1); 2068 idxold1 = 0; 2069 if (lockit) 2070 raw_spin_unlock_irqrestore(¤t->pi_lock, flags); 2071 } 2072 2073 /* Delay if neither beginning nor end and there was a change. */ 2074 if ((statesnew || statesold) && *readstate && newstate) 2075 cur_ops->read_delay(trsp, rtrsp); 2076 2077 /* Update the reader state. */ 2078 if (idxnew1 == -1) 2079 idxnew1 = idxold1 & RCUTORTURE_RDR_MASK_1; 2080 WARN_ON_ONCE(idxnew1 < 0); 2081 if (idxnew2 == -1) 2082 idxnew2 = idxold2 & RCUTORTURE_RDR_MASK_2; 2083 WARN_ON_ONCE(idxnew2 < 0); 2084 *readstate = idxnew1 | idxnew2 | newstate; 2085 WARN_ON_ONCE(*readstate < 0); 2086 if (WARN_ON_ONCE(*readstate & ~RCUTORTURE_RDR_ALLBITS)) 2087 pr_info("Unexpected readstate value of %#x\n", *readstate); 2088 rcutorture_one_extend_check("after change", *readstate, statesnew, statesold, insoftirq); 2089 } 2090 2091 /* Return the biggest extendables mask given current RCU and boot parameters. */ 2092 static int rcutorture_extend_mask_max(void) 2093 { 2094 int mask; 2095 2096 WARN_ON_ONCE(extendables & ~RCUTORTURE_MAX_EXTEND); 2097 mask = extendables & RCUTORTURE_MAX_EXTEND & cur_ops->extendables; 2098 mask = mask | RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2; 2099 return mask; 2100 } 2101 2102 /* Return a random protection state mask, but with at least one bit set. */ 2103 static int 2104 rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp) 2105 { 2106 int mask = rcutorture_extend_mask_max(); 2107 unsigned long randmask1 = torture_random(trsp); 2108 unsigned long randmask2 = randmask1 >> 3; 2109 unsigned long preempts = RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED; 2110 unsigned long preempts_irq = preempts | RCUTORTURE_RDR_IRQ; 2111 unsigned long bhs = RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH; 2112 2113 WARN_ON_ONCE(mask >> RCUTORTURE_RDR_SHIFT_1); // Can't have reader idx bits. 2114 /* Mostly only one bit (need preemption!), sometimes lots of bits. */ 2115 if (!(randmask1 & 0x7)) 2116 mask = mask & randmask2; 2117 else 2118 mask = mask & (1 << (randmask2 % RCUTORTURE_RDR_NBITS)); 2119 2120 // Can't have nested RCU reader without outer RCU reader. 2121 if (!(mask & RCUTORTURE_RDR_RCU_1) && (mask & RCUTORTURE_RDR_RCU_2)) { 2122 if (oldmask & RCUTORTURE_RDR_RCU_1) 2123 mask &= ~RCUTORTURE_RDR_RCU_2; 2124 else 2125 mask |= RCUTORTURE_RDR_RCU_1; 2126 } 2127 2128 /* 2129 * Can't enable bh w/irq disabled. 2130 */ 2131 if (mask & RCUTORTURE_RDR_IRQ) 2132 mask |= oldmask & bhs; 2133 2134 /* 2135 * Ideally these sequences would be detected in debug builds 2136 * (regardless of RT), but until then don't stop testing 2137 * them on non-RT. 2138 */ 2139 if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 2140 /* Can't modify BH in atomic context */ 2141 if (oldmask & preempts_irq) 2142 mask &= ~bhs; 2143 if ((oldmask | mask) & preempts_irq) 2144 mask |= oldmask & bhs; 2145 } 2146 2147 return mask ?: RCUTORTURE_RDR_RCU_1; 2148 } 2149 2150 /* 2151 * Do a randomly selected number of extensions of an existing RCU read-side 2152 * critical section. 2153 */ 2154 static struct rt_read_seg * 2155 rcutorture_loop_extend(int *readstate, bool insoftirq, struct torture_random_state *trsp, 2156 struct rt_read_seg *rtrsp) 2157 { 2158 int i; 2159 int j; 2160 int mask = rcutorture_extend_mask_max(); 2161 2162 WARN_ON_ONCE(!*readstate); /* -Existing- RCU read-side critsect! */ 2163 if (!((mask - 1) & mask)) 2164 return rtrsp; /* Current RCU reader not extendable. */ 2165 /* Bias towards larger numbers of loops. */ 2166 i = torture_random(trsp); 2167 i = ((i | (i >> 3)) & RCUTORTURE_RDR_MAX_LOOPS) + 1; 2168 for (j = 0; j < i; j++) { 2169 mask = rcutorture_extend_mask(*readstate, trsp); 2170 rcutorture_one_extend(readstate, mask, insoftirq, trsp, &rtrsp[j]); 2171 } 2172 return &rtrsp[j]; 2173 } 2174 2175 struct rcu_torture_one_read_state { 2176 bool checkpolling; 2177 unsigned long cookie; 2178 struct rcu_gp_oldstate cookie_full; 2179 unsigned long started; 2180 struct rcu_torture *p; 2181 int readstate; 2182 struct rt_read_seg rtseg[RCUTORTURE_RDR_MAX_SEGS]; 2183 struct rt_read_seg *rtrsp; 2184 unsigned long long ts; 2185 }; 2186 2187 static void init_rcu_torture_one_read_state(struct rcu_torture_one_read_state *rtorsp, 2188 struct torture_random_state *trsp) 2189 { 2190 memset(rtorsp, 0, sizeof(*rtorsp)); 2191 rtorsp->checkpolling = !(torture_random(trsp) & 0xfff); 2192 rtorsp->rtrsp = &rtorsp->rtseg[0]; 2193 } 2194 2195 /* 2196 * Set up the first segment of a series of overlapping read-side 2197 * critical sections. The caller must have actually initiated the 2198 * outermost read-side critical section. 2199 */ 2200 static bool rcu_torture_one_read_start(struct rcu_torture_one_read_state *rtorsp, 2201 struct torture_random_state *trsp, long myid) 2202 { 2203 if (rtorsp->checkpolling) { 2204 if (cur_ops->get_gp_state && cur_ops->poll_gp_state) 2205 rtorsp->cookie = cur_ops->get_gp_state(); 2206 if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) 2207 cur_ops->get_gp_state_full(&rtorsp->cookie_full); 2208 } 2209 rtorsp->started = cur_ops->get_gp_seq(); 2210 rtorsp->ts = rcu_trace_clock_local(); 2211 rtorsp->p = rcu_dereference_check(rcu_torture_current, 2212 !cur_ops->readlock_held || cur_ops->readlock_held()); 2213 if (rtorsp->p == NULL) { 2214 /* Wait for rcu_torture_writer to get underway */ 2215 rcutorture_one_extend(&rtorsp->readstate, 0, myid < 0, trsp, rtorsp->rtrsp); 2216 return false; 2217 } 2218 if (rtorsp->p->rtort_mbtest == 0) 2219 atomic_inc(&n_rcu_torture_mberror); 2220 rcu_torture_reader_do_mbchk(myid, rtorsp->p, trsp); 2221 return true; 2222 } 2223 2224 /* 2225 * Complete the last segment of a series of overlapping read-side 2226 * critical sections and check for errors. 2227 */ 2228 static void rcu_torture_one_read_end(struct rcu_torture_one_read_state *rtorsp, 2229 struct torture_random_state *trsp, long myid) 2230 { 2231 int i; 2232 unsigned long completed; 2233 int pipe_count; 2234 bool preempted = false; 2235 struct rt_read_seg *rtrsp1; 2236 2237 preempt_disable(); 2238 pipe_count = READ_ONCE(rtorsp->p->rtort_pipe_count); 2239 if (pipe_count > RCU_TORTURE_PIPE_LEN) { 2240 // Should not happen in a correct RCU implementation, 2241 // happens quite often for torture_type=busted. 2242 pipe_count = RCU_TORTURE_PIPE_LEN; 2243 } 2244 completed = cur_ops->get_gp_seq(); 2245 if (pipe_count > 1) { 2246 do_trace_rcu_torture_read(cur_ops->name, &rtorsp->p->rtort_rcu, 2247 rtorsp->ts, rtorsp->started, completed); 2248 rcu_ftrace_dump(DUMP_ALL); 2249 } 2250 __this_cpu_inc(rcu_torture_count[pipe_count]); 2251 completed = rcutorture_seq_diff(completed, rtorsp->started); 2252 if (completed > RCU_TORTURE_PIPE_LEN) { 2253 /* Should not happen, but... */ 2254 completed = RCU_TORTURE_PIPE_LEN; 2255 } 2256 __this_cpu_inc(rcu_torture_batch[completed]); 2257 preempt_enable(); 2258 if (rtorsp->checkpolling) { 2259 if (cur_ops->get_gp_state && cur_ops->poll_gp_state) 2260 WARN_ONCE(cur_ops->poll_gp_state(rtorsp->cookie), 2261 "%s: Cookie check 2 failed %s(%d) %lu->%lu\n", 2262 __func__, 2263 rcu_torture_writer_state_getname(), 2264 rcu_torture_writer_state, 2265 rtorsp->cookie, cur_ops->get_gp_state()); 2266 if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) 2267 WARN_ONCE(cur_ops->poll_gp_state_full(&rtorsp->cookie_full), 2268 "%s: Cookie check 6 failed %s(%d) online %*pbl\n", 2269 __func__, 2270 rcu_torture_writer_state_getname(), 2271 rcu_torture_writer_state, 2272 cpumask_pr_args(cpu_online_mask)); 2273 } 2274 if (cur_ops->reader_blocked) 2275 preempted = cur_ops->reader_blocked(); 2276 rcutorture_one_extend(&rtorsp->readstate, 0, myid < 0, trsp, rtorsp->rtrsp); 2277 WARN_ON_ONCE(rtorsp->readstate); 2278 // This next splat is expected behavior if leakpointer, especially 2279 // for CONFIG_RCU_STRICT_GRACE_PERIOD=y kernels. 2280 WARN_ON_ONCE(leakpointer && READ_ONCE(rtorsp->p->rtort_pipe_count) > 1); 2281 2282 /* If error or close call, record the sequence of reader protections. */ 2283 if ((pipe_count > 1 || completed > 1) && !xchg(&err_segs_recorded, 1)) { 2284 i = 0; 2285 for (rtrsp1 = &rtorsp->rtseg[0]; rtrsp1 < rtorsp->rtrsp; rtrsp1++) 2286 err_segs[i++] = *rtrsp1; 2287 rt_read_nsegs = i; 2288 rt_read_preempted = preempted; 2289 } 2290 } 2291 2292 /* 2293 * Do one read-side critical section, returning false if there was 2294 * no data to read. Can be invoked both from process context and 2295 * from a timer handler. 2296 */ 2297 static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) 2298 { 2299 int newstate; 2300 struct rcu_torture_one_read_state rtors; 2301 2302 WARN_ON_ONCE(!rcu_is_watching()); 2303 init_rcu_torture_one_read_state(&rtors, trsp); 2304 newstate = rcutorture_extend_mask(rtors.readstate, trsp); 2305 rcutorture_one_extend(&rtors.readstate, newstate, myid < 0, trsp, rtors.rtrsp++); 2306 if (!rcu_torture_one_read_start(&rtors, trsp, myid)) { 2307 rcutorture_one_extend(&rtors.readstate, 0, myid < 0, trsp, rtors.rtrsp); 2308 return false; 2309 } 2310 rtors.rtrsp = rcutorture_loop_extend(&rtors.readstate, myid < 0, trsp, rtors.rtrsp); 2311 rcu_torture_one_read_end(&rtors, trsp, myid); 2312 return true; 2313 } 2314 2315 static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_timer_rand); 2316 2317 /* 2318 * RCU torture reader from timer handler. Dereferences rcu_torture_current, 2319 * incrementing the corresponding element of the pipeline array. The 2320 * counter in the element should never be greater than 1, otherwise, the 2321 * RCU implementation is broken. 2322 */ 2323 static void rcu_torture_timer(struct timer_list *unused) 2324 { 2325 atomic_long_inc(&n_rcu_torture_timers); 2326 (void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_timer_rand), -1); 2327 2328 /* Test call_rcu() invocation from interrupt handler. */ 2329 if (cur_ops->call) { 2330 struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_NOWAIT); 2331 2332 if (rhp) 2333 cur_ops->call(rhp, rcu_torture_timer_cb); 2334 } 2335 } 2336 2337 /* 2338 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 2339 * incrementing the corresponding element of the pipeline array. The 2340 * counter in the element should never be greater than 1, otherwise, the 2341 * RCU implementation is broken. 2342 */ 2343 static int 2344 rcu_torture_reader(void *arg) 2345 { 2346 unsigned long lastsleep = jiffies; 2347 long myid = (long)arg; 2348 int mynumonline = myid; 2349 DEFINE_TORTURE_RANDOM(rand); 2350 struct timer_list t; 2351 2352 VERBOSE_TOROUT_STRING("rcu_torture_reader task started"); 2353 set_user_nice(current, MAX_NICE); 2354 if (irqreader && cur_ops->irq_capable) 2355 timer_setup_on_stack(&t, rcu_torture_timer, 0); 2356 tick_dep_set_task(current, TICK_DEP_BIT_RCU); // CPU bound, so need tick. 2357 do { 2358 if (irqreader && cur_ops->irq_capable) { 2359 if (!timer_pending(&t)) 2360 mod_timer(&t, jiffies + 1); 2361 } 2362 if (!rcu_torture_one_read(&rand, myid) && !torture_must_stop()) 2363 schedule_timeout_interruptible(HZ); 2364 if (time_after(jiffies, lastsleep) && !torture_must_stop()) { 2365 torture_hrtimeout_us(500, 1000, &rand); 2366 lastsleep = jiffies + 10; 2367 } 2368 while (torture_num_online_cpus() < mynumonline && !torture_must_stop()) 2369 schedule_timeout_interruptible(HZ / 5); 2370 stutter_wait("rcu_torture_reader"); 2371 } while (!torture_must_stop()); 2372 if (irqreader && cur_ops->irq_capable) { 2373 timer_delete_sync(&t); 2374 timer_destroy_on_stack(&t); 2375 } 2376 tick_dep_clear_task(current, TICK_DEP_BIT_RCU); 2377 torture_kthread_stopping("rcu_torture_reader"); 2378 return 0; 2379 } 2380 2381 /* 2382 * Randomly Toggle CPUs' callback-offload state. This uses hrtimers to 2383 * increase race probabilities and fuzzes the interval between toggling. 2384 */ 2385 static int rcu_nocb_toggle(void *arg) 2386 { 2387 int cpu; 2388 int maxcpu = -1; 2389 int oldnice = task_nice(current); 2390 long r; 2391 DEFINE_TORTURE_RANDOM(rand); 2392 ktime_t toggle_delay; 2393 unsigned long toggle_fuzz; 2394 ktime_t toggle_interval = ms_to_ktime(nocbs_toggle); 2395 2396 VERBOSE_TOROUT_STRING("rcu_nocb_toggle task started"); 2397 while (!rcu_inkernel_boot_has_ended()) 2398 schedule_timeout_interruptible(HZ / 10); 2399 for_each_possible_cpu(cpu) 2400 maxcpu = cpu; 2401 WARN_ON(maxcpu < 0); 2402 if (toggle_interval > ULONG_MAX) 2403 toggle_fuzz = ULONG_MAX >> 3; 2404 else 2405 toggle_fuzz = toggle_interval >> 3; 2406 if (toggle_fuzz <= 0) 2407 toggle_fuzz = NSEC_PER_USEC; 2408 do { 2409 r = torture_random(&rand); 2410 cpu = (r >> 1) % (maxcpu + 1); 2411 if (r & 0x1) { 2412 rcu_nocb_cpu_offload(cpu); 2413 atomic_long_inc(&n_nocb_offload); 2414 } else { 2415 rcu_nocb_cpu_deoffload(cpu); 2416 atomic_long_inc(&n_nocb_deoffload); 2417 } 2418 toggle_delay = torture_random(&rand) % toggle_fuzz + toggle_interval; 2419 set_current_state(TASK_INTERRUPTIBLE); 2420 schedule_hrtimeout(&toggle_delay, HRTIMER_MODE_REL); 2421 if (stutter_wait("rcu_nocb_toggle")) 2422 sched_set_normal(current, oldnice); 2423 } while (!torture_must_stop()); 2424 torture_kthread_stopping("rcu_nocb_toggle"); 2425 return 0; 2426 } 2427 2428 /* 2429 * Print torture statistics. Caller must ensure that there is only 2430 * one call to this function at a given time!!! This is normally 2431 * accomplished by relying on the module system to only have one copy 2432 * of the module loaded, and then by giving the rcu_torture_stats 2433 * kthread full control (or the init/cleanup functions when rcu_torture_stats 2434 * thread is not running). 2435 */ 2436 static void 2437 rcu_torture_stats_print(void) 2438 { 2439 int cpu; 2440 int i; 2441 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 2442 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; 2443 long n_gpwraps = 0; 2444 struct rcu_torture *rtcp; 2445 static unsigned long rtcv_snap = ULONG_MAX; 2446 static bool splatted; 2447 struct task_struct *wtp; 2448 2449 for_each_possible_cpu(cpu) { 2450 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 2451 pipesummary[i] += READ_ONCE(per_cpu(rcu_torture_count, cpu)[i]); 2452 batchsummary[i] += READ_ONCE(per_cpu(rcu_torture_batch, cpu)[i]); 2453 } 2454 if (cur_ops->get_gpwrap_count) 2455 n_gpwraps += cur_ops->get_gpwrap_count(cpu); 2456 } 2457 for (i = RCU_TORTURE_PIPE_LEN; i >= 0; i--) { 2458 if (pipesummary[i] != 0) 2459 break; 2460 } 2461 2462 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 2463 rtcp = rcu_access_pointer(rcu_torture_current); 2464 pr_cont("rtc: %p %s: %lu tfle: %d rta: %d rtaf: %d rtf: %d ", 2465 rtcp, 2466 rtcp && !rcu_stall_is_suppressed_at_boot() ? "ver" : "VER", 2467 rcu_torture_current_version, 2468 list_empty(&rcu_torture_freelist), 2469 atomic_read(&n_rcu_torture_alloc), 2470 atomic_read(&n_rcu_torture_alloc_fail), 2471 atomic_read(&n_rcu_torture_free)); 2472 pr_cont("rtmbe: %d rtmbkf: %d/%d rtbe: %ld rtbke: %ld ", 2473 atomic_read(&n_rcu_torture_mberror), 2474 atomic_read(&n_rcu_torture_mbchk_fail), atomic_read(&n_rcu_torture_mbchk_tries), 2475 n_rcu_torture_barrier_error, 2476 n_rcu_torture_boost_ktrerror); 2477 pr_cont("rtbf: %ld rtb: %ld nt: %ld ", 2478 n_rcu_torture_boost_failure, 2479 n_rcu_torture_boosts, 2480 atomic_long_read(&n_rcu_torture_timers)); 2481 torture_onoff_stats(); 2482 pr_cont("barrier: %ld/%ld:%ld ", 2483 data_race(n_barrier_successes), 2484 data_race(n_barrier_attempts), 2485 data_race(n_rcu_torture_barrier_error)); 2486 pr_cont("read-exits: %ld ", data_race(n_read_exits)); // Statistic. 2487 pr_cont("nocb-toggles: %ld:%ld ", 2488 atomic_long_read(&n_nocb_offload), atomic_long_read(&n_nocb_deoffload)); 2489 pr_cont("gpwraps: %ld\n", n_gpwraps); 2490 2491 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 2492 if (atomic_read(&n_rcu_torture_mberror) || 2493 atomic_read(&n_rcu_torture_mbchk_fail) || 2494 n_rcu_torture_barrier_error || n_rcu_torture_boost_ktrerror || 2495 n_rcu_torture_boost_failure || i > 1) { 2496 pr_cont("%s", "!!! "); 2497 atomic_inc(&n_rcu_torture_error); 2498 WARN_ON_ONCE(atomic_read(&n_rcu_torture_mberror)); 2499 WARN_ON_ONCE(atomic_read(&n_rcu_torture_mbchk_fail)); 2500 WARN_ON_ONCE(n_rcu_torture_barrier_error); // rcu_barrier() 2501 WARN_ON_ONCE(n_rcu_torture_boost_ktrerror); // no boost kthread 2502 WARN_ON_ONCE(n_rcu_torture_boost_failure); // boost failed (TIMER_SOFTIRQ RT prio?) 2503 WARN_ON_ONCE(i > 1); // Too-short grace period 2504 } 2505 pr_cont("Reader Pipe: "); 2506 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 2507 pr_cont(" %ld", pipesummary[i]); 2508 pr_cont("\n"); 2509 2510 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 2511 pr_cont("Reader Batch: "); 2512 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 2513 pr_cont(" %ld", batchsummary[i]); 2514 pr_cont("\n"); 2515 2516 pr_alert("%s%s ", torture_type, TORTURE_FLAG); 2517 pr_cont("Free-Block Circulation: "); 2518 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 2519 pr_cont(" %d", atomic_read(&rcu_torture_wcount[i])); 2520 } 2521 pr_cont("\n"); 2522 2523 if (cur_ops->stats) 2524 cur_ops->stats(); 2525 if (rtcv_snap == rcu_torture_current_version && 2526 rcu_access_pointer(rcu_torture_current) && 2527 !rcu_stall_is_suppressed()) { 2528 int __maybe_unused flags = 0; 2529 unsigned long __maybe_unused gp_seq = 0; 2530 2531 if (cur_ops->get_gp_data) 2532 cur_ops->get_gp_data(&flags, &gp_seq); 2533 wtp = READ_ONCE(writer_task); 2534 pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %#x cpu %d\n", 2535 rcu_torture_writer_state_getname(), 2536 rcu_torture_writer_state, gp_seq, flags, 2537 wtp == NULL ? ~0U : wtp->__state, 2538 wtp == NULL ? -1 : (int)task_cpu(wtp)); 2539 if (!splatted && wtp) { 2540 sched_show_task(wtp); 2541 splatted = true; 2542 } 2543 if (cur_ops->gp_kthread_dbg) 2544 cur_ops->gp_kthread_dbg(); 2545 rcu_ftrace_dump(DUMP_ALL); 2546 } 2547 rtcv_snap = rcu_torture_current_version; 2548 } 2549 2550 /* 2551 * Periodically prints torture statistics, if periodic statistics printing 2552 * was specified via the stat_interval module parameter. 2553 */ 2554 static int 2555 rcu_torture_stats(void *arg) 2556 { 2557 VERBOSE_TOROUT_STRING("rcu_torture_stats task started"); 2558 do { 2559 schedule_timeout_interruptible(stat_interval * HZ); 2560 rcu_torture_stats_print(); 2561 torture_shutdown_absorb("rcu_torture_stats"); 2562 } while (!torture_must_stop()); 2563 torture_kthread_stopping("rcu_torture_stats"); 2564 return 0; 2565 } 2566 2567 /* Test mem_dump_obj() and friends. */ 2568 static void rcu_torture_mem_dump_obj(void) 2569 { 2570 struct rcu_head *rhp; 2571 struct kmem_cache *kcp; 2572 static int z; 2573 2574 kcp = kmem_cache_create("rcuscale", 136, 8, SLAB_STORE_USER, NULL); 2575 if (WARN_ON_ONCE(!kcp)) 2576 return; 2577 rhp = kmem_cache_alloc(kcp, GFP_KERNEL); 2578 if (WARN_ON_ONCE(!rhp)) { 2579 kmem_cache_destroy(kcp); 2580 return; 2581 } 2582 pr_alert("mem_dump_obj() slab test: rcu_torture_stats = %px, &rhp = %px, rhp = %px, &z = %px\n", stats_task, &rhp, rhp, &z); 2583 pr_alert("mem_dump_obj(ZERO_SIZE_PTR):"); 2584 mem_dump_obj(ZERO_SIZE_PTR); 2585 pr_alert("mem_dump_obj(NULL):"); 2586 mem_dump_obj(NULL); 2587 pr_alert("mem_dump_obj(%px):", &rhp); 2588 mem_dump_obj(&rhp); 2589 pr_alert("mem_dump_obj(%px):", rhp); 2590 mem_dump_obj(rhp); 2591 pr_alert("mem_dump_obj(%px):", &rhp->func); 2592 mem_dump_obj(&rhp->func); 2593 pr_alert("mem_dump_obj(%px):", &z); 2594 mem_dump_obj(&z); 2595 kmem_cache_free(kcp, rhp); 2596 kmem_cache_destroy(kcp); 2597 rhp = kmalloc(sizeof(*rhp), GFP_KERNEL); 2598 if (WARN_ON_ONCE(!rhp)) 2599 return; 2600 pr_alert("mem_dump_obj() kmalloc test: rcu_torture_stats = %px, &rhp = %px, rhp = %px\n", stats_task, &rhp, rhp); 2601 pr_alert("mem_dump_obj(kmalloc %px):", rhp); 2602 mem_dump_obj(rhp); 2603 pr_alert("mem_dump_obj(kmalloc %px):", &rhp->func); 2604 mem_dump_obj(&rhp->func); 2605 kfree(rhp); 2606 rhp = vmalloc(4096); 2607 if (WARN_ON_ONCE(!rhp)) 2608 return; 2609 pr_alert("mem_dump_obj() vmalloc test: rcu_torture_stats = %px, &rhp = %px, rhp = %px\n", stats_task, &rhp, rhp); 2610 pr_alert("mem_dump_obj(vmalloc %px):", rhp); 2611 mem_dump_obj(rhp); 2612 pr_alert("mem_dump_obj(vmalloc %px):", &rhp->func); 2613 mem_dump_obj(&rhp->func); 2614 vfree(rhp); 2615 } 2616 2617 static void 2618 rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) 2619 { 2620 pr_alert("%s" TORTURE_FLAG 2621 "--- %s: nreaders=%d nfakewriters=%d " 2622 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 2623 "shuffle_interval=%d stutter=%d irqreader=%d " 2624 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " 2625 "test_boost=%d/%d test_boost_interval=%d " 2626 "test_boost_duration=%d test_boost_holdoff=%d shutdown_secs=%d " 2627 "stall_cpu=%d stall_cpu_holdoff=%d stall_cpu_irqsoff=%d " 2628 "stall_cpu_block=%d stall_cpu_repeat=%d " 2629 "n_barrier_cbs=%d " 2630 "onoff_interval=%d onoff_holdoff=%d " 2631 "read_exit_delay=%d read_exit_burst=%d " 2632 "reader_flavor=%x " 2633 "nocbs_nthreads=%d nocbs_toggle=%d " 2634 "test_nmis=%d " 2635 "preempt_duration=%d preempt_interval=%d\n", 2636 torture_type, tag, nrealreaders, nrealfakewriters, 2637 stat_interval, verbose, test_no_idle_hz, shuffle_interval, 2638 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, 2639 test_boost, cur_ops->can_boost, 2640 test_boost_interval, test_boost_duration, test_boost_holdoff, shutdown_secs, 2641 stall_cpu, stall_cpu_holdoff, stall_cpu_irqsoff, 2642 stall_cpu_block, stall_cpu_repeat, 2643 n_barrier_cbs, 2644 onoff_interval, onoff_holdoff, 2645 read_exit_delay, read_exit_burst, 2646 reader_flavor, 2647 nocbs_nthreads, nocbs_toggle, 2648 test_nmis, 2649 preempt_duration, preempt_interval); 2650 } 2651 2652 static int rcutorture_booster_cleanup(unsigned int cpu) 2653 { 2654 struct task_struct *t; 2655 2656 if (boost_tasks[cpu] == NULL) 2657 return 0; 2658 mutex_lock(&boost_mutex); 2659 t = boost_tasks[cpu]; 2660 boost_tasks[cpu] = NULL; 2661 rcu_torture_enable_rt_throttle(); 2662 mutex_unlock(&boost_mutex); 2663 2664 /* This must be outside of the mutex, otherwise deadlock! */ 2665 torture_stop_kthread(rcu_torture_boost, t); 2666 return 0; 2667 } 2668 2669 static int rcutorture_booster_init(unsigned int cpu) 2670 { 2671 int retval; 2672 2673 if (boost_tasks[cpu] != NULL) 2674 return 0; /* Already created, nothing more to do. */ 2675 2676 // Testing RCU priority boosting requires rcutorture do 2677 // some serious abuse. Counter this by running ksoftirqd 2678 // at higher priority. 2679 if (IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)) { 2680 struct sched_param sp; 2681 struct task_struct *t; 2682 2683 t = per_cpu(ksoftirqd, cpu); 2684 WARN_ON_ONCE(!t); 2685 sp.sched_priority = 2; 2686 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); 2687 #ifdef CONFIG_IRQ_FORCED_THREADING 2688 if (force_irqthreads()) { 2689 t = per_cpu(ktimerd, cpu); 2690 WARN_ON_ONCE(!t); 2691 sp.sched_priority = 2; 2692 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); 2693 } 2694 #endif 2695 } 2696 2697 /* Don't allow time recalculation while creating a new task. */ 2698 mutex_lock(&boost_mutex); 2699 rcu_torture_disable_rt_throttle(); 2700 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task"); 2701 boost_tasks[cpu] = kthread_run_on_cpu(rcu_torture_boost, NULL, 2702 cpu, "rcu_torture_boost_%u"); 2703 if (IS_ERR(boost_tasks[cpu])) { 2704 retval = PTR_ERR(boost_tasks[cpu]); 2705 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed"); 2706 n_rcu_torture_boost_ktrerror++; 2707 boost_tasks[cpu] = NULL; 2708 mutex_unlock(&boost_mutex); 2709 return retval; 2710 } 2711 mutex_unlock(&boost_mutex); 2712 return 0; 2713 } 2714 2715 static int rcu_torture_stall_nf(struct notifier_block *nb, unsigned long v, void *ptr) 2716 { 2717 pr_info("%s: v=%lu, duration=%lu.\n", __func__, v, (unsigned long)ptr); 2718 return NOTIFY_OK; 2719 } 2720 2721 static struct notifier_block rcu_torture_stall_block = { 2722 .notifier_call = rcu_torture_stall_nf, 2723 }; 2724 2725 /* 2726 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then 2727 * induces a CPU stall for the time specified by stall_cpu. If a new 2728 * stall test is added, stallsdone in rcu_torture_writer() must be adjusted. 2729 */ 2730 static void rcu_torture_stall_one(int rep, int irqsoff) 2731 { 2732 int idx; 2733 unsigned long stop_at; 2734 2735 if (stall_cpu_holdoff > 0) { 2736 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff"); 2737 schedule_timeout_interruptible(stall_cpu_holdoff * HZ); 2738 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff"); 2739 } 2740 if (!kthread_should_stop() && stall_gp_kthread > 0) { 2741 VERBOSE_TOROUT_STRING("rcu_torture_stall begin GP stall"); 2742 rcu_gp_set_torture_wait(stall_gp_kthread * HZ); 2743 for (idx = 0; idx < stall_gp_kthread + 2; idx++) { 2744 if (kthread_should_stop()) 2745 break; 2746 schedule_timeout_uninterruptible(HZ); 2747 } 2748 } 2749 if (!kthread_should_stop() && stall_cpu > 0) { 2750 VERBOSE_TOROUT_STRING("rcu_torture_stall begin CPU stall"); 2751 stop_at = ktime_get_seconds() + stall_cpu; 2752 /* RCU CPU stall is expected behavior in following code. */ 2753 idx = cur_ops->readlock(); 2754 if (irqsoff) 2755 local_irq_disable(); 2756 else if (!stall_cpu_block) 2757 preempt_disable(); 2758 pr_alert("%s start stall episode %d on CPU %d.\n", 2759 __func__, rep + 1, raw_smp_processor_id()); 2760 while (ULONG_CMP_LT((unsigned long)ktime_get_seconds(), stop_at) && 2761 !kthread_should_stop()) 2762 if (stall_cpu_block) { 2763 #ifdef CONFIG_PREEMPTION 2764 preempt_schedule(); 2765 #else 2766 schedule_timeout_uninterruptible(HZ); 2767 #endif 2768 } else if (stall_no_softlockup) { 2769 touch_softlockup_watchdog(); 2770 } 2771 if (irqsoff) 2772 local_irq_enable(); 2773 else if (!stall_cpu_block) 2774 preempt_enable(); 2775 cur_ops->readunlock(idx); 2776 } 2777 } 2778 2779 /* 2780 * CPU-stall kthread. Invokes rcu_torture_stall_one() once, and then as many 2781 * additional times as specified by the stall_cpu_repeat module parameter. 2782 * Note that stall_cpu_irqsoff is ignored on the second and subsequent 2783 * stall. 2784 */ 2785 static int rcu_torture_stall(void *args) 2786 { 2787 int i; 2788 int repeat = stall_cpu_repeat; 2789 int ret; 2790 2791 VERBOSE_TOROUT_STRING("rcu_torture_stall task started"); 2792 if (repeat < 0) { 2793 repeat = 0; 2794 WARN_ON_ONCE(IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)); 2795 } 2796 if (rcu_cpu_stall_notifiers) { 2797 ret = rcu_stall_chain_notifier_register(&rcu_torture_stall_block); 2798 if (ret) 2799 pr_info("%s: rcu_stall_chain_notifier_register() returned %d, %sexpected.\n", 2800 __func__, ret, !IS_ENABLED(CONFIG_RCU_STALL_COMMON) ? "un" : ""); 2801 } 2802 for (i = 0; i <= repeat; i++) { 2803 if (kthread_should_stop()) 2804 break; 2805 rcu_torture_stall_one(i, i == 0 ? stall_cpu_irqsoff : 0); 2806 } 2807 pr_alert("%s end.\n", __func__); 2808 if (rcu_cpu_stall_notifiers && !ret) { 2809 ret = rcu_stall_chain_notifier_unregister(&rcu_torture_stall_block); 2810 if (ret) 2811 pr_info("%s: rcu_stall_chain_notifier_unregister() returned %d.\n", __func__, ret); 2812 } 2813 torture_shutdown_absorb("rcu_torture_stall"); 2814 while (!kthread_should_stop()) 2815 schedule_timeout_interruptible(10 * HZ); 2816 return 0; 2817 } 2818 2819 /* Spawn CPU-stall kthread, if stall_cpu specified. */ 2820 static int __init rcu_torture_stall_init(void) 2821 { 2822 if (stall_cpu <= 0 && stall_gp_kthread <= 0) 2823 return 0; 2824 return torture_create_kthread(rcu_torture_stall, NULL, stall_task); 2825 } 2826 2827 /* State structure for forward-progress self-propagating RCU callback. */ 2828 struct fwd_cb_state { 2829 struct rcu_head rh; 2830 int stop; 2831 }; 2832 2833 /* 2834 * Forward-progress self-propagating RCU callback function. Because 2835 * callbacks run from softirq, this function is an implicit RCU read-side 2836 * critical section. 2837 */ 2838 static void rcu_torture_fwd_prog_cb(struct rcu_head *rhp) 2839 { 2840 struct fwd_cb_state *fcsp = container_of(rhp, struct fwd_cb_state, rh); 2841 2842 if (READ_ONCE(fcsp->stop)) { 2843 WRITE_ONCE(fcsp->stop, 2); 2844 return; 2845 } 2846 cur_ops->call(&fcsp->rh, rcu_torture_fwd_prog_cb); 2847 } 2848 2849 /* State for continuous-flood RCU callbacks. */ 2850 struct rcu_fwd_cb { 2851 struct rcu_head rh; 2852 struct rcu_fwd_cb *rfc_next; 2853 struct rcu_fwd *rfc_rfp; 2854 int rfc_gps; 2855 }; 2856 2857 #define MAX_FWD_CB_JIFFIES (8 * HZ) /* Maximum CB test duration. */ 2858 #define MIN_FWD_CB_LAUNDERS 3 /* This many CB invocations to count. */ 2859 #define MIN_FWD_CBS_LAUNDERED 100 /* Number of counted CBs. */ 2860 #define FWD_CBS_HIST_DIV 10 /* Histogram buckets/second. */ 2861 #define N_LAUNDERS_HIST (2 * MAX_FWD_CB_JIFFIES / (HZ / FWD_CBS_HIST_DIV)) 2862 2863 struct rcu_launder_hist { 2864 long n_launders; 2865 unsigned long launder_gp_seq; 2866 }; 2867 2868 struct rcu_fwd { 2869 spinlock_t rcu_fwd_lock; 2870 struct rcu_fwd_cb *rcu_fwd_cb_head; 2871 struct rcu_fwd_cb **rcu_fwd_cb_tail; 2872 long n_launders_cb; 2873 unsigned long rcu_fwd_startat; 2874 struct rcu_launder_hist n_launders_hist[N_LAUNDERS_HIST]; 2875 unsigned long rcu_launder_gp_seq_start; 2876 int rcu_fwd_id; 2877 }; 2878 2879 static DEFINE_MUTEX(rcu_fwd_mutex); 2880 static struct rcu_fwd *rcu_fwds; 2881 static unsigned long rcu_fwd_seq; 2882 static atomic_long_t rcu_fwd_max_cbs; 2883 static bool rcu_fwd_emergency_stop; 2884 2885 static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp) 2886 { 2887 unsigned long gps; 2888 unsigned long gps_old; 2889 int i; 2890 int j; 2891 2892 for (i = ARRAY_SIZE(rfp->n_launders_hist) - 1; i > 0; i--) 2893 if (rfp->n_launders_hist[i].n_launders > 0) 2894 break; 2895 pr_alert("%s: Callback-invocation histogram %d (duration %lu jiffies):", 2896 __func__, rfp->rcu_fwd_id, jiffies - rfp->rcu_fwd_startat); 2897 gps_old = rfp->rcu_launder_gp_seq_start; 2898 for (j = 0; j <= i; j++) { 2899 gps = rfp->n_launders_hist[j].launder_gp_seq; 2900 pr_cont(" %ds/%d: %ld:%ld", 2901 j + 1, FWD_CBS_HIST_DIV, 2902 rfp->n_launders_hist[j].n_launders, 2903 rcutorture_seq_diff(gps, gps_old)); 2904 gps_old = gps; 2905 } 2906 pr_cont("\n"); 2907 } 2908 2909 /* Callback function for continuous-flood RCU callbacks. */ 2910 static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp) 2911 { 2912 unsigned long flags; 2913 int i; 2914 struct rcu_fwd_cb *rfcp = container_of(rhp, struct rcu_fwd_cb, rh); 2915 struct rcu_fwd_cb **rfcpp; 2916 struct rcu_fwd *rfp = rfcp->rfc_rfp; 2917 2918 rfcp->rfc_next = NULL; 2919 rfcp->rfc_gps++; 2920 spin_lock_irqsave(&rfp->rcu_fwd_lock, flags); 2921 rfcpp = rfp->rcu_fwd_cb_tail; 2922 rfp->rcu_fwd_cb_tail = &rfcp->rfc_next; 2923 smp_store_release(rfcpp, rfcp); 2924 WRITE_ONCE(rfp->n_launders_cb, rfp->n_launders_cb + 1); 2925 i = ((jiffies - rfp->rcu_fwd_startat) / (HZ / FWD_CBS_HIST_DIV)); 2926 if (i >= ARRAY_SIZE(rfp->n_launders_hist)) 2927 i = ARRAY_SIZE(rfp->n_launders_hist) - 1; 2928 rfp->n_launders_hist[i].n_launders++; 2929 rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq(); 2930 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); 2931 } 2932 2933 // Give the scheduler a chance, even on nohz_full CPUs. 2934 static void rcu_torture_fwd_prog_cond_resched(unsigned long iter) 2935 { 2936 if (IS_ENABLED(CONFIG_PREEMPTION) && IS_ENABLED(CONFIG_NO_HZ_FULL)) { 2937 // Real call_rcu() floods hit userspace, so emulate that. 2938 if (need_resched() || (iter & 0xfff)) 2939 schedule(); 2940 return; 2941 } 2942 // No userspace emulation: CB invocation throttles call_rcu() 2943 cond_resched(); 2944 } 2945 2946 /* 2947 * Free all callbacks on the rcu_fwd_cb_head list, either because the 2948 * test is over or because we hit an OOM event. 2949 */ 2950 static unsigned long rcu_torture_fwd_prog_cbfree(struct rcu_fwd *rfp) 2951 { 2952 unsigned long flags; 2953 unsigned long freed = 0; 2954 struct rcu_fwd_cb *rfcp; 2955 2956 for (;;) { 2957 spin_lock_irqsave(&rfp->rcu_fwd_lock, flags); 2958 rfcp = rfp->rcu_fwd_cb_head; 2959 if (!rfcp) { 2960 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); 2961 break; 2962 } 2963 rfp->rcu_fwd_cb_head = rfcp->rfc_next; 2964 if (!rfp->rcu_fwd_cb_head) 2965 rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head; 2966 spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); 2967 kfree(rfcp); 2968 freed++; 2969 rcu_torture_fwd_prog_cond_resched(freed); 2970 if (tick_nohz_full_enabled()) { 2971 local_irq_save(flags); 2972 rcu_momentary_eqs(); 2973 local_irq_restore(flags); 2974 } 2975 } 2976 return freed; 2977 } 2978 2979 /* Carry out need_resched()/cond_resched() forward-progress testing. */ 2980 static void rcu_torture_fwd_prog_nr(struct rcu_fwd *rfp, 2981 int *tested, int *tested_tries) 2982 { 2983 unsigned long cver; 2984 unsigned long dur; 2985 struct fwd_cb_state fcs; 2986 unsigned long gps; 2987 int idx; 2988 int sd; 2989 int sd4; 2990 bool selfpropcb = false; 2991 unsigned long stopat; 2992 static DEFINE_TORTURE_RANDOM(trs); 2993 2994 pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); 2995 if (!cur_ops->sync) 2996 return; // Cannot do need_resched() forward progress testing without ->sync. 2997 if (cur_ops->call && cur_ops->cb_barrier) { 2998 init_rcu_head_on_stack(&fcs.rh); 2999 selfpropcb = true; 3000 } 3001 3002 /* Tight loop containing cond_resched(). */ 3003 atomic_inc(&rcu_fwd_cb_nodelay); 3004 cur_ops->sync(); /* Later readers see above write. */ 3005 if (selfpropcb) { 3006 WRITE_ONCE(fcs.stop, 0); 3007 cur_ops->call(&fcs.rh, rcu_torture_fwd_prog_cb); 3008 } 3009 cver = READ_ONCE(rcu_torture_current_version); 3010 gps = cur_ops->get_gp_seq(); 3011 sd = cur_ops->stall_dur() + 1; 3012 sd4 = (sd + fwd_progress_div - 1) / fwd_progress_div; 3013 dur = sd4 + torture_random(&trs) % (sd - sd4); 3014 WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); 3015 stopat = rfp->rcu_fwd_startat + dur; 3016 while (time_before(jiffies, stopat) && 3017 !shutdown_time_arrived() && 3018 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { 3019 idx = cur_ops->readlock(); 3020 udelay(10); 3021 cur_ops->readunlock(idx); 3022 if (!fwd_progress_need_resched || need_resched()) 3023 cond_resched(); 3024 } 3025 (*tested_tries)++; 3026 if (!time_before(jiffies, stopat) && 3027 !shutdown_time_arrived() && 3028 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { 3029 (*tested)++; 3030 cver = READ_ONCE(rcu_torture_current_version) - cver; 3031 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps); 3032 WARN_ON(!cver && gps < 2); 3033 pr_alert("%s: %d Duration %ld cver %ld gps %ld\n", __func__, 3034 rfp->rcu_fwd_id, dur, cver, gps); 3035 } 3036 if (selfpropcb) { 3037 WRITE_ONCE(fcs.stop, 1); 3038 cur_ops->sync(); /* Wait for running CB to complete. */ 3039 pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id); 3040 cur_ops->cb_barrier(); /* Wait for queued callbacks. */ 3041 } 3042 3043 if (selfpropcb) { 3044 WARN_ON(READ_ONCE(fcs.stop) != 2); 3045 destroy_rcu_head_on_stack(&fcs.rh); 3046 } 3047 schedule_timeout_uninterruptible(HZ / 10); /* Let kthreads recover. */ 3048 atomic_dec(&rcu_fwd_cb_nodelay); 3049 } 3050 3051 /* Carry out call_rcu() forward-progress testing. */ 3052 static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp) 3053 { 3054 unsigned long cver; 3055 unsigned long flags; 3056 unsigned long gps; 3057 int i; 3058 long n_launders; 3059 long n_launders_cb_snap; 3060 long n_launders_sa; 3061 long n_max_cbs; 3062 long n_max_gps; 3063 struct rcu_fwd_cb *rfcp; 3064 struct rcu_fwd_cb *rfcpn; 3065 unsigned long stopat; 3066 unsigned long stoppedat; 3067 3068 pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); 3069 if (READ_ONCE(rcu_fwd_emergency_stop)) 3070 return; /* Get out of the way quickly, no GP wait! */ 3071 if (!cur_ops->call) 3072 return; /* Can't do call_rcu() fwd prog without ->call. */ 3073 3074 /* Loop continuously posting RCU callbacks. */ 3075 atomic_inc(&rcu_fwd_cb_nodelay); 3076 cur_ops->sync(); /* Later readers see above write. */ 3077 WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); 3078 stopat = rfp->rcu_fwd_startat + MAX_FWD_CB_JIFFIES; 3079 n_launders = 0; 3080 rfp->n_launders_cb = 0; // Hoist initialization for multi-kthread 3081 n_launders_sa = 0; 3082 n_max_cbs = 0; 3083 n_max_gps = 0; 3084 for (i = 0; i < ARRAY_SIZE(rfp->n_launders_hist); i++) 3085 rfp->n_launders_hist[i].n_launders = 0; 3086 cver = READ_ONCE(rcu_torture_current_version); 3087 gps = cur_ops->get_gp_seq(); 3088 rfp->rcu_launder_gp_seq_start = gps; 3089 tick_dep_set_task(current, TICK_DEP_BIT_RCU); // CPU bound, so need tick. 3090 while (time_before(jiffies, stopat) && 3091 !shutdown_time_arrived() && 3092 !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { 3093 rfcp = READ_ONCE(rfp->rcu_fwd_cb_head); 3094 rfcpn = NULL; 3095 if (rfcp) 3096 rfcpn = READ_ONCE(rfcp->rfc_next); 3097 if (rfcpn) { 3098 if (rfcp->rfc_gps >= MIN_FWD_CB_LAUNDERS && 3099 ++n_max_gps >= MIN_FWD_CBS_LAUNDERED) 3100 break; 3101 rfp->rcu_fwd_cb_head = rfcpn; 3102 n_launders++; 3103 n_launders_sa++; 3104 } else if (!cur_ops->cbflood_max || cur_ops->cbflood_max > n_max_cbs) { 3105 rfcp = kmalloc(sizeof(*rfcp), GFP_KERNEL); 3106 if (WARN_ON_ONCE(!rfcp)) { 3107 schedule_timeout_interruptible(1); 3108 continue; 3109 } 3110 n_max_cbs++; 3111 n_launders_sa = 0; 3112 rfcp->rfc_gps = 0; 3113 rfcp->rfc_rfp = rfp; 3114 } else { 3115 rfcp = NULL; 3116 } 3117 if (rfcp) 3118 cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr); 3119 rcu_torture_fwd_prog_cond_resched(n_launders + n_max_cbs); 3120 if (tick_nohz_full_enabled()) { 3121 local_irq_save(flags); 3122 rcu_momentary_eqs(); 3123 local_irq_restore(flags); 3124 } 3125 } 3126 stoppedat = jiffies; 3127 n_launders_cb_snap = READ_ONCE(rfp->n_launders_cb); 3128 cver = READ_ONCE(rcu_torture_current_version) - cver; 3129 gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps); 3130 pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id); 3131 cur_ops->cb_barrier(); /* Wait for callbacks to be invoked. */ 3132 (void)rcu_torture_fwd_prog_cbfree(rfp); 3133 3134 if (!torture_must_stop() && !READ_ONCE(rcu_fwd_emergency_stop) && 3135 !shutdown_time_arrived()) { 3136 if (WARN_ON(n_max_gps < MIN_FWD_CBS_LAUNDERED) && cur_ops->gp_kthread_dbg) 3137 cur_ops->gp_kthread_dbg(); 3138 pr_alert("%s Duration %lu barrier: %lu pending %ld n_launders: %ld n_launders_sa: %ld n_max_gps: %ld n_max_cbs: %ld cver %ld gps %ld #online %u\n", 3139 __func__, 3140 stoppedat - rfp->rcu_fwd_startat, jiffies - stoppedat, 3141 n_launders + n_max_cbs - n_launders_cb_snap, 3142 n_launders, n_launders_sa, 3143 n_max_gps, n_max_cbs, cver, gps, num_online_cpus()); 3144 atomic_long_add(n_max_cbs, &rcu_fwd_max_cbs); 3145 mutex_lock(&rcu_fwd_mutex); // Serialize histograms. 3146 rcu_torture_fwd_cb_hist(rfp); 3147 mutex_unlock(&rcu_fwd_mutex); 3148 } 3149 schedule_timeout_uninterruptible(HZ); /* Let CBs drain. */ 3150 tick_dep_clear_task(current, TICK_DEP_BIT_RCU); 3151 atomic_dec(&rcu_fwd_cb_nodelay); 3152 } 3153 3154 3155 /* 3156 * OOM notifier, but this only prints diagnostic information for the 3157 * current forward-progress test. 3158 */ 3159 static int rcutorture_oom_notify(struct notifier_block *self, 3160 unsigned long notused, void *nfreed) 3161 { 3162 int i; 3163 long ncbs; 3164 struct rcu_fwd *rfp; 3165 3166 mutex_lock(&rcu_fwd_mutex); 3167 rfp = rcu_fwds; 3168 if (!rfp) { 3169 mutex_unlock(&rcu_fwd_mutex); 3170 return NOTIFY_OK; 3171 } 3172 WARN(1, "%s invoked upon OOM during forward-progress testing.\n", 3173 __func__); 3174 for (i = 0; i < fwd_progress; i++) { 3175 rcu_torture_fwd_cb_hist(&rfp[i]); 3176 rcu_fwd_progress_check(1 + (jiffies - READ_ONCE(rfp[i].rcu_fwd_startat)) / 2); 3177 } 3178 WRITE_ONCE(rcu_fwd_emergency_stop, true); 3179 smp_mb(); /* Emergency stop before free and wait to avoid hangs. */ 3180 ncbs = 0; 3181 for (i = 0; i < fwd_progress; i++) 3182 ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); 3183 pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); 3184 cur_ops->cb_barrier(); 3185 ncbs = 0; 3186 for (i = 0; i < fwd_progress; i++) 3187 ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); 3188 pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); 3189 cur_ops->cb_barrier(); 3190 ncbs = 0; 3191 for (i = 0; i < fwd_progress; i++) 3192 ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); 3193 pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); 3194 smp_mb(); /* Frees before return to avoid redoing OOM. */ 3195 (*(unsigned long *)nfreed)++; /* Forward progress CBs freed! */ 3196 pr_info("%s returning after OOM processing.\n", __func__); 3197 mutex_unlock(&rcu_fwd_mutex); 3198 return NOTIFY_OK; 3199 } 3200 3201 static struct notifier_block rcutorture_oom_nb = { 3202 .notifier_call = rcutorture_oom_notify 3203 }; 3204 3205 /* Carry out grace-period forward-progress testing. */ 3206 static int rcu_torture_fwd_prog(void *args) 3207 { 3208 bool firsttime = true; 3209 long max_cbs; 3210 int oldnice = task_nice(current); 3211 unsigned long oldseq = READ_ONCE(rcu_fwd_seq); 3212 struct rcu_fwd *rfp = args; 3213 int tested = 0; 3214 int tested_tries = 0; 3215 3216 VERBOSE_TOROUT_STRING("rcu_torture_fwd_progress task started"); 3217 rcu_bind_current_to_nocb(); 3218 if (!IS_ENABLED(CONFIG_SMP) || !IS_ENABLED(CONFIG_RCU_BOOST)) 3219 set_user_nice(current, MAX_NICE); 3220 do { 3221 if (!rfp->rcu_fwd_id) { 3222 schedule_timeout_interruptible(fwd_progress_holdoff * HZ); 3223 WRITE_ONCE(rcu_fwd_emergency_stop, false); 3224 if (!firsttime) { 3225 max_cbs = atomic_long_xchg(&rcu_fwd_max_cbs, 0); 3226 pr_alert("%s n_max_cbs: %ld\n", __func__, max_cbs); 3227 } 3228 firsttime = false; 3229 WRITE_ONCE(rcu_fwd_seq, rcu_fwd_seq + 1); 3230 } else { 3231 while (READ_ONCE(rcu_fwd_seq) == oldseq && !torture_must_stop()) 3232 schedule_timeout_interruptible(HZ / 20); 3233 oldseq = READ_ONCE(rcu_fwd_seq); 3234 } 3235 pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); 3236 if (rcu_inkernel_boot_has_ended() && torture_num_online_cpus() > rfp->rcu_fwd_id) 3237 rcu_torture_fwd_prog_cr(rfp); 3238 if ((cur_ops->stall_dur && cur_ops->stall_dur() > 0) && 3239 (!IS_ENABLED(CONFIG_TINY_RCU) || 3240 (rcu_inkernel_boot_has_ended() && 3241 torture_num_online_cpus() > rfp->rcu_fwd_id))) 3242 rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries); 3243 3244 /* Avoid slow periods, better to test when busy. */ 3245 if (stutter_wait("rcu_torture_fwd_prog")) 3246 sched_set_normal(current, oldnice); 3247 } while (!torture_must_stop()); 3248 /* Short runs might not contain a valid forward-progress attempt. */ 3249 if (!rfp->rcu_fwd_id) { 3250 WARN_ON(!tested && tested_tries >= 5); 3251 pr_alert("%s: tested %d tested_tries %d\n", __func__, tested, tested_tries); 3252 } 3253 torture_kthread_stopping("rcu_torture_fwd_prog"); 3254 return 0; 3255 } 3256 3257 /* If forward-progress checking is requested and feasible, spawn the thread. */ 3258 static int __init rcu_torture_fwd_prog_init(void) 3259 { 3260 int i; 3261 int ret = 0; 3262 struct rcu_fwd *rfp; 3263 3264 if (!fwd_progress) 3265 return 0; /* Not requested, so don't do it. */ 3266 if (fwd_progress >= nr_cpu_ids) { 3267 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Limiting fwd_progress to # CPUs.\n"); 3268 fwd_progress = nr_cpu_ids; 3269 } else if (fwd_progress < 0) { 3270 fwd_progress = nr_cpu_ids; 3271 } 3272 if ((!cur_ops->sync && !cur_ops->call) || 3273 (!cur_ops->cbflood_max && (!cur_ops->stall_dur || cur_ops->stall_dur() <= 0)) || 3274 cur_ops == &rcu_busted_ops) { 3275 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, unsupported by RCU flavor under test"); 3276 fwd_progress = 0; 3277 return 0; 3278 } 3279 if (stall_cpu > 0 || (preempt_duration > 0 && IS_ENABLED(CONFIG_RCU_NOCB_CPU))) { 3280 VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, conflicts with CPU-stall and/or preemption testing"); 3281 fwd_progress = 0; 3282 if (IS_MODULE(CONFIG_RCU_TORTURE_TEST)) 3283 return -EINVAL; /* In module, can fail back to user. */ 3284 WARN_ON(1); /* Make sure rcutorture scripting notices conflict. */ 3285 return 0; 3286 } 3287 if (fwd_progress_holdoff <= 0) 3288 fwd_progress_holdoff = 1; 3289 if (fwd_progress_div <= 0) 3290 fwd_progress_div = 4; 3291 rfp = kcalloc(fwd_progress, sizeof(*rfp), GFP_KERNEL); 3292 fwd_prog_tasks = kcalloc(fwd_progress, sizeof(*fwd_prog_tasks), GFP_KERNEL); 3293 if (!rfp || !fwd_prog_tasks) { 3294 kfree(rfp); 3295 kfree(fwd_prog_tasks); 3296 fwd_prog_tasks = NULL; 3297 fwd_progress = 0; 3298 return -ENOMEM; 3299 } 3300 for (i = 0; i < fwd_progress; i++) { 3301 spin_lock_init(&rfp[i].rcu_fwd_lock); 3302 rfp[i].rcu_fwd_cb_tail = &rfp[i].rcu_fwd_cb_head; 3303 rfp[i].rcu_fwd_id = i; 3304 } 3305 mutex_lock(&rcu_fwd_mutex); 3306 rcu_fwds = rfp; 3307 mutex_unlock(&rcu_fwd_mutex); 3308 register_oom_notifier(&rcutorture_oom_nb); 3309 for (i = 0; i < fwd_progress; i++) { 3310 ret = torture_create_kthread(rcu_torture_fwd_prog, &rcu_fwds[i], fwd_prog_tasks[i]); 3311 if (ret) { 3312 fwd_progress = i; 3313 return ret; 3314 } 3315 } 3316 return 0; 3317 } 3318 3319 static void rcu_torture_fwd_prog_cleanup(void) 3320 { 3321 int i; 3322 struct rcu_fwd *rfp; 3323 3324 if (!rcu_fwds || !fwd_prog_tasks) 3325 return; 3326 for (i = 0; i < fwd_progress; i++) 3327 torture_stop_kthread(rcu_torture_fwd_prog, fwd_prog_tasks[i]); 3328 unregister_oom_notifier(&rcutorture_oom_nb); 3329 mutex_lock(&rcu_fwd_mutex); 3330 rfp = rcu_fwds; 3331 rcu_fwds = NULL; 3332 mutex_unlock(&rcu_fwd_mutex); 3333 kfree(rfp); 3334 kfree(fwd_prog_tasks); 3335 fwd_prog_tasks = NULL; 3336 } 3337 3338 /* Callback function for RCU barrier testing. */ 3339 static void rcu_torture_barrier_cbf(struct rcu_head *rcu) 3340 { 3341 atomic_inc(&barrier_cbs_invoked); 3342 } 3343 3344 /* IPI handler to get callback posted on desired CPU, if online. */ 3345 static int rcu_torture_barrier1cb(void *rcu_void) 3346 { 3347 struct rcu_head *rhp = rcu_void; 3348 3349 cur_ops->call(rhp, rcu_torture_barrier_cbf); 3350 return 0; 3351 } 3352 3353 /* kthread function to register callbacks used to test RCU barriers. */ 3354 static int rcu_torture_barrier_cbs(void *arg) 3355 { 3356 long myid = (long)arg; 3357 bool lastphase = false; 3358 bool newphase; 3359 struct rcu_head rcu; 3360 3361 init_rcu_head_on_stack(&rcu); 3362 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started"); 3363 set_user_nice(current, MAX_NICE); 3364 do { 3365 wait_event(barrier_cbs_wq[myid], 3366 (newphase = 3367 smp_load_acquire(&barrier_phase)) != lastphase || 3368 torture_must_stop()); 3369 lastphase = newphase; 3370 if (torture_must_stop()) 3371 break; 3372 /* 3373 * The above smp_load_acquire() ensures barrier_phase load 3374 * is ordered before the following ->call(). 3375 */ 3376 if (smp_call_on_cpu(myid, rcu_torture_barrier1cb, &rcu, 1)) 3377 cur_ops->call(&rcu, rcu_torture_barrier_cbf); 3378 3379 if (atomic_dec_and_test(&barrier_cbs_count)) 3380 wake_up(&barrier_wq); 3381 } while (!torture_must_stop()); 3382 if (cur_ops->cb_barrier != NULL) 3383 cur_ops->cb_barrier(); 3384 destroy_rcu_head_on_stack(&rcu); 3385 torture_kthread_stopping("rcu_torture_barrier_cbs"); 3386 return 0; 3387 } 3388 3389 /* kthread function to drive and coordinate RCU barrier testing. */ 3390 static int rcu_torture_barrier(void *arg) 3391 { 3392 int i; 3393 3394 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting"); 3395 do { 3396 atomic_set(&barrier_cbs_invoked, 0); 3397 atomic_set(&barrier_cbs_count, n_barrier_cbs); 3398 /* Ensure barrier_phase ordered after prior assignments. */ 3399 smp_store_release(&barrier_phase, !barrier_phase); 3400 for (i = 0; i < n_barrier_cbs; i++) 3401 wake_up(&barrier_cbs_wq[i]); 3402 wait_event(barrier_wq, 3403 atomic_read(&barrier_cbs_count) == 0 || 3404 torture_must_stop()); 3405 if (torture_must_stop()) 3406 break; 3407 n_barrier_attempts++; 3408 cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */ 3409 if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) { 3410 n_rcu_torture_barrier_error++; 3411 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n", 3412 atomic_read(&barrier_cbs_invoked), 3413 n_barrier_cbs); 3414 WARN_ON(1); 3415 // Wait manually for the remaining callbacks 3416 i = 0; 3417 do { 3418 if (WARN_ON(i++ > HZ)) 3419 i = INT_MIN; 3420 schedule_timeout_interruptible(1); 3421 cur_ops->cb_barrier(); 3422 } while (atomic_read(&barrier_cbs_invoked) != 3423 n_barrier_cbs && 3424 !torture_must_stop()); 3425 smp_mb(); // Can't trust ordering if broken. 3426 if (!torture_must_stop()) 3427 pr_err("Recovered: barrier_cbs_invoked = %d\n", 3428 atomic_read(&barrier_cbs_invoked)); 3429 } else { 3430 n_barrier_successes++; 3431 } 3432 schedule_timeout_interruptible(HZ / 10); 3433 } while (!torture_must_stop()); 3434 torture_kthread_stopping("rcu_torture_barrier"); 3435 return 0; 3436 } 3437 3438 /* Initialize RCU barrier testing. */ 3439 static int rcu_torture_barrier_init(void) 3440 { 3441 int i; 3442 int ret; 3443 3444 if (n_barrier_cbs <= 0) 3445 return 0; 3446 if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { 3447 pr_alert("%s" TORTURE_FLAG 3448 " Call or barrier ops missing for %s,\n", 3449 torture_type, cur_ops->name); 3450 pr_alert("%s" TORTURE_FLAG 3451 " RCU barrier testing omitted from run.\n", 3452 torture_type); 3453 return 0; 3454 } 3455 atomic_set(&barrier_cbs_count, 0); 3456 atomic_set(&barrier_cbs_invoked, 0); 3457 barrier_cbs_tasks = 3458 kcalloc(n_barrier_cbs, sizeof(barrier_cbs_tasks[0]), 3459 GFP_KERNEL); 3460 barrier_cbs_wq = 3461 kcalloc(n_barrier_cbs, sizeof(barrier_cbs_wq[0]), GFP_KERNEL); 3462 if (barrier_cbs_tasks == NULL || !barrier_cbs_wq) 3463 return -ENOMEM; 3464 for (i = 0; i < n_barrier_cbs; i++) { 3465 init_waitqueue_head(&barrier_cbs_wq[i]); 3466 ret = torture_create_kthread(rcu_torture_barrier_cbs, 3467 (void *)(long)i, 3468 barrier_cbs_tasks[i]); 3469 if (ret) 3470 return ret; 3471 } 3472 return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task); 3473 } 3474 3475 /* Clean up after RCU barrier testing. */ 3476 static void rcu_torture_barrier_cleanup(void) 3477 { 3478 int i; 3479 3480 torture_stop_kthread(rcu_torture_barrier, barrier_task); 3481 if (barrier_cbs_tasks != NULL) { 3482 for (i = 0; i < n_barrier_cbs; i++) 3483 torture_stop_kthread(rcu_torture_barrier_cbs, 3484 barrier_cbs_tasks[i]); 3485 kfree(barrier_cbs_tasks); 3486 barrier_cbs_tasks = NULL; 3487 } 3488 if (barrier_cbs_wq != NULL) { 3489 kfree(barrier_cbs_wq); 3490 barrier_cbs_wq = NULL; 3491 } 3492 } 3493 3494 static bool rcu_torture_can_boost(void) 3495 { 3496 static int boost_warn_once; 3497 int prio; 3498 3499 if (!(test_boost == 1 && cur_ops->can_boost) && test_boost != 2) 3500 return false; 3501 if (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state) 3502 return false; 3503 3504 prio = rcu_get_gp_kthreads_prio(); 3505 if (!prio) 3506 return false; 3507 3508 if (prio < 2) { 3509 if (boost_warn_once == 1) 3510 return false; 3511 3512 pr_alert("%s: WARN: RCU kthread priority too low to test boosting. Skipping RCU boost test. Try passing rcutree.kthread_prio > 1 on the kernel command line.\n", KBUILD_MODNAME); 3513 boost_warn_once = 1; 3514 return false; 3515 } 3516 3517 return true; 3518 } 3519 3520 static bool read_exit_child_stop; 3521 static bool read_exit_child_stopped; 3522 static wait_queue_head_t read_exit_wq; 3523 3524 // Child kthread which just does an rcutorture reader and exits. 3525 static int rcu_torture_read_exit_child(void *trsp_in) 3526 { 3527 struct torture_random_state *trsp = trsp_in; 3528 3529 set_user_nice(current, MAX_NICE); 3530 // Minimize time between reading and exiting. 3531 while (!kthread_should_stop()) 3532 schedule_timeout_uninterruptible(HZ / 20); 3533 (void)rcu_torture_one_read(trsp, -1); 3534 return 0; 3535 } 3536 3537 // Parent kthread which creates and destroys read-exit child kthreads. 3538 static int rcu_torture_read_exit(void *unused) 3539 { 3540 bool errexit = false; 3541 int i; 3542 struct task_struct *tsp; 3543 DEFINE_TORTURE_RANDOM(trs); 3544 3545 // Allocate and initialize. 3546 set_user_nice(current, MAX_NICE); 3547 VERBOSE_TOROUT_STRING("rcu_torture_read_exit: Start of test"); 3548 3549 // Each pass through this loop does one read-exit episode. 3550 do { 3551 VERBOSE_TOROUT_STRING("rcu_torture_read_exit: Start of episode"); 3552 for (i = 0; i < read_exit_burst; i++) { 3553 if (READ_ONCE(read_exit_child_stop)) 3554 break; 3555 stutter_wait("rcu_torture_read_exit"); 3556 // Spawn child. 3557 tsp = kthread_run(rcu_torture_read_exit_child, 3558 &trs, "%s", "rcu_torture_read_exit_child"); 3559 if (IS_ERR(tsp)) { 3560 TOROUT_ERRSTRING("out of memory"); 3561 errexit = true; 3562 break; 3563 } 3564 cond_resched(); 3565 kthread_stop(tsp); 3566 n_read_exits++; 3567 } 3568 VERBOSE_TOROUT_STRING("rcu_torture_read_exit: End of episode"); 3569 rcu_barrier(); // Wait for task_struct free, avoid OOM. 3570 i = 0; 3571 for (; !errexit && !READ_ONCE(read_exit_child_stop) && i < read_exit_delay; i++) 3572 schedule_timeout_uninterruptible(HZ); 3573 } while (!errexit && !READ_ONCE(read_exit_child_stop)); 3574 3575 // Clean up and exit. 3576 smp_store_release(&read_exit_child_stopped, true); // After reaping. 3577 smp_mb(); // Store before wakeup. 3578 wake_up(&read_exit_wq); 3579 while (!torture_must_stop()) 3580 schedule_timeout_uninterruptible(HZ / 20); 3581 torture_kthread_stopping("rcu_torture_read_exit"); 3582 return 0; 3583 } 3584 3585 static int rcu_torture_read_exit_init(void) 3586 { 3587 if (read_exit_burst <= 0) 3588 return 0; 3589 init_waitqueue_head(&read_exit_wq); 3590 read_exit_child_stop = false; 3591 read_exit_child_stopped = false; 3592 return torture_create_kthread(rcu_torture_read_exit, NULL, 3593 read_exit_task); 3594 } 3595 3596 static void rcu_torture_read_exit_cleanup(void) 3597 { 3598 if (!read_exit_task) 3599 return; 3600 WRITE_ONCE(read_exit_child_stop, true); 3601 smp_mb(); // Above write before wait. 3602 wait_event(read_exit_wq, smp_load_acquire(&read_exit_child_stopped)); 3603 torture_stop_kthread(rcutorture_read_exit, read_exit_task); 3604 } 3605 3606 static void rcutorture_test_nmis(int n) 3607 { 3608 #if IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) 3609 int cpu; 3610 int dumpcpu; 3611 int i; 3612 3613 for (i = 0; i < n; i++) { 3614 preempt_disable(); 3615 cpu = smp_processor_id(); 3616 dumpcpu = cpu + 1; 3617 if (dumpcpu >= nr_cpu_ids) 3618 dumpcpu = 0; 3619 pr_alert("%s: CPU %d invoking dump_cpu_task(%d)\n", __func__, cpu, dumpcpu); 3620 dump_cpu_task(dumpcpu); 3621 preempt_enable(); 3622 schedule_timeout_uninterruptible(15 * HZ); 3623 } 3624 #else // #if IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) 3625 WARN_ONCE(n, "Non-zero rcutorture.test_nmis=%d permitted only when rcutorture is built in.\n", test_nmis); 3626 #endif // #else // #if IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) 3627 } 3628 3629 // Randomly preempt online CPUs. 3630 static int rcu_torture_preempt(void *unused) 3631 { 3632 int cpu = -1; 3633 DEFINE_TORTURE_RANDOM(rand); 3634 3635 schedule_timeout_idle(stall_cpu_holdoff); 3636 do { 3637 // Wait for preempt_interval ms with up to 100us fuzz. 3638 torture_hrtimeout_ms(preempt_interval, 100, &rand); 3639 // Select online CPU. 3640 cpu = cpumask_next(cpu, cpu_online_mask); 3641 if (cpu >= nr_cpu_ids) 3642 cpu = cpumask_next(-1, cpu_online_mask); 3643 WARN_ON_ONCE(cpu >= nr_cpu_ids); 3644 // Move to that CPU, if can't do so, retry later. 3645 if (torture_sched_setaffinity(current->pid, cpumask_of(cpu), false)) 3646 continue; 3647 // Preempt at high-ish priority, then reset to normal. 3648 sched_set_fifo(current); 3649 torture_sched_setaffinity(current->pid, cpu_present_mask, true); 3650 mdelay(preempt_duration); 3651 sched_set_normal(current, 0); 3652 stutter_wait("rcu_torture_preempt"); 3653 } while (!torture_must_stop()); 3654 torture_kthread_stopping("rcu_torture_preempt"); 3655 return 0; 3656 } 3657 3658 static enum cpuhp_state rcutor_hp; 3659 3660 static struct hrtimer gpwrap_lag_timer; 3661 static bool gpwrap_lag_active; 3662 3663 /* Timer handler for toggling RCU grace-period sequence overflow test lag value */ 3664 static enum hrtimer_restart rcu_gpwrap_lag_timer(struct hrtimer *timer) 3665 { 3666 ktime_t next_delay; 3667 3668 if (gpwrap_lag_active) { 3669 pr_alert("rcu-torture: Disabling gpwrap lag (value=0)\n"); 3670 cur_ops->set_gpwrap_lag(0); 3671 gpwrap_lag_active = false; 3672 next_delay = ktime_set((gpwrap_lag_cycle_mins - gpwrap_lag_active_mins) * 60, 0); 3673 } else { 3674 pr_alert("rcu-torture: Enabling gpwrap lag (value=%d)\n", gpwrap_lag_gps); 3675 cur_ops->set_gpwrap_lag(gpwrap_lag_gps); 3676 gpwrap_lag_active = true; 3677 next_delay = ktime_set(gpwrap_lag_active_mins * 60, 0); 3678 } 3679 3680 if (torture_must_stop_irq()) 3681 return HRTIMER_NORESTART; 3682 3683 hrtimer_forward_now(timer, next_delay); 3684 return HRTIMER_RESTART; 3685 } 3686 3687 static int rcu_gpwrap_lag_init(void) 3688 { 3689 if (!gpwrap_lag) 3690 return 0; 3691 3692 if (gpwrap_lag_cycle_mins <= 0 || gpwrap_lag_active_mins <= 0) { 3693 pr_alert("rcu-torture: lag timing parameters must be positive\n"); 3694 return -EINVAL; 3695 } 3696 3697 hrtimer_setup(&gpwrap_lag_timer, rcu_gpwrap_lag_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 3698 gpwrap_lag_active = false; 3699 hrtimer_start(&gpwrap_lag_timer, 3700 ktime_set((gpwrap_lag_cycle_mins - gpwrap_lag_active_mins) * 60, 0), HRTIMER_MODE_REL); 3701 3702 return 0; 3703 } 3704 3705 static void rcu_gpwrap_lag_cleanup(void) 3706 { 3707 hrtimer_cancel(&gpwrap_lag_timer); 3708 cur_ops->set_gpwrap_lag(0); 3709 gpwrap_lag_active = false; 3710 } 3711 static void 3712 rcu_torture_cleanup(void) 3713 { 3714 int firsttime; 3715 int flags = 0; 3716 unsigned long gp_seq = 0; 3717 int i; 3718 int j; 3719 3720 if (torture_cleanup_begin()) { 3721 if (cur_ops->cb_barrier != NULL) { 3722 pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier); 3723 cur_ops->cb_barrier(); 3724 } 3725 if (cur_ops->gp_slow_unregister) 3726 cur_ops->gp_slow_unregister(NULL); 3727 return; 3728 } 3729 if (!cur_ops) { 3730 torture_cleanup_end(); 3731 return; 3732 } 3733 3734 rcutorture_test_nmis(test_nmis); 3735 3736 if (cur_ops->gp_kthread_dbg) 3737 cur_ops->gp_kthread_dbg(); 3738 torture_stop_kthread(rcu_torture_preempt, preempt_task); 3739 rcu_torture_read_exit_cleanup(); 3740 rcu_torture_barrier_cleanup(); 3741 rcu_torture_fwd_prog_cleanup(); 3742 torture_stop_kthread(rcu_torture_stall, stall_task); 3743 torture_stop_kthread(rcu_torture_writer, writer_task); 3744 3745 if (nocb_tasks) { 3746 for (i = 0; i < nrealnocbers; i++) 3747 torture_stop_kthread(rcu_nocb_toggle, nocb_tasks[i]); 3748 kfree(nocb_tasks); 3749 nocb_tasks = NULL; 3750 } 3751 3752 if (reader_tasks) { 3753 for (i = 0; i < nrealreaders; i++) 3754 torture_stop_kthread(rcu_torture_reader, 3755 reader_tasks[i]); 3756 kfree(reader_tasks); 3757 reader_tasks = NULL; 3758 } 3759 kfree(rcu_torture_reader_mbchk); 3760 rcu_torture_reader_mbchk = NULL; 3761 3762 if (fakewriter_tasks) { 3763 for (i = 0; i < nrealfakewriters; i++) 3764 torture_stop_kthread(rcu_torture_fakewriter, 3765 fakewriter_tasks[i]); 3766 kfree(fakewriter_tasks); 3767 fakewriter_tasks = NULL; 3768 } 3769 3770 if (cur_ops->get_gp_data) 3771 cur_ops->get_gp_data(&flags, &gp_seq); 3772 pr_alert("%s: End-test grace-period state: g%ld f%#x total-gps=%ld\n", 3773 cur_ops->name, (long)gp_seq, flags, 3774 rcutorture_seq_diff(gp_seq, start_gp_seq)); 3775 torture_stop_kthread(rcu_torture_stats, stats_task); 3776 torture_stop_kthread(rcu_torture_fqs, fqs_task); 3777 if (rcu_torture_can_boost() && rcutor_hp >= 0) 3778 cpuhp_remove_state(rcutor_hp); 3779 3780 /* 3781 * Wait for all RCU callbacks to fire, then do torture-type-specific 3782 * cleanup operations. 3783 */ 3784 if (cur_ops->cb_barrier != NULL) { 3785 pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier); 3786 cur_ops->cb_barrier(); 3787 } 3788 if (cur_ops->cleanup != NULL) 3789 cur_ops->cleanup(); 3790 3791 rcu_torture_mem_dump_obj(); 3792 3793 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 3794 3795 if (err_segs_recorded) { 3796 pr_alert("Failure/close-call rcutorture reader segments:\n"); 3797 if (rt_read_nsegs == 0) 3798 pr_alert("\t: No segments recorded!!!\n"); 3799 firsttime = 1; 3800 for (i = 0; i < rt_read_nsegs; i++) { 3801 if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP)) 3802 pr_alert("\t%lluus ", div64_u64(err_segs[i].rt_ts, 1000ULL)); 3803 else 3804 pr_alert("\t"); 3805 pr_cont("%d: %#4x", i, err_segs[i].rt_readstate); 3806 if (err_segs[i].rt_delay_jiffies != 0) { 3807 pr_cont("%s%ldjiffies", firsttime ? "" : "+", 3808 err_segs[i].rt_delay_jiffies); 3809 firsttime = 0; 3810 } 3811 if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)) { 3812 pr_cont(" CPU %2d", err_segs[i].rt_cpu); 3813 if (err_segs[i].rt_cpu != err_segs[i].rt_end_cpu) 3814 pr_cont("->%-2d", err_segs[i].rt_end_cpu); 3815 else 3816 pr_cont(" ..."); 3817 } 3818 if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP) && 3819 cur_ops->gather_gp_seqs && cur_ops->format_gp_seqs) { 3820 char buf1[20+1]; 3821 char buf2[20+1]; 3822 char sepchar = '-'; 3823 3824 cur_ops->format_gp_seqs(err_segs[i].rt_gp_seq, 3825 buf1, ARRAY_SIZE(buf1)); 3826 cur_ops->format_gp_seqs(err_segs[i].rt_gp_seq_end, 3827 buf2, ARRAY_SIZE(buf2)); 3828 if (err_segs[i].rt_gp_seq == err_segs[i].rt_gp_seq_end) { 3829 if (buf2[0]) { 3830 for (j = 0; buf2[j]; j++) 3831 buf2[j] = '.'; 3832 if (j) 3833 buf2[j - 1] = ' '; 3834 } 3835 sepchar = ' '; 3836 } 3837 pr_cont(" %s%c%s", buf1, sepchar, buf2); 3838 } 3839 if (err_segs[i].rt_delay_ms != 0) { 3840 pr_cont(" %s%ldms", firsttime ? "" : "+", 3841 err_segs[i].rt_delay_ms); 3842 firsttime = 0; 3843 } 3844 if (err_segs[i].rt_delay_us != 0) { 3845 pr_cont(" %s%ldus", firsttime ? "" : "+", 3846 err_segs[i].rt_delay_us); 3847 firsttime = 0; 3848 } 3849 pr_cont("%s", err_segs[i].rt_preempted ? " preempted" : ""); 3850 if (err_segs[i].rt_readstate & RCUTORTURE_RDR_BH) 3851 pr_cont(" BH"); 3852 if (err_segs[i].rt_readstate & RCUTORTURE_RDR_IRQ) 3853 pr_cont(" IRQ"); 3854 if (err_segs[i].rt_readstate & RCUTORTURE_RDR_PREEMPT) 3855 pr_cont(" PREEMPT"); 3856 if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RBH) 3857 pr_cont(" RBH"); 3858 if (err_segs[i].rt_readstate & RCUTORTURE_RDR_SCHED) 3859 pr_cont(" SCHED"); 3860 if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RCU_1) 3861 pr_cont(" RCU_1"); 3862 if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RCU_2) 3863 pr_cont(" RCU_2"); 3864 pr_cont("\n"); 3865 3866 } 3867 if (rt_read_preempted) 3868 pr_alert("\tReader was preempted.\n"); 3869 } 3870 if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) 3871 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); 3872 else if (torture_onoff_failures()) 3873 rcu_torture_print_module_parms(cur_ops, 3874 "End of test: RCU_HOTPLUG"); 3875 else 3876 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); 3877 torture_cleanup_end(); 3878 if (cur_ops->gp_slow_unregister) 3879 cur_ops->gp_slow_unregister(NULL); 3880 3881 if (gpwrap_lag && cur_ops->set_gpwrap_lag) 3882 rcu_gpwrap_lag_cleanup(); 3883 } 3884 3885 static void rcu_torture_leak_cb(struct rcu_head *rhp) 3886 { 3887 } 3888 3889 static void rcu_torture_err_cb(struct rcu_head *rhp) 3890 { 3891 /* 3892 * This -might- happen due to race conditions, but is unlikely. 3893 * The scenario that leads to this happening is that the 3894 * first of the pair of duplicate callbacks is queued, 3895 * someone else starts a grace period that includes that 3896 * callback, then the second of the pair must wait for the 3897 * next grace period. Unlikely, but can happen. If it 3898 * does happen, the debug-objects subsystem won't have splatted. 3899 */ 3900 pr_alert("%s: duplicated callback was invoked.\n", KBUILD_MODNAME); 3901 } 3902 3903 /* 3904 * Verify that double-free causes debug-objects to complain, but only 3905 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test 3906 * cannot be carried out. 3907 */ 3908 static void rcu_test_debug_objects(void) 3909 { 3910 struct rcu_head rh1; 3911 struct rcu_head rh2; 3912 int idx; 3913 3914 if (!IS_ENABLED(CONFIG_DEBUG_OBJECTS_RCU_HEAD)) { 3915 pr_alert("%s: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_%s()\n", 3916 KBUILD_MODNAME, cur_ops->name); 3917 return; 3918 } 3919 3920 if (WARN_ON_ONCE(cur_ops->debug_objects && 3921 (!cur_ops->call || !cur_ops->cb_barrier))) 3922 return; 3923 3924 struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_KERNEL); 3925 3926 init_rcu_head_on_stack(&rh1); 3927 init_rcu_head_on_stack(&rh2); 3928 pr_alert("%s: WARN: Duplicate call_%s() test starting.\n", KBUILD_MODNAME, cur_ops->name); 3929 3930 /* Try to queue the rh2 pair of callbacks for the same grace period. */ 3931 idx = cur_ops->readlock(); /* Make it impossible to finish a grace period. */ 3932 cur_ops->call(&rh1, rcu_torture_leak_cb); /* Start grace period. */ 3933 cur_ops->call(&rh2, rcu_torture_leak_cb); 3934 cur_ops->call(&rh2, rcu_torture_err_cb); /* Duplicate callback. */ 3935 if (rhp) { 3936 cur_ops->call(rhp, rcu_torture_leak_cb); 3937 cur_ops->call(rhp, rcu_torture_err_cb); /* Another duplicate callback. */ 3938 } 3939 cur_ops->readunlock(idx); 3940 3941 /* Wait for them all to get done so we can safely return. */ 3942 cur_ops->cb_barrier(); 3943 pr_alert("%s: WARN: Duplicate call_%s() test complete.\n", KBUILD_MODNAME, cur_ops->name); 3944 destroy_rcu_head_on_stack(&rh1); 3945 destroy_rcu_head_on_stack(&rh2); 3946 kfree(rhp); 3947 } 3948 3949 static void rcutorture_sync(void) 3950 { 3951 static unsigned long n; 3952 3953 if (cur_ops->sync && !(++n & 0xfff)) 3954 cur_ops->sync(); 3955 } 3956 3957 static DEFINE_MUTEX(mut0); 3958 static DEFINE_MUTEX(mut1); 3959 static DEFINE_MUTEX(mut2); 3960 static DEFINE_MUTEX(mut3); 3961 static DEFINE_MUTEX(mut4); 3962 static DEFINE_MUTEX(mut5); 3963 static DEFINE_MUTEX(mut6); 3964 static DEFINE_MUTEX(mut7); 3965 static DEFINE_MUTEX(mut8); 3966 static DEFINE_MUTEX(mut9); 3967 3968 static DECLARE_RWSEM(rwsem0); 3969 static DECLARE_RWSEM(rwsem1); 3970 static DECLARE_RWSEM(rwsem2); 3971 static DECLARE_RWSEM(rwsem3); 3972 static DECLARE_RWSEM(rwsem4); 3973 static DECLARE_RWSEM(rwsem5); 3974 static DECLARE_RWSEM(rwsem6); 3975 static DECLARE_RWSEM(rwsem7); 3976 static DECLARE_RWSEM(rwsem8); 3977 static DECLARE_RWSEM(rwsem9); 3978 3979 DEFINE_STATIC_SRCU(srcu0); 3980 DEFINE_STATIC_SRCU(srcu1); 3981 DEFINE_STATIC_SRCU(srcu2); 3982 DEFINE_STATIC_SRCU(srcu3); 3983 DEFINE_STATIC_SRCU(srcu4); 3984 DEFINE_STATIC_SRCU(srcu5); 3985 DEFINE_STATIC_SRCU(srcu6); 3986 DEFINE_STATIC_SRCU(srcu7); 3987 DEFINE_STATIC_SRCU(srcu8); 3988 DEFINE_STATIC_SRCU(srcu9); 3989 3990 static int srcu_lockdep_next(const char *f, const char *fl, const char *fs, const char *fu, int i, 3991 int cyclelen, int deadlock) 3992 { 3993 int j = i + 1; 3994 3995 if (j >= cyclelen) 3996 j = deadlock ? 0 : -1; 3997 if (j >= 0) 3998 pr_info("%s: %s(%d), %s(%d), %s(%d)\n", f, fl, i, fs, j, fu, i); 3999 else 4000 pr_info("%s: %s(%d), %s(%d)\n", f, fl, i, fu, i); 4001 return j; 4002 } 4003 4004 // Test lockdep on SRCU-based deadlock scenarios. 4005 static void rcu_torture_init_srcu_lockdep(void) 4006 { 4007 int cyclelen; 4008 int deadlock; 4009 bool err = false; 4010 int i; 4011 int j; 4012 int idx; 4013 struct mutex *muts[] = { &mut0, &mut1, &mut2, &mut3, &mut4, 4014 &mut5, &mut6, &mut7, &mut8, &mut9 }; 4015 struct rw_semaphore *rwsems[] = { &rwsem0, &rwsem1, &rwsem2, &rwsem3, &rwsem4, 4016 &rwsem5, &rwsem6, &rwsem7, &rwsem8, &rwsem9 }; 4017 struct srcu_struct *srcus[] = { &srcu0, &srcu1, &srcu2, &srcu3, &srcu4, 4018 &srcu5, &srcu6, &srcu7, &srcu8, &srcu9 }; 4019 int testtype; 4020 4021 if (!test_srcu_lockdep) 4022 return; 4023 4024 deadlock = test_srcu_lockdep / 1000; 4025 testtype = (test_srcu_lockdep / 10) % 100; 4026 cyclelen = test_srcu_lockdep % 10; 4027 WARN_ON_ONCE(ARRAY_SIZE(muts) != ARRAY_SIZE(srcus)); 4028 if (WARN_ONCE(deadlock != !!deadlock, 4029 "%s: test_srcu_lockdep=%d and deadlock digit %d must be zero or one.\n", 4030 __func__, test_srcu_lockdep, deadlock)) 4031 err = true; 4032 if (WARN_ONCE(cyclelen <= 0, 4033 "%s: test_srcu_lockdep=%d and cycle-length digit %d must be greater than zero.\n", 4034 __func__, test_srcu_lockdep, cyclelen)) 4035 err = true; 4036 if (err) 4037 goto err_out; 4038 4039 if (testtype == 0) { 4040 pr_info("%s: test_srcu_lockdep = %05d: SRCU %d-way %sdeadlock.\n", 4041 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 4042 if (deadlock && cyclelen == 1) 4043 pr_info("%s: Expect hang.\n", __func__); 4044 for (i = 0; i < cyclelen; i++) { 4045 j = srcu_lockdep_next(__func__, "srcu_read_lock", "synchronize_srcu", 4046 "srcu_read_unlock", i, cyclelen, deadlock); 4047 idx = srcu_read_lock(srcus[i]); 4048 if (j >= 0) 4049 synchronize_srcu(srcus[j]); 4050 srcu_read_unlock(srcus[i], idx); 4051 } 4052 return; 4053 } 4054 4055 if (testtype == 1) { 4056 pr_info("%s: test_srcu_lockdep = %05d: SRCU/mutex %d-way %sdeadlock.\n", 4057 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 4058 for (i = 0; i < cyclelen; i++) { 4059 pr_info("%s: srcu_read_lock(%d), mutex_lock(%d), mutex_unlock(%d), srcu_read_unlock(%d)\n", 4060 __func__, i, i, i, i); 4061 idx = srcu_read_lock(srcus[i]); 4062 mutex_lock(muts[i]); 4063 mutex_unlock(muts[i]); 4064 srcu_read_unlock(srcus[i], idx); 4065 4066 j = srcu_lockdep_next(__func__, "mutex_lock", "synchronize_srcu", 4067 "mutex_unlock", i, cyclelen, deadlock); 4068 mutex_lock(muts[i]); 4069 if (j >= 0) 4070 synchronize_srcu(srcus[j]); 4071 mutex_unlock(muts[i]); 4072 } 4073 return; 4074 } 4075 4076 if (testtype == 2) { 4077 pr_info("%s: test_srcu_lockdep = %05d: SRCU/rwsem %d-way %sdeadlock.\n", 4078 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 4079 for (i = 0; i < cyclelen; i++) { 4080 pr_info("%s: srcu_read_lock(%d), down_read(%d), up_read(%d), srcu_read_unlock(%d)\n", 4081 __func__, i, i, i, i); 4082 idx = srcu_read_lock(srcus[i]); 4083 down_read(rwsems[i]); 4084 up_read(rwsems[i]); 4085 srcu_read_unlock(srcus[i], idx); 4086 4087 j = srcu_lockdep_next(__func__, "down_write", "synchronize_srcu", 4088 "up_write", i, cyclelen, deadlock); 4089 down_write(rwsems[i]); 4090 if (j >= 0) 4091 synchronize_srcu(srcus[j]); 4092 up_write(rwsems[i]); 4093 } 4094 return; 4095 } 4096 4097 #ifdef CONFIG_TASKS_TRACE_RCU 4098 if (testtype == 3) { 4099 pr_info("%s: test_srcu_lockdep = %05d: SRCU and Tasks Trace RCU %d-way %sdeadlock.\n", 4100 __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); 4101 if (deadlock && cyclelen == 1) 4102 pr_info("%s: Expect hang.\n", __func__); 4103 for (i = 0; i < cyclelen; i++) { 4104 char *fl = i == 0 ? "rcu_read_lock_trace" : "srcu_read_lock"; 4105 char *fs = i == cyclelen - 1 ? "synchronize_rcu_tasks_trace" 4106 : "synchronize_srcu"; 4107 char *fu = i == 0 ? "rcu_read_unlock_trace" : "srcu_read_unlock"; 4108 4109 j = srcu_lockdep_next(__func__, fl, fs, fu, i, cyclelen, deadlock); 4110 if (i == 0) 4111 rcu_read_lock_trace(); 4112 else 4113 idx = srcu_read_lock(srcus[i]); 4114 if (j >= 0) { 4115 if (i == cyclelen - 1) 4116 synchronize_rcu_tasks_trace(); 4117 else 4118 synchronize_srcu(srcus[j]); 4119 } 4120 if (i == 0) 4121 rcu_read_unlock_trace(); 4122 else 4123 srcu_read_unlock(srcus[i], idx); 4124 } 4125 return; 4126 } 4127 #endif // #ifdef CONFIG_TASKS_TRACE_RCU 4128 4129 err_out: 4130 pr_info("%s: test_srcu_lockdep = %05d does nothing.\n", __func__, test_srcu_lockdep); 4131 pr_info("%s: test_srcu_lockdep = DNNL.\n", __func__); 4132 pr_info("%s: D: Deadlock if nonzero.\n", __func__); 4133 pr_info("%s: NN: Test number, 0=SRCU, 1=SRCU/mutex, 2=SRCU/rwsem, 3=SRCU/Tasks Trace RCU.\n", __func__); 4134 pr_info("%s: L: Cycle length.\n", __func__); 4135 if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU)) 4136 pr_info("%s: NN=3 disallowed because kernel is built with CONFIG_TASKS_TRACE_RCU=n\n", __func__); 4137 } 4138 4139 static int __init 4140 rcu_torture_init(void) 4141 { 4142 long i; 4143 int cpu; 4144 int firsterr = 0; 4145 int flags = 0; 4146 unsigned long gp_seq = 0; 4147 static struct rcu_torture_ops *torture_ops[] = { 4148 &rcu_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops, &busted_srcud_ops, 4149 TASKS_OPS TASKS_RUDE_OPS TASKS_TRACING_OPS 4150 &trivial_ops, 4151 }; 4152 4153 if (!torture_init_begin(torture_type, verbose)) 4154 return -EBUSY; 4155 4156 /* Process args and tell the world that the torturer is on the job. */ 4157 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 4158 cur_ops = torture_ops[i]; 4159 if (strcmp(torture_type, cur_ops->name) == 0) 4160 break; 4161 } 4162 if (i == ARRAY_SIZE(torture_ops)) { 4163 pr_alert("rcu-torture: invalid torture type: \"%s\"\n", 4164 torture_type); 4165 pr_alert("rcu-torture types:"); 4166 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) 4167 pr_cont(" %s", torture_ops[i]->name); 4168 pr_cont("\n"); 4169 firsterr = -EINVAL; 4170 cur_ops = NULL; 4171 goto unwind; 4172 } 4173 if (cur_ops->fqs == NULL && fqs_duration != 0) { 4174 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); 4175 fqs_duration = 0; 4176 } 4177 if (nocbs_nthreads != 0 && (cur_ops != &rcu_ops || 4178 !IS_ENABLED(CONFIG_RCU_NOCB_CPU))) { 4179 pr_alert("rcu-torture types: %s and CONFIG_RCU_NOCB_CPU=%d, nocb toggle disabled.\n", 4180 cur_ops->name, IS_ENABLED(CONFIG_RCU_NOCB_CPU)); 4181 nocbs_nthreads = 0; 4182 } 4183 if (cur_ops->init) 4184 cur_ops->init(); 4185 4186 rcu_torture_init_srcu_lockdep(); 4187 4188 if (nfakewriters >= 0) { 4189 nrealfakewriters = nfakewriters; 4190 } else { 4191 nrealfakewriters = num_online_cpus() - 2 - nfakewriters; 4192 if (nrealfakewriters <= 0) 4193 nrealfakewriters = 1; 4194 } 4195 4196 if (nreaders >= 0) { 4197 nrealreaders = nreaders; 4198 } else { 4199 nrealreaders = num_online_cpus() - 2 - nreaders; 4200 if (nrealreaders <= 0) 4201 nrealreaders = 1; 4202 } 4203 rcu_torture_print_module_parms(cur_ops, "Start of test"); 4204 if (cur_ops->get_gp_data) 4205 cur_ops->get_gp_data(&flags, &gp_seq); 4206 start_gp_seq = gp_seq; 4207 pr_alert("%s: Start-test grace-period state: g%ld f%#x\n", 4208 cur_ops->name, (long)gp_seq, flags); 4209 4210 /* Set up the freelist. */ 4211 4212 INIT_LIST_HEAD(&rcu_torture_freelist); 4213 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { 4214 rcu_tortures[i].rtort_mbtest = 0; 4215 list_add_tail(&rcu_tortures[i].rtort_free, 4216 &rcu_torture_freelist); 4217 } 4218 4219 /* Initialize the statistics so that each run gets its own numbers. */ 4220 4221 rcu_torture_current = NULL; 4222 rcu_torture_current_version = 0; 4223 atomic_set(&n_rcu_torture_alloc, 0); 4224 atomic_set(&n_rcu_torture_alloc_fail, 0); 4225 atomic_set(&n_rcu_torture_free, 0); 4226 atomic_set(&n_rcu_torture_mberror, 0); 4227 atomic_set(&n_rcu_torture_mbchk_fail, 0); 4228 atomic_set(&n_rcu_torture_mbchk_tries, 0); 4229 atomic_set(&n_rcu_torture_error, 0); 4230 n_rcu_torture_barrier_error = 0; 4231 n_rcu_torture_boost_ktrerror = 0; 4232 n_rcu_torture_boost_failure = 0; 4233 n_rcu_torture_boosts = 0; 4234 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 4235 atomic_set(&rcu_torture_wcount[i], 0); 4236 for_each_possible_cpu(cpu) { 4237 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { 4238 per_cpu(rcu_torture_count, cpu)[i] = 0; 4239 per_cpu(rcu_torture_batch, cpu)[i] = 0; 4240 } 4241 } 4242 err_segs_recorded = 0; 4243 rt_read_nsegs = 0; 4244 4245 /* Start up the kthreads. */ 4246 4247 rcu_torture_write_types(); 4248 firsterr = torture_create_kthread(rcu_torture_writer, NULL, 4249 writer_task); 4250 if (torture_init_error(firsterr)) 4251 goto unwind; 4252 4253 if (nrealfakewriters > 0) { 4254 fakewriter_tasks = kcalloc(nrealfakewriters, 4255 sizeof(fakewriter_tasks[0]), 4256 GFP_KERNEL); 4257 if (fakewriter_tasks == NULL) { 4258 TOROUT_ERRSTRING("out of memory"); 4259 firsterr = -ENOMEM; 4260 goto unwind; 4261 } 4262 } 4263 for (i = 0; i < nrealfakewriters; i++) { 4264 firsterr = torture_create_kthread(rcu_torture_fakewriter, 4265 NULL, fakewriter_tasks[i]); 4266 if (torture_init_error(firsterr)) 4267 goto unwind; 4268 } 4269 reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]), 4270 GFP_KERNEL); 4271 rcu_torture_reader_mbchk = kcalloc(nrealreaders, sizeof(*rcu_torture_reader_mbchk), 4272 GFP_KERNEL); 4273 if (!reader_tasks || !rcu_torture_reader_mbchk) { 4274 TOROUT_ERRSTRING("out of memory"); 4275 firsterr = -ENOMEM; 4276 goto unwind; 4277 } 4278 for (i = 0; i < nrealreaders; i++) { 4279 rcu_torture_reader_mbchk[i].rtc_chkrdr = -1; 4280 firsterr = torture_create_kthread(rcu_torture_reader, (void *)i, 4281 reader_tasks[i]); 4282 if (torture_init_error(firsterr)) 4283 goto unwind; 4284 } 4285 nrealnocbers = nocbs_nthreads; 4286 if (WARN_ON(nrealnocbers < 0)) 4287 nrealnocbers = 1; 4288 if (WARN_ON(nocbs_toggle < 0)) 4289 nocbs_toggle = HZ; 4290 if (nrealnocbers > 0) { 4291 nocb_tasks = kcalloc(nrealnocbers, sizeof(nocb_tasks[0]), GFP_KERNEL); 4292 if (nocb_tasks == NULL) { 4293 TOROUT_ERRSTRING("out of memory"); 4294 firsterr = -ENOMEM; 4295 goto unwind; 4296 } 4297 } else { 4298 nocb_tasks = NULL; 4299 } 4300 for (i = 0; i < nrealnocbers; i++) { 4301 firsterr = torture_create_kthread(rcu_nocb_toggle, NULL, nocb_tasks[i]); 4302 if (torture_init_error(firsterr)) 4303 goto unwind; 4304 } 4305 if (stat_interval > 0) { 4306 firsterr = torture_create_kthread(rcu_torture_stats, NULL, 4307 stats_task); 4308 if (torture_init_error(firsterr)) 4309 goto unwind; 4310 } 4311 if (test_no_idle_hz && shuffle_interval > 0) { 4312 firsterr = torture_shuffle_init(shuffle_interval * HZ); 4313 if (torture_init_error(firsterr)) 4314 goto unwind; 4315 } 4316 if (stutter < 0) 4317 stutter = 0; 4318 if (stutter) { 4319 int t; 4320 4321 t = cur_ops->stall_dur ? cur_ops->stall_dur() : stutter * HZ; 4322 firsterr = torture_stutter_init(stutter * HZ, t); 4323 if (torture_init_error(firsterr)) 4324 goto unwind; 4325 } 4326 if (fqs_duration < 0) 4327 fqs_duration = 0; 4328 if (fqs_holdoff < 0) 4329 fqs_holdoff = 0; 4330 if (fqs_duration && fqs_holdoff) { 4331 /* Create the fqs thread */ 4332 firsterr = torture_create_kthread(rcu_torture_fqs, NULL, 4333 fqs_task); 4334 if (torture_init_error(firsterr)) 4335 goto unwind; 4336 } 4337 if (test_boost_interval < 1) 4338 test_boost_interval = 1; 4339 if (test_boost_duration < 2) 4340 test_boost_duration = 2; 4341 if (rcu_torture_can_boost()) { 4342 4343 boost_starttime = jiffies + test_boost_interval * HZ; 4344 4345 firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE", 4346 rcutorture_booster_init, 4347 rcutorture_booster_cleanup); 4348 rcutor_hp = firsterr; 4349 if (torture_init_error(firsterr)) 4350 goto unwind; 4351 } 4352 shutdown_jiffies = jiffies + shutdown_secs * HZ; 4353 firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); 4354 if (torture_init_error(firsterr)) 4355 goto unwind; 4356 firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval, 4357 rcutorture_sync); 4358 if (torture_init_error(firsterr)) 4359 goto unwind; 4360 firsterr = rcu_torture_stall_init(); 4361 if (torture_init_error(firsterr)) 4362 goto unwind; 4363 firsterr = rcu_torture_fwd_prog_init(); 4364 if (torture_init_error(firsterr)) 4365 goto unwind; 4366 firsterr = rcu_torture_barrier_init(); 4367 if (torture_init_error(firsterr)) 4368 goto unwind; 4369 firsterr = rcu_torture_read_exit_init(); 4370 if (torture_init_error(firsterr)) 4371 goto unwind; 4372 if (preempt_duration > 0) { 4373 firsterr = torture_create_kthread(rcu_torture_preempt, NULL, preempt_task); 4374 if (torture_init_error(firsterr)) 4375 goto unwind; 4376 } 4377 if (object_debug) 4378 rcu_test_debug_objects(); 4379 4380 if (cur_ops->gp_slow_register && !WARN_ON_ONCE(!cur_ops->gp_slow_unregister)) 4381 cur_ops->gp_slow_register(&rcu_fwd_cb_nodelay); 4382 4383 if (gpwrap_lag && cur_ops->set_gpwrap_lag) { 4384 firsterr = rcu_gpwrap_lag_init(); 4385 if (torture_init_error(firsterr)) 4386 goto unwind; 4387 } 4388 4389 torture_init_end(); 4390 return 0; 4391 4392 unwind: 4393 torture_init_end(); 4394 rcu_torture_cleanup(); 4395 if (shutdown_secs) { 4396 WARN_ON(!IS_MODULE(CONFIG_RCU_TORTURE_TEST)); 4397 kernel_power_off(); 4398 } 4399 return firsterr; 4400 } 4401 4402 module_init(rcu_torture_init); 4403 module_exit(rcu_torture_cleanup); 4404