3  * kernel/workqueue.c - generic async execution with shared worker pool
25  * Please read Documentation/core-api/workqueue.rst for details.
35 #include <linux/workqueue.h>
239  * tools/workqueue/wq_monitor.py.
255  * The per-pool workqueue.  While queued, bits below WORK_PWQ_SHIFT
262 	struct workqueue_struct *wq;		/* I: the owning workqueue */
306  * Structure used to wait for workqueue flush.
317  * Unlike in a per-cpu workqueue where max_active limits its concurrency level
318  * on each CPU, in an unbound workqueue, max_active applies to the whole system.
337  * The externally visible workqueue.  It relays the issued work items to
375 	char			name[WQ_NAME_LEN]; /* I: workqueue name */
540 #include <trace/events/workqueue.h>
590  * for_each_pwq - iterate through all pool_workqueues of the specified workqueue
592  * @wq: the target workqueue
742  * unbound_effective_cpumask - effective cpumask of an unbound workqueue
743  * @wq: workqueue of interest
1094  * before the original execution finishes, workqueue will identify the
1309  * should be using an unbound workqueue instead.
1358 …printk_deferred(KERN_WARNING "workqueue: %ps hogged CPU for >%luus %llu times, consider switching …  in wq_cpu_intensive_report()
1542  * As this function doesn't involve any workqueue-related locking, it
1560  * @wq: workqueue of interest
1585  * @wq: workqueue to update
1736 		/* BH or per-cpu workqueue, pwq->nr_active is sufficient */  in pwq_tryinc_nr_active()
1745 	 * Unbound workqueue uses per-node shared nr_active $nna. If @pwq is  in pwq_tryinc_nr_active()
1960 	 * For a percpu workqueue, it's simple. Just need to kick the first  in pwq_dec_nr_active()
1969 	 * If @pwq is for an unbound workqueue, it's more complicated because  in pwq_dec_nr_active()
1995  * decrement nr_in_flight of its pwq and handle workqueue flushing.
2209  * same workqueue.
2236 		pr_warn_once("workqueue: round-robin CPU selection forced, expect performance impact\n");  in wq_select_unbound_cpu()
2265 	 * For a draining wq, only works from the same workqueue are  in __queue_work()
2270 		     WARN_ONCE(!is_chained_work(wq), "workqueue: cannot queue %ps on wq %s\n",  in __queue_work()
2292 	 * For ordered workqueue, work items must be queued on the newest pwq  in __queue_work()
2331 		WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",  in __queue_work()
2384  * @wq: workqueue to use
2446  * @wq: workqueue to use
2474 	 * If this is used with a per-cpu workqueue then the logic in  in queue_work_node()
2548  * @wq: workqueue to use
2586  * @wq: workqueue to use
2629  * @wq: workqueue to use
2779  * create_worker - create a new workqueue worker
2798 		pr_err_once("workqueue: Failed to allocate a worker ID: %pe\n",  in create_worker()
2805 		pr_err_once("workqueue: Failed to allocate a worker\n");  in create_worker()
2819 				pr_err("workqueue: Interrupted when creating a worker thread \"%s\"\n",  in create_worker()
2822 				pr_err_once("workqueue: Failed to create a worker thread: %pe",  in create_worker()
3290 		pr_err("BUG: workqueue leaked atomic, lock or RCU: %s[%d]\n"  in process_one_work()
3379  * work items regardless of their specific target workqueue.  The only
3390 	/* tell the scheduler that this is a workqueue worker */  in worker_thread()
3519  * Workqueue rescuer thread function.  There's one rescuer for each
3520  * workqueue which has WQ_MEM_RECLAIM set.
3553 	 * By the time the rescuer is requested to stop, the workqueue  in rescuer_thread()
3675  * TODO: Convert all tasklet users to workqueue and use softirq directly.
3774  * @target_wq: workqueue being flushed
3775  * @target_work: work item being flushed (NULL for workqueue flushes)
3782  * on a workqueue which doesn't have %WQ_MEM_RECLAIM as that can break forward-
3799 		  "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%ps",  in check_flush_dependency()
3803 		  "workqueue: WQ_MEM_RECLAIM %s:%ps is flushing !WQ_MEM_RECLAIM %s:%ps",  in check_flush_dependency()
3898  * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
3899  * @wq: workqueue being flushed
3903  * Prepare pwqs for workqueue flushing.
3941 	 * For unbound workqueue, pwqs will map to only a few pools.  in flush_workqueue_prep_pwqs()
4016  * @wq: workqueue to flush
4172  * drain_workqueue - drain a workqueue
4173  * @wq: workqueue to drain
4175  * Wait until the workqueue becomes empty.  While draining is in progress,
4213 			pr_warn("workqueue %s: %s() isn't complete after %u tries\n",  in drain_workqueue()
4264 	 * single-threaded or rescuer equipped workqueue.  in start_flush_work()
4299 	 * was queued on a BH workqueue, we also know that it was running in the  in __flush_work()
4402 		WARN_ONCE(true, "workqueue: work disable count overflowed\n");  in work_offqd_disable()
4410 		WARN_ONCE(true, "workqueue: work disable count underflowed\n");  in work_offqd_enable()
4470  * even if the work re-queues itself or migrates to another workqueue. On return
4478  * workqueue. Can also be called from non-hardirq atomic contexts including BH
4479  * if @work was last queued on a BH workqueue.
4552  * workqueue. Can also be called from non-hardirq atomic contexts including BH
4553  * if @work was last queued on a BH workqueue.
4633  * system workqueue and blocks until all CPUs have completed.
4752  * Some attrs fields are workqueue-only. Clear them for worker_pool's. See the
5169 		 * For ordered workqueue with a plugged dfl_pwq, restart it now.  in pwq_release_workfn()
5289  * @attrs: the wq_attrs of the default pwq of the target workqueue
5292  * Calculate the cpumask a workqueue with @attrs should use on @pod.
5335 	struct workqueue_struct	*wq;		/* target workqueue */
5471  * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
5472  * @wq: the target workqueue
5475  * Apply @attrs to an unbound workqueue @wq. Unless disabled, this function maps
5499  * @wq: the target workqueue
5509  * Note that when the last allowed CPU of a pod goes offline for a workqueue
5511  * executing the work items for the workqueue will lose their CPU affinity and
5513  * CPU_DOWN. If a workqueue user wants strict affinity, it's the user's
5544 		pr_warn("workqueue: allocation failed while updating CPU pod affinity of \"%s\"\n",  in unbound_wq_update_pwq()
5614 		     "ordering guarantee broken for workqueue %s\n", wq->name);  in alloc_and_link_pwqs()
5639 		pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",  in wq_clamp_max_active()
5662 		pr_err("workqueue: Failed to allocate a rescuer for wq \"%s\"\n",  in init_rescuer()
5673 		pr_err("workqueue: Failed to create a rescuer kthread for wq \"%s\": %pe",  in init_rescuer()
5694  * @wq: target workqueue
5794 		pr_warn_once("workqueue: name exceeds WQ_NAME_LEN. Truncating to: %s\n",  in __alloc_workqueue()
5934  * destroy_workqueue - safely terminate a workqueue
5935  * @wq: target workqueue
5937  * Safely destroy a workqueue. All work currently pending will be done first.
5941  * before destroying the workqueue. The fundamental problem is that, currently,
5942  * the workqueue has no way of accessing non-pending delayed_work. delayed_work
5961 	/* mark the workqueue destruction is in progress */  in destroy_workqueue()
6026  * workqueue_set_max_active - adjust max_active of a workqueue
6027  * @wq: target workqueue
6060  * workqueue_set_min_active - adjust min_active of an unbound workqueue
6061  * @wq: target unbound workqueue
6064  * Set min_active of an unbound workqueue. Unlike other types of workqueues, an
6065  * unbound workqueue is not guaranteed to be able to process max_active
6066  * interdependent work items. Instead, an unbound workqueue is guaranteed to be
6089  * Determine if %current task is a workqueue worker and what it's working on.
6092  * Return: work struct if %current task is a workqueue worker, %NULL otherwise.
6103  * current_is_workqueue_rescuer - is %current workqueue rescuer?
6105  * Determine whether %current is a workqueue rescuer.  Can be used from
6108  * Return: %true if %current is a workqueue rescuer. %false otherwise.
6118  * workqueue_congested - test whether a workqueue is congested
6120  * @wq: target workqueue
6122  * Test whether @wq's cpu workqueue for @cpu is congested.  There is
6129  * pool_workqueues, each with its own congested state. A workqueue being
6130  * congested on one CPU doesn't mean that the workqueue is contested on any
6218  * name of the workqueue being serviced and worker description set with
6244 	 * Carefully copy the associated workqueue's workfn, name and desc.  in print_worker_info()
6408  * show_one_workqueue - dump state of specified workqueue
6409  * @wq: workqueue whose state will be printed
6423 	if (idle) /* Nothing to print for idle workqueue */  in show_one_workqueue()
6426 	pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);  in show_one_workqueue()
6501  * show_all_workqueues - dump workqueue state
6525  * show_freezable_workqueues - dump freezable workqueue state
7031  * Update the unbound workqueue cpumask on top of the new housekeeping cpumask such
7051 	 * by any subsequent write to workqueue/cpumask sysfs file.  in workqueue_unbound_housekeeping_update()
7120  * /sys/bus/workqueue/devices/WQ_NAME.  All visible workqueues have the
7123  *  per_cpu		RO bool	: whether the workqueue is per-cpu or unbound
7356 	.name				= "workqueue",
7466  * workqueue_sysfs_register - make a workqueue visible in sysfs
7467  * @wq: the workqueue to register
7469  * Expose @wq in sysfs under /sys/bus/workqueue/devices.
7473  * Workqueue user should use this function directly iff it wants to apply
7474  * workqueue_attrs before making the workqueue visible in sysfs; otherwise,
7534  * @wq: the workqueue to unregister
7553  * Workqueue watchdog.
7557  * indefinitely.  Workqueue stalls can be very difficult to debug as the
7558  * usual warning mechanisms don't trigger and internal workqueue state is
7561  * Workqueue watchdog monitors all worker pools periodically and dumps
7566  * "workqueue.watchdog_thresh" which can be updated at runtime through the
7645 			panic("workqueue: %u stall(s) exceeded threshold %u\n",  in panic_on_wq_watchdog()
7650 		panic("workqueue: stall lasted %us, exceeding threshold %us\n",  in panic_on_wq_watchdog()
7732 			pr_emerg("BUG: workqueue lockup - pool");  in wq_watchdog_timer_fn()
7830 		pr_warn("workqueue: Restricting unbound_cpumask (%*pb) with %s (%*pb) leaves no CPU, ignoring\n",  in restrict_unbound_cpumask()
7855  * workqueue_init_early - early init for workqueue subsystem
7857  * This is the first step of three-staged workqueue subsystem initialization and
7884 		restrict_unbound_cpumask("workqueue.unbound_cpus", &wq_cmdline_cpumask);  in workqueue_init_early()
7895 	 * If nohz_full is enabled, set power efficient workqueue as unbound.  in workqueue_init_early()
7896 	 * This allows workqueue items to be moved to HK CPUs.  in workqueue_init_early()
8012  * workqueue_init - bring workqueue subsystem fully online
8014  * This is the second step of three-staged workqueue subsystem initialization
8043 		     "workqueue: failed to create early rescuer for %s",  in workqueue_init()
8138  * This is the third step of three-staged workqueue subsystem initialization and
8158 	 * worker pool. Explicitly call unbound_wq_update_pwq() on all workqueue  in workqueue_init_topology()
8185 		pr_warn("workqueue.unbound_cpus: incorrect CPU range, using default\n");  in workqueue_unbound_cpus_setup()
8190 __setup("workqueue.unbound_cpus=", workqueue_unbound_cpus_setup);