Lines Matching +full:full +full:- +full:pwr +full:- +full:cycle
1 // SPDX-License-Identifier: GPL-2.0-only
3 * kernel/workqueue.c - generic async execution with shared worker pool
10 * Kai Petzke <wpp@marie.physik.tu-berlin.de>
22 * pools for workqueues which are not bound to any specific CPU - the
25 * Please read Documentation/core-api/workqueue.rst for details.
89 UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */
114 * I: Modifiable by initialization/destruction paths and read-only for
120 * L: pool->lock protected. Access with pool->lock held.
122 * X: During normal operation, modification requires pool->lock and should
124 * cpu or grabbing pool->lock is enough for read access. If
133 * PW: wq_pool_mutex and wq->mutex protected for writes. Either for reads.
135 * PWR: wq_pool_mutex and wq->mutex protected for writes. Either or
138 * WQ: wq->mutex protected.
140 * WR: wq->mutex protected for writes. RCU protected for reads.
194 * The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS
195 * of work_struct->data are used for flags and the remaining high bits
210 struct list_head pwqs_node; /* WR: node on wq->pwqs */
211 struct list_head mayday_node; /* MD: node on wq->maydays */
217 * determined without grabbing wq->mutex.
278 struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwqs */
279 struct pool_workqueue __rcu *numa_pwq_tbl[]; /* PWR: unbound pwqs indexed by node */
303 static DEFINE_RAW_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
318 * following always forces round-robin CPU selection on unbound work items
328 /* the per-cpu worker pools */
333 /* PL: hash of all unbound pools keyed by pool->attrs */
371 !lockdep_is_held(&wq->mutex) && \
373 "RCU, wq->mutex or wq_pool_mutex should be held")
381 * for_each_pool - iterate through all worker_pools in the system
398 * for_each_pool_worker - iterate through all workers of a worker_pool
408 list_for_each_entry((worker), &(pool)->workers, node) \
413 * for_each_pwq - iterate through all pool_workqueues of the specified workqueue
417 * This must be called either with wq->mutex held or RCU read locked.
425 list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node, \
426 lockdep_is_held(&(wq->mutex)))
434 return ((struct work_struct *) addr)->func; in work_debug_hint()
446 * - an active object is initialized
464 * - an active object is freed
515 destroy_timer_on_stack(&work->timer); in destroy_delayed_work_on_stack()
516 debug_object_free(&work->work, &work_debug_descr); in destroy_delayed_work_on_stack()
526 * worker_pool_assign_id - allocate ID and assing it to @pool
530 * successfully, -errno on failure.
541 pool->id = ret; in worker_pool_assign_id()
548 * unbound_pwq_by_node - return the unbound pool_workqueue for the given node
552 * This must be called with any of wq_pool_mutex, wq->mutex or RCU
566 * delayed item is pending. The plan is to keep CPU -> NODE in unbound_pwq_by_node()
571 return wq->dfl_pwq; in unbound_pwq_by_node()
573 return rcu_dereference_raw(wq->numa_pwq_tbl[node]); in unbound_pwq_by_node()
584 ((1 << WORK_STRUCT_COLOR_BITS) - 1); in get_work_color()
599 * work->data. These functions should only be called while the work is
600 * owned - ie. while the PENDING bit is set.
616 atomic_long_set(&work->data, data | flags | work_static(work)); in set_work_data()
647 * work->current_func, which is executed afterwards. This possible in set_work_pool_and_clear_pending()
652 * ---------------------------- -------------------------------- in set_work_pool_and_clear_pending()
660 * 7 work->current_func() { in set_work_pool_and_clear_pending()
664 * Without an explicit full barrier speculative LOAD on line 8 can in set_work_pool_and_clear_pending()
683 unsigned long data = atomic_long_read(&work->data); in get_work_pwq()
692 * get_work_pool - return the worker_pool a given work was associated with
708 unsigned long data = atomic_long_read(&work->data); in get_work_pool()
715 (data & WORK_STRUCT_WQ_DATA_MASK))->pool; in get_work_pool()
725 * get_work_pool_id - return the worker pool ID a given work is associated with
733 unsigned long data = atomic_long_read(&work->data); in get_work_pool_id()
737 (data & WORK_STRUCT_WQ_DATA_MASK))->pool->id; in get_work_pool_id()
752 unsigned long data = atomic_long_read(&work->data); in work_is_canceling()
760 * they're being called with pool->lock held.
765 return !atomic_read(&pool->nr_running); in __need_more_worker()
778 return !list_empty(&pool->worklist) && __need_more_worker(pool); in need_more_worker()
784 return pool->nr_idle; in may_start_working()
790 return !list_empty(&pool->worklist) && in keep_working()
791 atomic_read(&pool->nr_running) <= 1; in keep_working()
803 bool managing = pool->flags & POOL_MANAGER_ACTIVE; in too_many_workers()
804 int nr_idle = pool->nr_idle + managing; /* manager is considered idle */ in too_many_workers()
805 int nr_busy = pool->nr_workers - nr_idle; in too_many_workers()
807 return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy; in too_many_workers()
817 if (unlikely(list_empty(&pool->idle_list))) in first_idle_worker()
820 return list_first_entry(&pool->idle_list, struct worker, entry); in first_idle_worker()
824 * wake_up_worker - wake up an idle worker
830 * raw_spin_lock_irq(pool->lock).
837 wake_up_process(worker->task); in wake_up_worker()
841 * wq_worker_running - a worker is running again
850 if (!worker->sleeping) in wq_worker_running()
852 if (!(worker->flags & WORKER_NOT_RUNNING)) in wq_worker_running()
853 atomic_inc(&worker->pool->nr_running); in wq_worker_running()
854 worker->sleeping = 0; in wq_worker_running()
858 * wq_worker_sleeping - a worker is going to sleep
862 * going to sleep. Preemption needs to be disabled to protect ->sleeping
875 if (worker->flags & WORKER_NOT_RUNNING) in wq_worker_sleeping()
878 pool = worker->pool; in wq_worker_sleeping()
881 if (worker->sleeping) in wq_worker_sleeping()
884 worker->sleeping = 1; in wq_worker_sleeping()
885 raw_spin_lock_irq(&pool->lock); in wq_worker_sleeping()
898 if (atomic_dec_and_test(&pool->nr_running) && in wq_worker_sleeping()
899 !list_empty(&pool->worklist)) { in wq_worker_sleeping()
902 wake_up_process(next->task); in wq_worker_sleeping()
904 raw_spin_unlock_irq(&pool->lock); in wq_worker_sleeping()
908 * wq_worker_last_func - retrieve worker's last work function
915 * raw_spin_lock_irq(rq->lock)
919 * dequeuing, to allow periodic aggregation to shut-off when that
922 * As this function doesn't involve any workqueue-related locking, it
935 return worker->last_func; in wq_worker_last_func()
939 * worker_set_flags - set worker flags and adjust nr_running accordingly
943 * Set @flags in @worker->flags and adjust nr_running accordingly.
946 * raw_spin_lock_irq(pool->lock)
950 struct worker_pool *pool = worker->pool; in worker_set_flags()
952 WARN_ON_ONCE(worker->task != current); in worker_set_flags()
956 !(worker->flags & WORKER_NOT_RUNNING)) { in worker_set_flags()
957 atomic_dec(&pool->nr_running); in worker_set_flags()
960 worker->flags |= flags; in worker_set_flags()
964 * worker_clr_flags - clear worker flags and adjust nr_running accordingly
968 * Clear @flags in @worker->flags and adjust nr_running accordingly.
971 * raw_spin_lock_irq(pool->lock)
975 struct worker_pool *pool = worker->pool; in worker_clr_flags()
976 unsigned int oflags = worker->flags; in worker_clr_flags()
978 WARN_ON_ONCE(worker->task != current); in worker_clr_flags()
980 worker->flags &= ~flags; in worker_clr_flags()
988 if (!(worker->flags & WORKER_NOT_RUNNING)) in worker_clr_flags()
989 atomic_inc(&pool->nr_running); in worker_clr_flags()
993 * find_worker_executing_work - find worker which is executing a work
998 * @pool->busy_hash which is keyed by the address of @work. For a worker
1019 * raw_spin_lock_irq(pool->lock).
1030 hash_for_each_possible(pool->busy_hash, worker, hentry, in find_worker_executing_work()
1032 if (worker->current_work == work && in find_worker_executing_work()
1033 worker->current_func == work->func) in find_worker_executing_work()
1040 * move_linked_works - move linked works to a list
1054 * raw_spin_lock_irq(pool->lock).
1066 list_move_tail(&work->entry, head); in move_linked_works()
1081 * get_pwq - get an extra reference on the specified pool_workqueue
1085 * @pwq has positive refcnt and be holding the matching pool->lock.
1089 lockdep_assert_held(&pwq->pool->lock); in get_pwq()
1090 WARN_ON_ONCE(pwq->refcnt <= 0); in get_pwq()
1091 pwq->refcnt++; in get_pwq()
1095 * put_pwq - put a pool_workqueue reference
1099 * destruction. The caller should be holding the matching pool->lock.
1103 lockdep_assert_held(&pwq->pool->lock); in put_pwq()
1104 if (likely(--pwq->refcnt)) in put_pwq()
1106 if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND))) in put_pwq()
1109 * @pwq can't be released under pool->lock, bounce to in put_pwq()
1111 * pool->lock as this path is taken only for unbound workqueues and in put_pwq()
1112 * the release work item is scheduled on a per-cpu workqueue. To in put_pwq()
1113 * avoid lockdep warning, unbound pool->locks are given lockdep in put_pwq()
1116 schedule_work(&pwq->unbound_release_work); in put_pwq()
1120 * put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
1132 raw_spin_lock_irq(&pwq->pool->lock); in put_pwq_unlocked()
1134 raw_spin_unlock_irq(&pwq->pool->lock); in put_pwq_unlocked()
1143 if (list_empty(&pwq->pool->worklist)) in pwq_activate_delayed_work()
1144 pwq->pool->watchdog_ts = jiffies; in pwq_activate_delayed_work()
1145 move_linked_works(work, &pwq->pool->worklist, NULL); in pwq_activate_delayed_work()
1147 pwq->nr_active++; in pwq_activate_delayed_work()
1152 struct work_struct *work = list_first_entry(&pwq->delayed_works, in pwq_activate_first_delayed()
1159 * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
1167 * raw_spin_lock_irq(pool->lock).
1175 pwq->nr_in_flight[color]--; in pwq_dec_nr_in_flight()
1177 pwq->nr_active--; in pwq_dec_nr_in_flight()
1178 if (!list_empty(&pwq->delayed_works)) { in pwq_dec_nr_in_flight()
1180 if (pwq->nr_active < pwq->max_active) in pwq_dec_nr_in_flight()
1185 if (likely(pwq->flush_color != color)) in pwq_dec_nr_in_flight()
1188 /* are there still in-flight works? */ in pwq_dec_nr_in_flight()
1189 if (pwq->nr_in_flight[color]) in pwq_dec_nr_in_flight()
1193 pwq->flush_color = -1; in pwq_dec_nr_in_flight()
1199 if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush)) in pwq_dec_nr_in_flight()
1200 complete(&pwq->wq->first_flusher->done); in pwq_dec_nr_in_flight()
1206 * try_to_grab_pending - steal work item from worklist and disable irq
1212 * stable state - idle, on timer or on worklist.
1219 * -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
1220 * -ENOENT if someone else is canceling @work, this state may persist
1227 * disabled on entry. This, combined with delayed_work->timer being
1228 * irqsafe, ensures that we return -EAGAIN for finite short period of time.
1248 * dwork->timer is irqsafe. If del_timer() fails, it's in try_to_grab_pending()
1252 if (likely(del_timer(&dwork->timer))) in try_to_grab_pending()
1263 * steal it from ->worklist without clearing WORK_STRUCT_PENDING. in try_to_grab_pending()
1269 raw_spin_lock(&pool->lock); in try_to_grab_pending()
1271 * work->data is guaranteed to point to pwq only while the work in try_to_grab_pending()
1272 * item is queued on pwq->wq, and both updating work->data to point in try_to_grab_pending()
1274 * pwq->pool->lock. This in turn guarantees that, if work->data in try_to_grab_pending()
1279 if (pwq && pwq->pool == pool) { in try_to_grab_pending()
1285 * on the delayed_list, will confuse pwq->nr_active in try_to_grab_pending()
1292 list_del_init(&work->entry); in try_to_grab_pending()
1295 /* work->data points to pwq iff queued, point to pool */ in try_to_grab_pending()
1296 set_work_pool_and_keep_pending(work, pool->id); in try_to_grab_pending()
1298 raw_spin_unlock(&pool->lock); in try_to_grab_pending()
1302 raw_spin_unlock(&pool->lock); in try_to_grab_pending()
1307 return -ENOENT; in try_to_grab_pending()
1309 return -EAGAIN; in try_to_grab_pending()
1313 * insert_work - insert a work into a pool
1323 * raw_spin_lock_irq(pool->lock).
1328 struct worker_pool *pool = pwq->pool; in insert_work()
1332 list_add_tail(&work->entry, head); in insert_work()
1359 return worker && worker->current_pwq->wq == wq; in is_chained_work()
1376 pr_warn("workqueue: round-robin CPU selection forced, expect performance impact\n"); in wq_select_unbound_cpu()
1406 * steal the PENDING will busy-loop waiting for it to either get in __queue_work()
1415 if (unlikely(wq->flags & __WQ_DRAINING) && in __queue_work()
1421 if (wq->flags & WQ_UNBOUND) { in __queue_work()
1428 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu); in __queue_work()
1434 * pool to guarantee non-reentrancy. in __queue_work()
1437 if (last_pool && last_pool != pwq->pool) { in __queue_work()
1440 raw_spin_lock(&last_pool->lock); in __queue_work()
1444 if (worker && worker->current_pwq->wq == wq) { in __queue_work()
1445 pwq = worker->current_pwq; in __queue_work()
1448 raw_spin_unlock(&last_pool->lock); in __queue_work()
1449 raw_spin_lock(&pwq->pool->lock); in __queue_work()
1452 raw_spin_lock(&pwq->pool->lock); in __queue_work()
1461 * make forward-progress. in __queue_work()
1463 if (unlikely(!pwq->refcnt)) { in __queue_work()
1464 if (wq->flags & WQ_UNBOUND) { in __queue_work()
1465 raw_spin_unlock(&pwq->pool->lock); in __queue_work()
1470 WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt", in __queue_work()
1471 wq->name, cpu); in __queue_work()
1477 if (WARN_ON(!list_empty(&work->entry))) in __queue_work()
1480 pwq->nr_in_flight[pwq->work_color]++; in __queue_work()
1481 work_flags = work_color_to_flags(pwq->work_color); in __queue_work()
1483 if (likely(pwq->nr_active < pwq->max_active)) { in __queue_work()
1485 pwq->nr_active++; in __queue_work()
1486 worklist = &pwq->pool->worklist; in __queue_work()
1488 pwq->pool->watchdog_ts = jiffies; in __queue_work()
1491 worklist = &pwq->delayed_works; in __queue_work()
1497 raw_spin_unlock(&pwq->pool->lock); in __queue_work()
1502 * queue_work_on - queue work on specific cpu
1531 * workqueue_select_cpu_near - Select a CPU based on NUMA node
1564 * queue_work_node - queue work on a "random" cpu for a given NUMA node
1594 * If this is used with a per-cpu workqueue then the logic in in queue_work_node()
1598 WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)); in queue_work_node()
1619 __queue_work(dwork->cpu, dwork->wq, &dwork->work); in delayed_work_timer_fn()
1626 struct timer_list *timer = &dwork->timer; in __queue_delayed_work()
1627 struct work_struct *work = &dwork->work; in __queue_delayed_work()
1630 WARN_ON_ONCE(timer->function != delayed_work_timer_fn); in __queue_delayed_work()
1632 WARN_ON_ONCE(!list_empty(&work->entry)); in __queue_delayed_work()
1635 * If @delay is 0, queue @dwork->work immediately. This is for in __queue_delayed_work()
1641 __queue_work(cpu, wq, &dwork->work); in __queue_delayed_work()
1645 dwork->wq = wq; in __queue_delayed_work()
1646 dwork->cpu = cpu; in __queue_delayed_work()
1647 timer->expires = jiffies + delay; in __queue_delayed_work()
1656 * queue_delayed_work_on - queue work on specific CPU after delay
1669 struct work_struct *work = &dwork->work; in queue_delayed_work_on()
1687 * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
1711 ret = try_to_grab_pending(&dwork->work, true, &flags); in mod_delayed_work_on()
1712 } while (unlikely(ret == -EAGAIN)); in mod_delayed_work_on()
1719 /* -ENOENT from try_to_grab_pending() becomes %true */ in mod_delayed_work_on()
1730 __queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work); in rcu_work_rcufn()
1735 * queue_rcu_work - queue work after a RCU grace period
1740 * that a full RCU grace period is guaranteed only after a %true return.
1742 * execution may happen before a full RCU grace period has passed.
1746 struct work_struct *work = &rwork->work; in queue_rcu_work()
1749 rwork->wq = wq; in queue_rcu_work()
1750 call_rcu(&rwork->rcu, rcu_work_rcufn); in queue_rcu_work()
1759 * worker_enter_idle - enter idle state
1766 * raw_spin_lock_irq(pool->lock).
1770 struct worker_pool *pool = worker->pool; in worker_enter_idle()
1772 if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) || in worker_enter_idle()
1773 WARN_ON_ONCE(!list_empty(&worker->entry) && in worker_enter_idle()
1774 (worker->hentry.next || worker->hentry.pprev))) in worker_enter_idle()
1778 worker->flags |= WORKER_IDLE; in worker_enter_idle()
1779 pool->nr_idle++; in worker_enter_idle()
1780 worker->last_active = jiffies; in worker_enter_idle()
1783 list_add(&worker->entry, &pool->idle_list); in worker_enter_idle()
1785 if (too_many_workers(pool) && !timer_pending(&pool->idle_timer)) in worker_enter_idle()
1786 mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT); in worker_enter_idle()
1790 * pool->lock between setting %WORKER_UNBOUND and zapping in worker_enter_idle()
1794 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) && in worker_enter_idle()
1795 pool->nr_workers == pool->nr_idle && in worker_enter_idle()
1796 atomic_read(&pool->nr_running)); in worker_enter_idle()
1800 * worker_leave_idle - leave idle state
1806 * raw_spin_lock_irq(pool->lock).
1810 struct worker_pool *pool = worker->pool; in worker_leave_idle()
1812 if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE))) in worker_leave_idle()
1815 pool->nr_idle--; in worker_leave_idle()
1816 list_del_init(&worker->entry); in worker_leave_idle()
1825 INIT_LIST_HEAD(&worker->entry); in alloc_worker()
1826 INIT_LIST_HEAD(&worker->scheduled); in alloc_worker()
1827 INIT_LIST_HEAD(&worker->node); in alloc_worker()
1829 worker->flags = WORKER_PREP; in alloc_worker()
1835 * worker_attach_to_pool() - attach a worker to a pool
1840 * cpu-binding of @worker are kept coordinated with the pool across
1841 * cpu-[un]hotplugs.
1850 * online CPUs. It'll be re-applied when any of the CPUs come up. in worker_attach_to_pool()
1852 set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask); in worker_attach_to_pool()
1859 if (pool->flags & POOL_DISASSOCIATED) in worker_attach_to_pool()
1860 worker->flags |= WORKER_UNBOUND; in worker_attach_to_pool()
1862 list_add_tail(&worker->node, &pool->workers); in worker_attach_to_pool()
1863 worker->pool = pool; in worker_attach_to_pool()
1869 * worker_detach_from_pool() - detach a worker from its pool
1878 struct worker_pool *pool = worker->pool; in worker_detach_from_pool()
1883 list_del(&worker->node); in worker_detach_from_pool()
1884 worker->pool = NULL; in worker_detach_from_pool()
1886 if (list_empty(&pool->workers)) in worker_detach_from_pool()
1887 detach_completion = pool->detach_completion; in worker_detach_from_pool()
1890 /* clear leftover flags without pool->lock after it is detached */ in worker_detach_from_pool()
1891 worker->flags &= ~(WORKER_UNBOUND | WORKER_REBOUND); in worker_detach_from_pool()
1898 * create_worker - create a new workqueue worker
1912 int id = -1; in create_worker()
1916 id = ida_simple_get(&pool->worker_ida, 0, 0, GFP_KERNEL); in create_worker()
1920 worker = alloc_worker(pool->node); in create_worker()
1924 worker->id = id; in create_worker()
1926 if (pool->cpu >= 0) in create_worker()
1927 snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id, in create_worker()
1928 pool->attrs->nice < 0 ? "H" : ""); in create_worker()
1930 snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id); in create_worker()
1932 worker->task = kthread_create_on_node(worker_thread, worker, pool->node, in create_worker()
1934 if (IS_ERR(worker->task)) in create_worker()
1937 set_user_nice(worker->task, pool->attrs->nice); in create_worker()
1938 kthread_bind_mask(worker->task, pool->attrs->cpumask); in create_worker()
1944 raw_spin_lock_irq(&pool->lock); in create_worker()
1945 worker->pool->nr_workers++; in create_worker()
1947 wake_up_process(worker->task); in create_worker()
1948 raw_spin_unlock_irq(&pool->lock); in create_worker()
1954 ida_simple_remove(&pool->worker_ida, id); in create_worker()
1960 * destroy_worker - destroy a workqueue worker
1967 * raw_spin_lock_irq(pool->lock).
1971 struct worker_pool *pool = worker->pool; in destroy_worker()
1973 lockdep_assert_held(&pool->lock); in destroy_worker()
1976 if (WARN_ON(worker->current_work) || in destroy_worker()
1977 WARN_ON(!list_empty(&worker->scheduled)) || in destroy_worker()
1978 WARN_ON(!(worker->flags & WORKER_IDLE))) in destroy_worker()
1981 pool->nr_workers--; in destroy_worker()
1982 pool->nr_idle--; in destroy_worker()
1984 list_del_init(&worker->entry); in destroy_worker()
1985 worker->flags |= WORKER_DIE; in destroy_worker()
1986 wake_up_process(worker->task); in destroy_worker()
1993 raw_spin_lock_irq(&pool->lock); in idle_worker_timeout()
2000 worker = list_entry(pool->idle_list.prev, struct worker, entry); in idle_worker_timeout()
2001 expires = worker->last_active + IDLE_WORKER_TIMEOUT; in idle_worker_timeout()
2004 mod_timer(&pool->idle_timer, expires); in idle_worker_timeout()
2011 raw_spin_unlock_irq(&pool->lock); in idle_worker_timeout()
2017 struct workqueue_struct *wq = pwq->wq; in send_mayday()
2021 if (!wq->rescuer) in send_mayday()
2025 if (list_empty(&pwq->mayday_node)) { in send_mayday()
2032 list_add_tail(&pwq->mayday_node, &wq->maydays); in send_mayday()
2033 wake_up_process(wq->rescuer->task); in send_mayday()
2042 raw_spin_lock_irq(&pool->lock); in pool_mayday_timeout()
2043 raw_spin_lock(&wq_mayday_lock); /* for wq->maydays */ in pool_mayday_timeout()
2052 list_for_each_entry(work, &pool->worklist, entry) in pool_mayday_timeout()
2057 raw_spin_unlock_irq(&pool->lock); in pool_mayday_timeout()
2059 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL); in pool_mayday_timeout()
2063 * maybe_create_worker - create a new worker if necessary
2076 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2081 __releases(&pool->lock) in maybe_create_worker()
2082 __acquires(&pool->lock) in maybe_create_worker()
2085 raw_spin_unlock_irq(&pool->lock); in maybe_create_worker()
2088 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT); in maybe_create_worker()
2100 del_timer_sync(&pool->mayday_timer); in maybe_create_worker()
2101 raw_spin_lock_irq(&pool->lock); in maybe_create_worker()
2104 * created as @pool->lock was dropped and the new worker might have in maybe_create_worker()
2112 * manage_workers - manage worker pool
2124 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2135 struct worker_pool *pool = worker->pool; in manage_workers()
2137 if (pool->flags & POOL_MANAGER_ACTIVE) in manage_workers()
2140 pool->flags |= POOL_MANAGER_ACTIVE; in manage_workers()
2141 pool->manager = worker; in manage_workers()
2145 pool->manager = NULL; in manage_workers()
2146 pool->flags &= ~POOL_MANAGER_ACTIVE; in manage_workers()
2152 * process_one_work - process single work
2163 * raw_spin_lock_irq(pool->lock) which is released and regrabbed.
2166 __releases(&pool->lock) in process_one_work()
2167 __acquires(&pool->lock) in process_one_work()
2170 struct worker_pool *pool = worker->pool; in process_one_work()
2171 bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE; in process_one_work()
2180 * work->lockdep_map, make a copy and use that here. in process_one_work()
2184 lockdep_copy_map(&lockdep_map, &work->lockdep_map); in process_one_work()
2187 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) && in process_one_work()
2188 raw_smp_processor_id() != pool->cpu); in process_one_work()
2198 move_linked_works(work, &collision->scheduled, NULL); in process_one_work()
2204 hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work); in process_one_work()
2205 worker->current_work = work; in process_one_work()
2206 worker->current_func = work->func; in process_one_work()
2207 worker->current_pwq = pwq; in process_one_work()
2214 strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN); in process_one_work()
2216 list_del_init(&work->entry); in process_one_work()
2229 * false for normal per-cpu workers since nr_running would always in process_one_work()
2239 * update to @work. Also, do this inside @pool->lock so that in process_one_work()
2243 set_work_pool_and_clear_pending(work, pool->id); in process_one_work()
2245 raw_spin_unlock_irq(&pool->lock); in process_one_work()
2247 lock_map_acquire(&pwq->wq->lockdep_map); in process_one_work()
2260 * Which would create W1->C->W1 dependencies, even though there is no in process_one_work()
2262 * read-recursive acquire on the work(queue) 'locks', but this will then in process_one_work()
2267 * flush_work() and complete() primitives (except for single-threaded in process_one_work()
2272 worker->current_func(work); in process_one_work()
2277 trace_workqueue_execute_end(work, worker->current_func); in process_one_work()
2279 lock_map_release(&pwq->wq->lockdep_map); in process_one_work()
2284 current->comm, preempt_count(), task_pid_nr(current), in process_one_work()
2285 worker->current_func); in process_one_work()
2300 raw_spin_lock_irq(&pool->lock); in process_one_work()
2307 worker->last_func = worker->current_func; in process_one_work()
2310 hash_del(&worker->hentry); in process_one_work()
2311 worker->current_work = NULL; in process_one_work()
2312 worker->current_func = NULL; in process_one_work()
2313 worker->current_pwq = NULL; in process_one_work()
2318 * process_scheduled_works - process scheduled works
2326 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2331 while (!list_empty(&worker->scheduled)) { in process_scheduled_works()
2332 struct work_struct *work = list_first_entry(&worker->scheduled, in process_scheduled_works()
2342 current->flags |= PF_WQ_WORKER; in set_pf_worker()
2344 current->flags &= ~PF_WQ_WORKER; in set_pf_worker()
2349 * worker_thread - the worker thread function
2352 * The worker thread function. All workers belong to a worker_pool -
2353 * either a per-cpu one or dynamic unbound one. These workers process all
2363 struct worker_pool *pool = worker->pool; in worker_thread()
2368 raw_spin_lock_irq(&pool->lock); in worker_thread()
2371 if (unlikely(worker->flags & WORKER_DIE)) { in worker_thread()
2372 raw_spin_unlock_irq(&pool->lock); in worker_thread()
2373 WARN_ON_ONCE(!list_empty(&worker->entry)); in worker_thread()
2376 set_task_comm(worker->task, "kworker/dying"); in worker_thread()
2377 ida_simple_remove(&pool->worker_ida, worker->id); in worker_thread()
2394 * ->scheduled list can only be filled while a worker is in worker_thread()
2398 WARN_ON_ONCE(!list_empty(&worker->scheduled)); in worker_thread()
2411 list_first_entry(&pool->worklist, in worker_thread()
2414 pool->watchdog_ts = jiffies; in worker_thread()
2419 if (unlikely(!list_empty(&worker->scheduled))) in worker_thread()
2422 move_linked_works(work, &worker->scheduled, NULL); in worker_thread()
2430 * pool->lock is held and there's no work to process and no need to in worker_thread()
2432 * pool->lock or from local cpu, so setting the current state in worker_thread()
2433 * before releasing pool->lock is enough to prevent losing any in worker_thread()
2438 raw_spin_unlock_irq(&pool->lock); in worker_thread()
2444 * rescuer_thread - the rescuer thread function
2467 struct workqueue_struct *wq = rescuer->rescue_wq; in rescuer_thread()
2468 struct list_head *scheduled = &rescuer->scheduled; in rescuer_thread()
2483 * shouldn't have any work pending, but @wq->maydays may still have in rescuer_thread()
2484 * pwq(s) queued. This can happen by non-rescuer workers consuming in rescuer_thread()
2486 * @wq->maydays processing before acting on should_stop so that the in rescuer_thread()
2494 while (!list_empty(&wq->maydays)) { in rescuer_thread()
2495 struct pool_workqueue *pwq = list_first_entry(&wq->maydays, in rescuer_thread()
2497 struct worker_pool *pool = pwq->pool; in rescuer_thread()
2502 list_del_init(&pwq->mayday_node); in rescuer_thread()
2508 raw_spin_lock_irq(&pool->lock); in rescuer_thread()
2515 list_for_each_entry_safe(work, n, &pool->worklist, entry) { in rescuer_thread()
2518 pool->watchdog_ts = jiffies; in rescuer_thread()
2532 * that such back-to-back work items, which may be in rescuer_thread()
2536 if (pwq->nr_active && need_to_create_worker(pool)) { in rescuer_thread()
2542 if (wq->rescuer && list_empty(&pwq->mayday_node)) { in rescuer_thread()
2544 list_add_tail(&pwq->mayday_node, &wq->maydays); in rescuer_thread()
2564 raw_spin_unlock_irq(&pool->lock); in rescuer_thread()
2580 WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING)); in rescuer_thread()
2586 * check_flush_dependency - check for flush dependency sanity
2593 * %WQ_MEM_RECLAIM as that can break forward-progress guarantee leading to
2599 work_func_t target_func = target_work ? target_work->func : NULL; in check_flush_dependency()
2602 if (target_wq->flags & WQ_MEM_RECLAIM) in check_flush_dependency()
2607 WARN_ONCE(current->flags & PF_MEMALLOC, in check_flush_dependency()
2609 current->pid, current->comm, target_wq->name, target_func); in check_flush_dependency()
2610 WARN_ONCE(worker && ((worker->current_pwq->wq->flags & in check_flush_dependency()
2613 worker->current_pwq->wq->name, worker->current_func, in check_flush_dependency()
2614 target_wq->name, target_func); in check_flush_dependency()
2626 complete(&barr->done); in wq_barrier_func()
2630 * insert_wq_barrier - insert a barrier work
2647 * Note that when @worker is non-NULL, @target may be modified
2651 * raw_spin_lock_irq(pool->lock).
2661 * debugobject calls are safe here even with pool->lock locked in insert_wq_barrier()
2666 INIT_WORK_ONSTACK(&barr->work, wq_barrier_func); in insert_wq_barrier()
2667 __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work)); in insert_wq_barrier()
2669 init_completion_map(&barr->done, &target->lockdep_map); in insert_wq_barrier()
2671 barr->task = current; in insert_wq_barrier()
2678 head = worker->scheduled.next; in insert_wq_barrier()
2682 head = target->entry.next; in insert_wq_barrier()
2688 debug_work_activate(&barr->work); in insert_wq_barrier()
2689 insert_work(pwq, &barr->work, head, in insert_wq_barrier()
2694 * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
2696 * @flush_color: new flush color, < 0 for no-op
2697 * @work_color: new work color, < 0 for no-op
2701 * If @flush_color is non-negative, flush_color on all pwqs should be
2702 * -1. If no pwq has in-flight commands at the specified color, all
2703 * pwq->flush_color's stay at -1 and %false is returned. If any pwq
2704 * has in flight commands, its pwq->flush_color is set to
2705 * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
2708 * The caller should have initialized @wq->first_flusher prior to
2709 * calling this function with non-negative @flush_color. If
2713 * If @work_color is non-negative, all pwqs should have the same
2718 * mutex_lock(wq->mutex).
2731 WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush)); in flush_workqueue_prep_pwqs()
2732 atomic_set(&wq->nr_pwqs_to_flush, 1); in flush_workqueue_prep_pwqs()
2736 struct worker_pool *pool = pwq->pool; in flush_workqueue_prep_pwqs()
2738 raw_spin_lock_irq(&pool->lock); in flush_workqueue_prep_pwqs()
2741 WARN_ON_ONCE(pwq->flush_color != -1); in flush_workqueue_prep_pwqs()
2743 if (pwq->nr_in_flight[flush_color]) { in flush_workqueue_prep_pwqs()
2744 pwq->flush_color = flush_color; in flush_workqueue_prep_pwqs()
2745 atomic_inc(&wq->nr_pwqs_to_flush); in flush_workqueue_prep_pwqs()
2751 WARN_ON_ONCE(work_color != work_next_color(pwq->work_color)); in flush_workqueue_prep_pwqs()
2752 pwq->work_color = work_color; in flush_workqueue_prep_pwqs()
2755 raw_spin_unlock_irq(&pool->lock); in flush_workqueue_prep_pwqs()
2758 if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush)) in flush_workqueue_prep_pwqs()
2759 complete(&wq->first_flusher->done); in flush_workqueue_prep_pwqs()
2765 * flush_workqueue - ensure that any scheduled work has run to completion.
2775 .flush_color = -1, in flush_workqueue()
2776 .done = COMPLETION_INITIALIZER_ONSTACK_MAP(this_flusher.done, wq->lockdep_map), in flush_workqueue()
2783 lock_map_acquire(&wq->lockdep_map); in flush_workqueue()
2784 lock_map_release(&wq->lockdep_map); in flush_workqueue()
2786 mutex_lock(&wq->mutex); in flush_workqueue()
2789 * Start-to-wait phase in flush_workqueue()
2791 next_color = work_next_color(wq->work_color); in flush_workqueue()
2793 if (next_color != wq->flush_color) { in flush_workqueue()
2795 * Color space is not full. The current work_color in flush_workqueue()
2799 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow)); in flush_workqueue()
2800 this_flusher.flush_color = wq->work_color; in flush_workqueue()
2801 wq->work_color = next_color; in flush_workqueue()
2803 if (!wq->first_flusher) { in flush_workqueue()
2805 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color); in flush_workqueue()
2807 wq->first_flusher = &this_flusher; in flush_workqueue()
2809 if (!flush_workqueue_prep_pwqs(wq, wq->flush_color, in flush_workqueue()
2810 wq->work_color)) { in flush_workqueue()
2812 wq->flush_color = next_color; in flush_workqueue()
2813 wq->first_flusher = NULL; in flush_workqueue()
2818 WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color); in flush_workqueue()
2819 list_add_tail(&this_flusher.list, &wq->flusher_queue); in flush_workqueue()
2820 flush_workqueue_prep_pwqs(wq, -1, wq->work_color); in flush_workqueue()
2824 * Oops, color space is full, wait on overflow queue. in flush_workqueue()
2828 list_add_tail(&this_flusher.list, &wq->flusher_overflow); in flush_workqueue()
2833 mutex_unlock(&wq->mutex); in flush_workqueue()
2838 * Wake-up-and-cascade phase in flush_workqueue()
2841 * handling overflow. Non-first flushers can simply return. in flush_workqueue()
2843 if (READ_ONCE(wq->first_flusher) != &this_flusher) in flush_workqueue()
2846 mutex_lock(&wq->mutex); in flush_workqueue()
2849 if (wq->first_flusher != &this_flusher) in flush_workqueue()
2852 WRITE_ONCE(wq->first_flusher, NULL); in flush_workqueue()
2855 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color); in flush_workqueue()
2861 list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) { in flush_workqueue()
2862 if (next->flush_color != wq->flush_color) in flush_workqueue()
2864 list_del_init(&next->list); in flush_workqueue()
2865 complete(&next->done); in flush_workqueue()
2868 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) && in flush_workqueue()
2869 wq->flush_color != work_next_color(wq->work_color)); in flush_workqueue()
2872 wq->flush_color = work_next_color(wq->flush_color); in flush_workqueue()
2875 if (!list_empty(&wq->flusher_overflow)) { in flush_workqueue()
2879 * flusher_queue. This is the start-to-wait in flush_workqueue()
2882 list_for_each_entry(tmp, &wq->flusher_overflow, list) in flush_workqueue()
2883 tmp->flush_color = wq->work_color; in flush_workqueue()
2885 wq->work_color = work_next_color(wq->work_color); in flush_workqueue()
2887 list_splice_tail_init(&wq->flusher_overflow, in flush_workqueue()
2888 &wq->flusher_queue); in flush_workqueue()
2889 flush_workqueue_prep_pwqs(wq, -1, wq->work_color); in flush_workqueue()
2892 if (list_empty(&wq->flusher_queue)) { in flush_workqueue()
2893 WARN_ON_ONCE(wq->flush_color != wq->work_color); in flush_workqueue()
2901 WARN_ON_ONCE(wq->flush_color == wq->work_color); in flush_workqueue()
2902 WARN_ON_ONCE(wq->flush_color != next->flush_color); in flush_workqueue()
2904 list_del_init(&next->list); in flush_workqueue()
2905 wq->first_flusher = next; in flush_workqueue()
2907 if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1)) in flush_workqueue()
2914 wq->first_flusher = NULL; in flush_workqueue()
2918 mutex_unlock(&wq->mutex); in flush_workqueue()
2923 * drain_workqueue - drain a workqueue
2940 * hotter than drain_workqueue() and already looks at @wq->flags. in drain_workqueue()
2943 mutex_lock(&wq->mutex); in drain_workqueue()
2944 if (!wq->nr_drainers++) in drain_workqueue()
2945 wq->flags |= __WQ_DRAINING; in drain_workqueue()
2946 mutex_unlock(&wq->mutex); in drain_workqueue()
2950 mutex_lock(&wq->mutex); in drain_workqueue()
2955 raw_spin_lock_irq(&pwq->pool->lock); in drain_workqueue()
2956 drained = !pwq->nr_active && list_empty(&pwq->delayed_works); in drain_workqueue()
2957 raw_spin_unlock_irq(&pwq->pool->lock); in drain_workqueue()
2965 wq->name, flush_cnt); in drain_workqueue()
2967 mutex_unlock(&wq->mutex); in drain_workqueue()
2971 if (!--wq->nr_drainers) in drain_workqueue()
2972 wq->flags &= ~__WQ_DRAINING; in drain_workqueue()
2973 mutex_unlock(&wq->mutex); in drain_workqueue()
2993 raw_spin_lock_irq(&pool->lock); in start_flush_work()
2997 if (unlikely(pwq->pool != pool)) in start_flush_work()
3003 pwq = worker->current_pwq; in start_flush_work()
3006 check_flush_dependency(pwq->wq, work); in start_flush_work()
3009 raw_spin_unlock_irq(&pool->lock); in start_flush_work()
3013 * single-threaded or rescuer equipped workqueue. in start_flush_work()
3021 (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)) { in start_flush_work()
3022 lock_map_acquire(&pwq->wq->lockdep_map); in start_flush_work()
3023 lock_map_release(&pwq->wq->lockdep_map); in start_flush_work()
3028 raw_spin_unlock_irq(&pool->lock); in start_flush_work()
3040 if (WARN_ON(!work->func)) in __flush_work()
3044 lock_map_acquire(&work->lockdep_map); in __flush_work()
3045 lock_map_release(&work->lockdep_map); in __flush_work()
3058 * flush_work - wait for a work to finish executing the last queueing instance
3083 if (cwait->work != key) in cwt_wakefn()
3101 * CANCELING - flush_work() will return false immediately in __cancel_work_timer()
3103 * return -ENOENT as @work is still being canceled and the in __cancel_work_timer()
3112 if (unlikely(ret == -ENOENT)) { in __cancel_work_timer()
3153 * cancel_work_sync - cancel a work and wait for it to finish
3157 * can be used even if the work re-queues itself or migrates to
3161 * cancel_work_sync(&delayed_work->work) must not be used for
3177 * flush_delayed_work - wait for a dwork to finish executing the last queueing
3191 if (del_timer_sync(&dwork->timer)) in flush_delayed_work()
3192 __queue_work(dwork->cpu, dwork->wq, &dwork->work); in flush_delayed_work()
3194 return flush_work(&dwork->work); in flush_delayed_work()
3199 * flush_rcu_work - wait for a rwork to finish executing the last queueing
3208 if (test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&rwork->work))) { in flush_rcu_work()
3210 flush_work(&rwork->work); in flush_rcu_work()
3213 return flush_work(&rwork->work); in flush_rcu_work()
3225 } while (unlikely(ret == -EAGAIN)); in __cancel_work()
3236 * cancel_delayed_work - cancel a delayed work
3246 * it returns %true and the work doesn't re-arm itself. Explicitly flush or
3253 return __cancel_work(&dwork->work, true); in cancel_delayed_work()
3258 * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
3268 return __cancel_work_timer(&dwork->work, true); in cancel_delayed_work_sync()
3273 * schedule_on_each_cpu - execute a function synchronously on each online CPU
3281 * 0 on success, -errno on failure.
3290 return -ENOMEM; in schedule_on_each_cpu()
3310 * execute_in_process_context - reliably execute the routine with user context
3318 * Return: 0 - function was executed
3319 * 1 - function was scheduled for execution
3324 fn(&ew->work); in execute_in_process_context()
3328 INIT_WORK(&ew->work, fn); in execute_in_process_context()
3329 schedule_work(&ew->work); in execute_in_process_context()
3336 * free_workqueue_attrs - free a workqueue_attrs
3344 free_cpumask_var(attrs->cpumask); in free_workqueue_attrs()
3350 * alloc_workqueue_attrs - allocate a workqueue_attrs
3364 if (!alloc_cpumask_var(&attrs->cpumask, GFP_KERNEL)) in alloc_workqueue_attrs()
3367 cpumask_copy(attrs->cpumask, cpu_possible_mask); in alloc_workqueue_attrs()
3377 to->nice = from->nice; in copy_workqueue_attrs()
3378 cpumask_copy(to->cpumask, from->cpumask); in copy_workqueue_attrs()
3381 * ->no_numa as it is used for both pool and wq attrs. Instead, in copy_workqueue_attrs()
3382 * get_unbound_pool() explicitly clears ->no_numa after copying. in copy_workqueue_attrs()
3384 to->no_numa = from->no_numa; in copy_workqueue_attrs()
3392 hash = jhash_1word(attrs->nice, hash); in wqattrs_hash()
3393 hash = jhash(cpumask_bits(attrs->cpumask), in wqattrs_hash()
3402 if (a->nice != b->nice) in wqattrs_equal()
3404 if (!cpumask_equal(a->cpumask, b->cpumask)) in wqattrs_equal()
3410 * init_worker_pool - initialize a newly zalloc'd worker_pool
3413 * Initialize a newly zalloc'd @pool. It also allocates @pool->attrs.
3415 * Return: 0 on success, -errno on failure. Even on failure, all fields
3421 raw_spin_lock_init(&pool->lock); in init_worker_pool()
3422 pool->id = -1; in init_worker_pool()
3423 pool->cpu = -1; in init_worker_pool()
3424 pool->node = NUMA_NO_NODE; in init_worker_pool()
3425 pool->flags |= POOL_DISASSOCIATED; in init_worker_pool()
3426 pool->watchdog_ts = jiffies; in init_worker_pool()
3427 INIT_LIST_HEAD(&pool->worklist); in init_worker_pool()
3428 INIT_LIST_HEAD(&pool->idle_list); in init_worker_pool()
3429 hash_init(pool->busy_hash); in init_worker_pool()
3431 timer_setup(&pool->idle_timer, idle_worker_timeout, TIMER_DEFERRABLE); in init_worker_pool()
3433 timer_setup(&pool->mayday_timer, pool_mayday_timeout, 0); in init_worker_pool()
3435 INIT_LIST_HEAD(&pool->workers); in init_worker_pool()
3437 ida_init(&pool->worker_ida); in init_worker_pool()
3438 INIT_HLIST_NODE(&pool->hash_node); in init_worker_pool()
3439 pool->refcnt = 1; in init_worker_pool()
3442 pool->attrs = alloc_workqueue_attrs(); in init_worker_pool()
3443 if (!pool->attrs) in init_worker_pool()
3444 return -ENOMEM; in init_worker_pool()
3453 lockdep_register_key(&wq->key); in wq_init_lockdep()
3454 lock_name = kasprintf(GFP_KERNEL, "%s%s", "(wq_completion)", wq->name); in wq_init_lockdep()
3456 lock_name = wq->name; in wq_init_lockdep()
3458 wq->lock_name = lock_name; in wq_init_lockdep()
3459 lockdep_init_map(&wq->lockdep_map, lock_name, &wq->key, 0); in wq_init_lockdep()
3464 lockdep_unregister_key(&wq->key); in wq_unregister_lockdep()
3469 if (wq->lock_name != wq->name) in wq_free_lockdep()
3470 kfree(wq->lock_name); in wq_free_lockdep()
3493 if (!(wq->flags & WQ_UNBOUND)) in rcu_free_wq()
3494 free_percpu(wq->cpu_pwqs); in rcu_free_wq()
3496 free_workqueue_attrs(wq->unbound_attrs); in rcu_free_wq()
3505 ida_destroy(&pool->worker_ida); in rcu_free_pool()
3506 free_workqueue_attrs(pool->attrs); in rcu_free_pool()
3513 raw_spin_lock_irq(&pool->lock); in wq_manager_inactive()
3515 if (pool->flags & POOL_MANAGER_ACTIVE) { in wq_manager_inactive()
3516 raw_spin_unlock_irq(&pool->lock); in wq_manager_inactive()
3523 * put_unbound_pool - put a worker_pool
3540 if (--pool->refcnt) in put_unbound_pool()
3544 if (WARN_ON(!(pool->cpu < 0)) || in put_unbound_pool()
3545 WARN_ON(!list_empty(&pool->worklist))) in put_unbound_pool()
3549 if (pool->id >= 0) in put_unbound_pool()
3550 idr_remove(&worker_pool_idr, pool->id); in put_unbound_pool()
3551 hash_del(&pool->hash_node); in put_unbound_pool()
3562 pool->flags |= POOL_MANAGER_ACTIVE; in put_unbound_pool()
3566 WARN_ON(pool->nr_workers || pool->nr_idle); in put_unbound_pool()
3567 raw_spin_unlock_irq(&pool->lock); in put_unbound_pool()
3570 if (!list_empty(&pool->workers)) in put_unbound_pool()
3571 pool->detach_completion = &detach_completion; in put_unbound_pool()
3574 if (pool->detach_completion) in put_unbound_pool()
3575 wait_for_completion(pool->detach_completion); in put_unbound_pool()
3578 del_timer_sync(&pool->idle_timer); in put_unbound_pool()
3579 del_timer_sync(&pool->mayday_timer); in put_unbound_pool()
3582 call_rcu(&pool->rcu, rcu_free_pool); in put_unbound_pool()
3586 * get_unbound_pool - get a worker_pool with the specified attributes
3610 if (wqattrs_equal(pool->attrs, attrs)) { in get_unbound_pool()
3611 pool->refcnt++; in get_unbound_pool()
3619 if (cpumask_subset(attrs->cpumask, in get_unbound_pool()
3632 lockdep_set_subclass(&pool->lock, 1); /* see put_pwq() */ in get_unbound_pool()
3633 copy_workqueue_attrs(pool->attrs, attrs); in get_unbound_pool()
3634 pool->node = target_node; in get_unbound_pool()
3640 pool->attrs->no_numa = false; in get_unbound_pool()
3650 hash_add(unbound_pool_hash, &pool->hash_node, hash); in get_unbound_pool()
3673 struct workqueue_struct *wq = pwq->wq; in pwq_unbound_release_workfn()
3674 struct worker_pool *pool = pwq->pool; in pwq_unbound_release_workfn()
3677 if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND))) in pwq_unbound_release_workfn()
3680 mutex_lock(&wq->mutex); in pwq_unbound_release_workfn()
3681 list_del_rcu(&pwq->pwqs_node); in pwq_unbound_release_workfn()
3682 is_last = list_empty(&wq->pwqs); in pwq_unbound_release_workfn()
3683 mutex_unlock(&wq->mutex); in pwq_unbound_release_workfn()
3689 call_rcu(&pwq->rcu, rcu_free_pwq); in pwq_unbound_release_workfn()
3697 call_rcu(&wq->rcu, rcu_free_wq); in pwq_unbound_release_workfn()
3702 * pwq_adjust_max_active - update a pwq's max_active to the current setting
3705 * If @pwq isn't freezing, set @pwq->max_active to the associated
3707 * accordingly. If @pwq is freezing, clear @pwq->max_active to zero.
3711 struct workqueue_struct *wq = pwq->wq; in pwq_adjust_max_active()
3712 bool freezable = wq->flags & WQ_FREEZABLE; in pwq_adjust_max_active()
3715 /* for @wq->saved_max_active */ in pwq_adjust_max_active()
3716 lockdep_assert_held(&wq->mutex); in pwq_adjust_max_active()
3718 /* fast exit for non-freezable wqs */ in pwq_adjust_max_active()
3719 if (!freezable && pwq->max_active == wq->saved_max_active) in pwq_adjust_max_active()
3723 raw_spin_lock_irqsave(&pwq->pool->lock, flags); in pwq_adjust_max_active()
3731 pwq->max_active = wq->saved_max_active; in pwq_adjust_max_active()
3733 while (!list_empty(&pwq->delayed_works) && in pwq_adjust_max_active()
3734 pwq->nr_active < pwq->max_active) in pwq_adjust_max_active()
3741 wake_up_worker(pwq->pool); in pwq_adjust_max_active()
3743 pwq->max_active = 0; in pwq_adjust_max_active()
3746 raw_spin_unlock_irqrestore(&pwq->pool->lock, flags); in pwq_adjust_max_active()
3757 pwq->pool = pool; in init_pwq()
3758 pwq->wq = wq; in init_pwq()
3759 pwq->flush_color = -1; in init_pwq()
3760 pwq->refcnt = 1; in init_pwq()
3761 INIT_LIST_HEAD(&pwq->delayed_works); in init_pwq()
3762 INIT_LIST_HEAD(&pwq->pwqs_node); in init_pwq()
3763 INIT_LIST_HEAD(&pwq->mayday_node); in init_pwq()
3764 INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn); in init_pwq()
3770 struct workqueue_struct *wq = pwq->wq; in link_pwq()
3772 lockdep_assert_held(&wq->mutex); in link_pwq()
3775 if (!list_empty(&pwq->pwqs_node)) in link_pwq()
3779 pwq->work_color = wq->work_color; in link_pwq()
3785 list_add_rcu(&pwq->pwqs_node, &wq->pwqs); in link_pwq()
3801 pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node); in alloc_unbound_pwq()
3812 * wq_calc_node_cpumask - calculate a wq_attrs' cpumask for the specified node
3822 * If NUMA affinity is not enabled, @attrs->cpumask is always used. If
3825 * @attrs->cpumask.
3830 * Return: %true if the resulting @cpumask is different from @attrs->cpumask,
3836 if (!wq_numa_enabled || attrs->no_numa) in wq_calc_node_cpumask()
3840 cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask); in wq_calc_node_cpumask()
3848 cpumask_and(cpumask, attrs->cpumask, wq_numa_possible_cpumask[node]); in wq_calc_node_cpumask()
3856 return !cpumask_equal(cpumask, attrs->cpumask); in wq_calc_node_cpumask()
3859 cpumask_copy(cpumask, attrs->cpumask); in wq_calc_node_cpumask()
3871 lockdep_assert_held(&wq->mutex); in numa_pwq_tbl_install()
3876 old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]); in numa_pwq_tbl_install()
3877 rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq); in numa_pwq_tbl_install()
3897 put_pwq_unlocked(ctx->pwq_tbl[node]); in apply_wqattrs_cleanup()
3898 put_pwq_unlocked(ctx->dfl_pwq); in apply_wqattrs_cleanup()
3900 free_workqueue_attrs(ctx->attrs); in apply_wqattrs_cleanup()
3930 cpumask_and(new_attrs->cpumask, new_attrs->cpumask, wq_unbound_cpumask); in apply_wqattrs_prepare()
3931 if (unlikely(cpumask_empty(new_attrs->cpumask))) in apply_wqattrs_prepare()
3932 cpumask_copy(new_attrs->cpumask, wq_unbound_cpumask); in apply_wqattrs_prepare()
3943 * the default pwq covering whole @attrs->cpumask. Always create in apply_wqattrs_prepare()
3946 ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs); in apply_wqattrs_prepare()
3947 if (!ctx->dfl_pwq) in apply_wqattrs_prepare()
3951 if (wq_calc_node_cpumask(new_attrs, node, -1, tmp_attrs->cpumask)) { in apply_wqattrs_prepare()
3952 ctx->pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs); in apply_wqattrs_prepare()
3953 if (!ctx->pwq_tbl[node]) in apply_wqattrs_prepare()
3956 ctx->dfl_pwq->refcnt++; in apply_wqattrs_prepare()
3957 ctx->pwq_tbl[node] = ctx->dfl_pwq; in apply_wqattrs_prepare()
3963 cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask); in apply_wqattrs_prepare()
3964 ctx->attrs = new_attrs; in apply_wqattrs_prepare()
3966 ctx->wq = wq; in apply_wqattrs_prepare()
3983 mutex_lock(&ctx->wq->mutex); in apply_wqattrs_commit()
3985 copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs); in apply_wqattrs_commit()
3989 ctx->pwq_tbl[node] = numa_pwq_tbl_install(ctx->wq, node, in apply_wqattrs_commit()
3990 ctx->pwq_tbl[node]); in apply_wqattrs_commit()
3993 link_pwq(ctx->dfl_pwq); in apply_wqattrs_commit()
3994 swap(ctx->wq->dfl_pwq, ctx->dfl_pwq); in apply_wqattrs_commit()
3996 mutex_unlock(&ctx->wq->mutex); in apply_wqattrs_commit()
4018 if (WARN_ON(!(wq->flags & WQ_UNBOUND))) in apply_workqueue_attrs_locked()
4019 return -EINVAL; in apply_workqueue_attrs_locked()
4022 if (!list_empty(&wq->pwqs)) { in apply_workqueue_attrs_locked()
4023 if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT)) in apply_workqueue_attrs_locked()
4024 return -EINVAL; in apply_workqueue_attrs_locked()
4026 wq->flags &= ~__WQ_ORDERED; in apply_workqueue_attrs_locked()
4031 return -ENOMEM; in apply_workqueue_attrs_locked()
4041 * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
4047 * possibles CPUs in @attrs->cpumask so that work items are affine to the
4048 * NUMA node it was issued on. Older pwqs are released as in-flight work
4050 * back-to-back will stay on its current pwq.
4056 * Return: 0 on success and -errno on failure.
4073 * wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug
4083 * falls back to @wq->dfl_pwq which may not be optimal but is always
4089 * affinity and may execute on any CPU. This is similar to how per-cpu
4098 int cpu_off = online ? -1 : cpu; in wq_update_unbound_numa()
4105 if (!wq_numa_enabled || !(wq->flags & WQ_UNBOUND) || in wq_update_unbound_numa()
4106 wq->unbound_attrs->no_numa) in wq_update_unbound_numa()
4115 cpumask = target_attrs->cpumask; in wq_update_unbound_numa()
4117 copy_workqueue_attrs(target_attrs, wq->unbound_attrs); in wq_update_unbound_numa()
4126 if (wq_calc_node_cpumask(wq->dfl_pwq->pool->attrs, node, cpu_off, cpumask)) { in wq_update_unbound_numa()
4127 if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask)) in wq_update_unbound_numa()
4137 wq->name); in wq_update_unbound_numa()
4142 mutex_lock(&wq->mutex); in wq_update_unbound_numa()
4147 mutex_lock(&wq->mutex); in wq_update_unbound_numa()
4148 raw_spin_lock_irq(&wq->dfl_pwq->pool->lock); in wq_update_unbound_numa()
4149 get_pwq(wq->dfl_pwq); in wq_update_unbound_numa()
4150 raw_spin_unlock_irq(&wq->dfl_pwq->pool->lock); in wq_update_unbound_numa()
4151 old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq); in wq_update_unbound_numa()
4153 mutex_unlock(&wq->mutex); in wq_update_unbound_numa()
4159 bool highpri = wq->flags & WQ_HIGHPRI; in alloc_and_link_pwqs()
4162 if (!(wq->flags & WQ_UNBOUND)) { in alloc_and_link_pwqs()
4163 wq->cpu_pwqs = alloc_percpu(struct pool_workqueue); in alloc_and_link_pwqs()
4164 if (!wq->cpu_pwqs) in alloc_and_link_pwqs()
4165 return -ENOMEM; in alloc_and_link_pwqs()
4169 per_cpu_ptr(wq->cpu_pwqs, cpu); in alloc_and_link_pwqs()
4175 mutex_lock(&wq->mutex); in alloc_and_link_pwqs()
4177 mutex_unlock(&wq->mutex); in alloc_and_link_pwqs()
4183 if (wq->flags & __WQ_ORDERED) { in alloc_and_link_pwqs()
4186 WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node || in alloc_and_link_pwqs()
4187 wq->pwqs.prev != &wq->dfl_pwq->pwqs_node), in alloc_and_link_pwqs()
4188 "ordering guarantee broken for workqueue %s\n", wq->name); in alloc_and_link_pwqs()
4218 if (!(wq->flags & WQ_MEM_RECLAIM)) in init_rescuer()
4223 return -ENOMEM; in init_rescuer()
4225 rescuer->rescue_wq = wq; in init_rescuer()
4226 rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", wq->name); in init_rescuer()
4227 if (IS_ERR(rescuer->task)) { in init_rescuer()
4228 ret = PTR_ERR(rescuer->task); in init_rescuer()
4233 wq->rescuer = rescuer; in init_rescuer()
4234 kthread_bind_mask(rescuer->task, cpu_possible_mask); in init_rescuer()
4235 wake_up_process(rescuer->task); in init_rescuer()
4252 * longer the case on NUMA machines due to per-node pools. While in alloc_workqueue()
4266 tbl_size = nr_node_ids * sizeof(wq->numa_pwq_tbl[0]); in alloc_workqueue()
4273 wq->unbound_attrs = alloc_workqueue_attrs(); in alloc_workqueue()
4274 if (!wq->unbound_attrs) in alloc_workqueue()
4279 vsnprintf(wq->name, sizeof(wq->name), fmt, args); in alloc_workqueue()
4283 max_active = wq_clamp_max_active(max_active, flags, wq->name); in alloc_workqueue()
4286 wq->flags = flags; in alloc_workqueue()
4287 wq->saved_max_active = max_active; in alloc_workqueue()
4288 mutex_init(&wq->mutex); in alloc_workqueue()
4289 atomic_set(&wq->nr_pwqs_to_flush, 0); in alloc_workqueue()
4290 INIT_LIST_HEAD(&wq->pwqs); in alloc_workqueue()
4291 INIT_LIST_HEAD(&wq->flusher_queue); in alloc_workqueue()
4292 INIT_LIST_HEAD(&wq->flusher_overflow); in alloc_workqueue()
4293 INIT_LIST_HEAD(&wq->maydays); in alloc_workqueue()
4296 INIT_LIST_HEAD(&wq->list); in alloc_workqueue()
4304 if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq)) in alloc_workqueue()
4314 mutex_lock(&wq->mutex); in alloc_workqueue()
4317 mutex_unlock(&wq->mutex); in alloc_workqueue()
4319 list_add_tail_rcu(&wq->list, &workqueues); in alloc_workqueue()
4329 free_workqueue_attrs(wq->unbound_attrs); in alloc_workqueue()
4343 if (pwq->nr_in_flight[i]) in pwq_busy()
4346 if ((pwq != pwq->wq->dfl_pwq) && (pwq->refcnt > 1)) in pwq_busy()
4348 if (pwq->nr_active || !list_empty(&pwq->delayed_works)) in pwq_busy()
4355 * destroy_workqueue - safely terminate a workqueue
4375 if (wq->rescuer) { in destroy_workqueue()
4376 struct worker *rescuer = wq->rescuer; in destroy_workqueue()
4380 wq->rescuer = NULL; in destroy_workqueue()
4384 kthread_stop(rescuer->task); in destroy_workqueue()
4389 * Sanity checks - grab all the locks so that we wait for all in destroy_workqueue()
4390 * in-flight operations which may do put_pwq(). in destroy_workqueue()
4393 mutex_lock(&wq->mutex); in destroy_workqueue()
4395 raw_spin_lock_irq(&pwq->pool->lock); in destroy_workqueue()
4398 __func__, wq->name); in destroy_workqueue()
4400 raw_spin_unlock_irq(&pwq->pool->lock); in destroy_workqueue()
4401 mutex_unlock(&wq->mutex); in destroy_workqueue()
4406 raw_spin_unlock_irq(&pwq->pool->lock); in destroy_workqueue()
4408 mutex_unlock(&wq->mutex); in destroy_workqueue()
4414 list_del_rcu(&wq->list); in destroy_workqueue()
4417 if (!(wq->flags & WQ_UNBOUND)) { in destroy_workqueue()
4420 * The base ref is never dropped on per-cpu pwqs. Directly in destroy_workqueue()
4423 call_rcu(&wq->rcu, rcu_free_wq); in destroy_workqueue()
4431 pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]); in destroy_workqueue()
4432 RCU_INIT_POINTER(wq->numa_pwq_tbl[node], NULL); in destroy_workqueue()
4440 pwq = wq->dfl_pwq; in destroy_workqueue()
4441 wq->dfl_pwq = NULL; in destroy_workqueue()
4448 * workqueue_set_max_active - adjust max_active of a workqueue
4462 if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT)) in workqueue_set_max_active()
4465 max_active = wq_clamp_max_active(max_active, wq->flags, wq->name); in workqueue_set_max_active()
4467 mutex_lock(&wq->mutex); in workqueue_set_max_active()
4469 wq->flags &= ~__WQ_ORDERED; in workqueue_set_max_active()
4470 wq->saved_max_active = max_active; in workqueue_set_max_active()
4475 mutex_unlock(&wq->mutex); in workqueue_set_max_active()
4480 * current_work - retrieve %current task's work struct
4491 return worker ? worker->current_work : NULL; in current_work()
4496 * current_is_workqueue_rescuer - is %current workqueue rescuer?
4507 return worker && worker->rescue_wq; in current_is_workqueue_rescuer()
4511 * workqueue_congested - test whether a workqueue is congested
4520 * Note that both per-cpu and unbound workqueues may be associated with
4539 if (!(wq->flags & WQ_UNBOUND)) in workqueue_congested()
4540 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu); in workqueue_congested()
4544 ret = !list_empty(&pwq->delayed_works); in workqueue_congested()
4553 * work_busy - test whether a work is currently pending or running
4575 raw_spin_lock_irqsave(&pool->lock, flags); in work_busy()
4578 raw_spin_unlock_irqrestore(&pool->lock, flags); in work_busy()
4587 * set_worker_desc - set description for the current work item
4588 * @fmt: printf-style format string
4603 vsnprintf(worker->desc, sizeof(worker->desc), fmt, args); in set_worker_desc()
4610 * print_worker_info - print out worker information and description
4631 if (!(task->flags & PF_WQ_WORKER)) in print_worker_info()
4644 copy_from_kernel_nofault(&fn, &worker->current_func, sizeof(fn)); in print_worker_info()
4645 copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq)); in print_worker_info()
4646 copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq)); in print_worker_info()
4647 copy_from_kernel_nofault(name, wq->name, sizeof(name) - 1); in print_worker_info()
4648 copy_from_kernel_nofault(desc, worker->desc, sizeof(desc) - 1); in print_worker_info()
4660 pr_cont(" cpus=%*pbl", nr_cpumask_bits, pool->attrs->cpumask); in pr_cont_pool_info()
4661 if (pool->node != NUMA_NO_NODE) in pr_cont_pool_info()
4662 pr_cont(" node=%d", pool->node); in pr_cont_pool_info()
4663 pr_cont(" flags=0x%x nice=%d", pool->flags, pool->attrs->nice); in pr_cont_pool_info()
4668 if (work->func == wq_barrier_func) { in pr_cont_work()
4674 task_pid_nr(barr->task)); in pr_cont_work()
4676 pr_cont("%s %ps", comma ? "," : "", work->func); in pr_cont_work()
4682 struct worker_pool *pool = pwq->pool; in show_pwq()
4688 pr_info(" pwq %d:", pool->id); in show_pwq()
4692 pwq->nr_active, pwq->max_active, pwq->refcnt, in show_pwq()
4693 !list_empty(&pwq->mayday_node) ? " MAYDAY" : ""); in show_pwq()
4695 hash_for_each(pool->busy_hash, bkt, worker, hentry) { in show_pwq()
4696 if (worker->current_pwq == pwq) { in show_pwq()
4704 pr_info(" in-flight:"); in show_pwq()
4705 hash_for_each(pool->busy_hash, bkt, worker, hentry) { in show_pwq()
4706 if (worker->current_pwq != pwq) in show_pwq()
4710 task_pid_nr(worker->task), in show_pwq()
4711 worker->rescue_wq ? "(RESCUER)" : "", in show_pwq()
4712 worker->current_func); in show_pwq()
4713 list_for_each_entry(work, &worker->scheduled, entry) in show_pwq()
4720 list_for_each_entry(work, &pool->worklist, entry) { in show_pwq()
4730 list_for_each_entry(work, &pool->worklist, entry) { in show_pwq()
4740 if (!list_empty(&pwq->delayed_works)) { in show_pwq()
4744 list_for_each_entry(work, &pwq->delayed_works, entry) { in show_pwq()
4753 * show_workqueue_state - dump workqueue state
4774 if (pwq->nr_active || !list_empty(&pwq->delayed_works)) { in show_workqueue_state()
4782 pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags); in show_workqueue_state()
4785 raw_spin_lock_irqsave(&pwq->pool->lock, flags); in show_workqueue_state()
4786 if (pwq->nr_active || !list_empty(&pwq->delayed_works)) in show_workqueue_state()
4788 raw_spin_unlock_irqrestore(&pwq->pool->lock, flags); in show_workqueue_state()
4791 * sysrq-t -> show_workqueue_state(). Avoid triggering in show_workqueue_state()
4802 raw_spin_lock_irqsave(&pool->lock, flags); in show_workqueue_state()
4803 if (pool->nr_workers == pool->nr_idle) in show_workqueue_state()
4806 pr_info("pool %d:", pool->id); in show_workqueue_state()
4809 jiffies_to_msecs(jiffies - pool->watchdog_ts) / 1000, in show_workqueue_state()
4810 pool->nr_workers); in show_workqueue_state()
4811 if (pool->manager) in show_workqueue_state()
4813 task_pid_nr(pool->manager->task)); in show_workqueue_state()
4814 list_for_each_entry(worker, &pool->idle_list, entry) { in show_workqueue_state()
4816 task_pid_nr(worker->task)); in show_workqueue_state()
4821 raw_spin_unlock_irqrestore(&pool->lock, flags); in show_workqueue_state()
4824 * sysrq-t -> show_workqueue_state(). Avoid triggering in show_workqueue_state()
4839 off = strscpy(buf, task->comm, size); in wq_worker_comm()
4846 if (task->flags & PF_WQ_WORKER) { in wq_worker_comm()
4848 struct worker_pool *pool = worker->pool; in wq_worker_comm()
4851 raw_spin_lock_irq(&pool->lock); in wq_worker_comm()
4853 * ->desc tracks information (wq name or in wq_worker_comm()
4855 * current, prepend '+', otherwise '-'. in wq_worker_comm()
4857 if (worker->desc[0] != '\0') { in wq_worker_comm()
4858 if (worker->current_work) in wq_worker_comm()
4859 scnprintf(buf + off, size - off, "+%s", in wq_worker_comm()
4860 worker->desc); in wq_worker_comm()
4862 scnprintf(buf + off, size - off, "-%s", in wq_worker_comm()
4863 worker->desc); in wq_worker_comm()
4865 raw_spin_unlock_irq(&pool->lock); in wq_worker_comm()
4896 raw_spin_lock_irq(&pool->lock); in unbind_workers()
4906 worker->flags |= WORKER_UNBOUND; in unbind_workers()
4908 pool->flags |= POOL_DISASSOCIATED; in unbind_workers()
4910 raw_spin_unlock_irq(&pool->lock); in unbind_workers()
4914 * Call schedule() so that we cross rq->lock and thus can in unbind_workers()
4929 atomic_set(&pool->nr_running, 0); in unbind_workers()
4936 raw_spin_lock_irq(&pool->lock); in unbind_workers()
4938 raw_spin_unlock_irq(&pool->lock); in unbind_workers()
4943 * rebind_workers - rebind all workers of a pool to the associated CPU
4946 * @pool->cpu is coming online. Rebind all workers to the CPU.
4956 * be on the run-queue of the associated CPU before any local in rebind_workers()
4957 * wake-ups for concurrency management happen, restore CPU affinity in rebind_workers()
4962 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, in rebind_workers()
4963 pool->attrs->cpumask) < 0); in rebind_workers()
4965 raw_spin_lock_irq(&pool->lock); in rebind_workers()
4967 pool->flags &= ~POOL_DISASSOCIATED; in rebind_workers()
4970 unsigned int worker_flags = worker->flags; in rebind_workers()
4974 * of the associated CPU for local wake-ups targeting it to in rebind_workers()
4978 * be bound before @pool->lock is released. in rebind_workers()
4981 wake_up_process(worker->task); in rebind_workers()
4988 * it initiates the next execution cycle thus restoring in rebind_workers()
4992 * WRITE_ONCE() is necessary because @worker->flags may be in rebind_workers()
5001 WRITE_ONCE(worker->flags, worker_flags); in rebind_workers()
5004 raw_spin_unlock_irq(&pool->lock); in rebind_workers()
5008 * restore_unbound_workers_cpumask - restore cpumask of unbound workers
5025 if (!cpumask_test_cpu(cpu, pool->attrs->cpumask)) in restore_unbound_workers_cpumask()
5028 cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask); in restore_unbound_workers_cpumask()
5032 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, &cpumask) < 0); in restore_unbound_workers_cpumask()
5040 if (pool->nr_workers) in workqueue_prepare_cpu()
5043 return -ENOMEM; in workqueue_prepare_cpu()
5059 if (pool->cpu == cpu) in workqueue_online_cpu()
5061 else if (pool->cpu < 0) in workqueue_online_cpu()
5079 /* unbinding per-cpu workers should happen on the local CPU */ in workqueue_offline_cpu()
5081 return -1; in workqueue_offline_cpu()
5105 wfc->ret = wfc->fn(wfc->arg); in work_for_cpu_fn()
5109 * work_on_cpu - run a function in thread context on a particular cpu
5132 * work_on_cpu_safe - run a function in thread context on a particular cpu
5144 long ret = -ENODEV; in work_on_cpu_safe()
5158 * freeze_workqueues_begin - begin freezing workqueues
5162 * pool->worklist.
5165 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
5178 mutex_lock(&wq->mutex); in freeze_workqueues_begin()
5181 mutex_unlock(&wq->mutex); in freeze_workqueues_begin()
5188 * freeze_workqueues_busy - are freezable workqueues still busy?
5211 if (!(wq->flags & WQ_FREEZABLE)) in freeze_workqueues_busy()
5219 WARN_ON_ONCE(pwq->nr_active < 0); in freeze_workqueues_busy()
5220 if (pwq->nr_active) { in freeze_workqueues_busy()
5234 * thaw_workqueues - thaw workqueues
5240 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
5256 mutex_lock(&wq->mutex); in thaw_workqueues()
5259 mutex_unlock(&wq->mutex); in thaw_workqueues()
5277 if (!(wq->flags & WQ_UNBOUND)) in workqueue_apply_unbound_cpumask()
5280 if (wq->flags & __WQ_ORDERED) in workqueue_apply_unbound_cpumask()
5283 ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs); in workqueue_apply_unbound_cpumask()
5285 ret = -ENOMEM; in workqueue_apply_unbound_cpumask()
5289 list_add_tail(&ctx->list, &ctxs); in workqueue_apply_unbound_cpumask()
5302 * workqueue_set_unbound_cpumask - Set the low-level unbound cpumask
5305 * The low-level workqueues cpumask is a global cpumask that limits
5309 * Retun: 0 - Success
5310 * -EINVAL - Invalid @cpumask
5311 * -ENOMEM - Failed to allocate memory for attrs or pwqs.
5315 int ret = -EINVAL; in workqueue_set_unbound_cpumask()
5319 return -ENOMEM; in workqueue_set_unbound_cpumask()
5353 * per_cpu RO bool : whether the workqueue is per-cpu or unbound
5354 * max_active RW int : maximum number of in-flight work items
5372 return wq_dev->wq; in dev_to_wq()
5380 return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND)); in per_cpu_show()
5389 return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active); in max_active_show()
5400 return -EINVAL; in max_active_store()
5424 written += scnprintf(buf + written, PAGE_SIZE - written, in wq_pool_ids_show()
5426 unbound_pwq_by_node(wq, node)->pool->id); in wq_pool_ids_show()
5429 written += scnprintf(buf + written, PAGE_SIZE - written, "\n"); in wq_pool_ids_show()
5442 mutex_lock(&wq->mutex); in wq_nice_show()
5443 written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice); in wq_nice_show()
5444 mutex_unlock(&wq->mutex); in wq_nice_show()
5460 copy_workqueue_attrs(attrs, wq->unbound_attrs); in wq_sysfs_prep_attrs()
5469 int ret = -ENOMEM; in wq_nice_store()
5477 if (sscanf(buf, "%d", &attrs->nice) == 1 && in wq_nice_store()
5478 attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE) in wq_nice_store()
5481 ret = -EINVAL; in wq_nice_store()
5495 mutex_lock(&wq->mutex); in wq_cpumask_show()
5497 cpumask_pr_args(wq->unbound_attrs->cpumask)); in wq_cpumask_show()
5498 mutex_unlock(&wq->mutex); in wq_cpumask_show()
5508 int ret = -ENOMEM; in wq_cpumask_store()
5516 ret = cpumask_parse(buf, attrs->cpumask); in wq_cpumask_store()
5532 mutex_lock(&wq->mutex); in wq_numa_show()
5534 !wq->unbound_attrs->no_numa); in wq_numa_show()
5535 mutex_unlock(&wq->mutex); in wq_numa_show()
5545 int v, ret = -ENOMEM; in wq_numa_store()
5553 ret = -EINVAL; in wq_numa_store()
5555 attrs->no_numa = !v; in wq_numa_store()
5598 return -ENOMEM; in wq_unbound_cpumask_store()
5632 * workqueue_sysfs_register - make a workqueue visible in sysfs
5644 * Return: 0 on success, -errno on failure.
5656 if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT)) in workqueue_sysfs_register()
5657 return -EINVAL; in workqueue_sysfs_register()
5659 wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL); in workqueue_sysfs_register()
5661 return -ENOMEM; in workqueue_sysfs_register()
5663 wq_dev->wq = wq; in workqueue_sysfs_register()
5664 wq_dev->dev.bus = &wq_subsys; in workqueue_sysfs_register()
5665 wq_dev->dev.release = wq_device_release; in workqueue_sysfs_register()
5666 dev_set_name(&wq_dev->dev, "%s", wq->name); in workqueue_sysfs_register()
5672 dev_set_uevent_suppress(&wq_dev->dev, true); in workqueue_sysfs_register()
5674 ret = device_register(&wq_dev->dev); in workqueue_sysfs_register()
5676 put_device(&wq_dev->dev); in workqueue_sysfs_register()
5677 wq->wq_dev = NULL; in workqueue_sysfs_register()
5681 if (wq->flags & WQ_UNBOUND) { in workqueue_sysfs_register()
5684 for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) { in workqueue_sysfs_register()
5685 ret = device_create_file(&wq_dev->dev, attr); in workqueue_sysfs_register()
5687 device_unregister(&wq_dev->dev); in workqueue_sysfs_register()
5688 wq->wq_dev = NULL; in workqueue_sysfs_register()
5694 dev_set_uevent_suppress(&wq_dev->dev, false); in workqueue_sysfs_register()
5695 kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD); in workqueue_sysfs_register()
5700 * workqueue_sysfs_unregister - undo workqueue_sysfs_register()
5707 struct wq_device *wq_dev = wq->wq_dev; in workqueue_sysfs_unregister()
5709 if (!wq->wq_dev) in workqueue_sysfs_unregister()
5712 wq->wq_dev = NULL; in workqueue_sysfs_unregister()
5713 device_unregister(&wq_dev->dev); in workqueue_sysfs_unregister()
5722 * Stall may be caused by various bugs - missing WQ_MEM_RECLAIM, illegal
5730 * forward progress is defined as the first item on ->worklist changing.
5768 if (list_empty(&pool->worklist)) in wq_watchdog_timer_fn()
5772 pool_ts = READ_ONCE(pool->watchdog_ts); in wq_watchdog_timer_fn()
5780 if (pool->cpu >= 0) { in wq_watchdog_timer_fn()
5783 pool->cpu)); in wq_watchdog_timer_fn()
5791 pr_emerg("BUG: workqueue lockup - pool"); in wq_watchdog_timer_fn()
5794 jiffies_to_msecs(jiffies - pool_ts) / 1000); in wq_watchdog_timer_fn()
5908 * workqueue_init_early - early init for workqueue subsystem
5910 * This is the first half of two-staged workqueue subsystem initialization
5911 * and invoked as soon as the bare basics - memory allocation, cpumasks and
5937 pool->cpu = cpu; in workqueue_init_early()
5938 cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu)); in workqueue_init_early()
5939 pool->attrs->nice = std_nice[i++]; in workqueue_init_early()
5940 pool->node = cpu_to_node(cpu); in workqueue_init_early()
5954 attrs->nice = std_nice[i]; in workqueue_init_early()
5963 attrs->nice = std_nice[i]; in workqueue_init_early()
5964 attrs->no_numa = true; in workqueue_init_early()
5987 * workqueue_init - bring workqueue subsystem fully online
5989 * This is the latter half of two-staged workqueue subsystem initialization
6004 * archs such as power and arm64. As per-cpu pools created in workqueue_init()
6016 pool->node = cpu_to_node(cpu); in workqueue_init()
6024 wq->name); in workqueue_init()
6032 pool->flags &= ~POOL_DISASSOCIATED; in workqueue_init()