11  * See the COPYING.LIB file in the top-level directory.
20 #include "qemu/coroutine-tls.h"
29  * Coroutine creation and deletion is expensive so a pool of unused coroutines
30  * is kept as a cache. When the pool has coroutines available, they are
32  * the pool upon termination.
34  * The pool is global but each thread maintains a small local pool to avoid
35  * global pool contention. Threads fetch and return batches of coroutines from
36  * the global pool to maintain their local pool. The local pool holds up to two
37  * batches whereas the maximum size of the global pool is controlled by the
40  * .-----------------------------------.
42  * `-----------------------------------'
44  * .-------------------.
45  * | Batch 1 | Batch 2 | per-thread local_pool (maximum 2 batches)
46  * `-------------------'
53     QSLIST_HEAD(, Coroutine) list;
74     QSLIST_INIT(&batch->list);  in coroutine_pool_batch_new()
75     batch->size = 0;  in coroutine_pool_batch_new()
81     Coroutine *co;  in coroutine_pool_batch_delete()
82     Coroutine *tmp;  in coroutine_pool_batch_delete()
84     QSLIST_FOREACH_SAFE(co, &batch->list, pool_next, tmp) {  in coroutine_pool_batch_delete()
85         QSLIST_REMOVE_HEAD(&batch->list, pool_next);  in coroutine_pool_batch_delete()
107     if (!notifier->notify) {  in local_pool_cleanup_init_once()
108         notifier->notify = local_pool_cleanup;  in local_pool_cleanup_init_once()
113 /* Helper to get the next unused coroutine from the local pool */
114 static Coroutine *coroutine_pool_get_local(void)  in coroutine_pool_get_local()
118     Coroutine *co;  in coroutine_pool_get_local()
124     co = QSLIST_FIRST(&batch->list);  in coroutine_pool_get_local()
125     QSLIST_REMOVE_HEAD(&batch->list, pool_next);  in coroutine_pool_get_local()
126     batch->size--;  in coroutine_pool_get_local()
128     if (batch->size == 0) {  in coroutine_pool_get_local()
135 /* Get the next batch from the global pool */
146             global_pool_size -= batch->size;  in coroutine_pool_refill_local()
156 /* Add a batch of coroutines to the global pool */
166             /* Overshooting the max pool size is allowed */  in coroutine_pool_put_global()
167             global_pool_size += batch->size;  in coroutine_pool_put_global()
172     /* The global pool was full, so throw away this batch */  in coroutine_pool_put_global()
176 /* Get the next unused coroutine from the pool or return NULL */
177 static Coroutine *coroutine_pool_get(void)  in coroutine_pool_get()
179     Coroutine *co;  in coroutine_pool_get()
189 static void coroutine_pool_put(Coroutine *co)  in coroutine_pool_put()
200     if (unlikely(batch->size >= COROUTINE_POOL_BATCH_MAX_SIZE)) {  in coroutine_pool_put()
203         /* Is the local pool full? */  in coroutine_pool_put()
213     QSLIST_INSERT_HEAD(&batch->list, co, pool_next);  in coroutine_pool_put()
214     batch->size++;  in coroutine_pool_put()
217 Coroutine *qemu_coroutine_create(CoroutineEntry *entry, void *opaque)  in qemu_coroutine_create()
219     Coroutine *co = NULL;  in qemu_coroutine_create()
229     co->entry = entry;  in qemu_coroutine_create()
230     co->entry_arg = opaque;  in qemu_coroutine_create()
231     QSIMPLEQ_INIT(&co->co_queue_wakeup);  in qemu_coroutine_create()
235 static void coroutine_delete(Coroutine *co)  in coroutine_delete()
237     co->caller = NULL;  in coroutine_delete()
246 void qemu_aio_coroutine_enter(AioContext *ctx, Coroutine *co)  in qemu_aio_coroutine_enter()
248     QSIMPLEQ_HEAD(, Coroutine) pending = QSIMPLEQ_HEAD_INITIALIZER(pending);  in qemu_aio_coroutine_enter()
249     Coroutine *from = qemu_coroutine_self();  in qemu_aio_coroutine_enter()
255         Coroutine *to = QSIMPLEQ_FIRST(&pending);  in qemu_aio_coroutine_enter()
259          * Read to before to->scheduled; pairs with qatomic_cmpxchg in  in qemu_aio_coroutine_enter()
264         const char *scheduled = qatomic_read(&to->scheduled);  in qemu_aio_coroutine_enter()
268         trace_qemu_aio_coroutine_enter(ctx, from, to, to->entry_arg);  in qemu_aio_coroutine_enter()
270         /* if the Coroutine has already been scheduled, entering it again will  in qemu_aio_coroutine_enter()
271          * cause us to enter it twice, potentially even after the coroutine has  in qemu_aio_coroutine_enter()
275                     "%s: Co-routine was already scheduled in '%s'\n",  in qemu_aio_coroutine_enter()
280         if (to->caller) {  in qemu_aio_coroutine_enter()
281             fprintf(stderr, "Co-routine re-entered recursively\n");  in qemu_aio_coroutine_enter()
285         to->caller = from;  in qemu_aio_coroutine_enter()
286         to->ctx = ctx;  in qemu_aio_coroutine_enter()
288         /* Store to->ctx before anything that stores to.  Matches  in qemu_aio_coroutine_enter()
295         /* Queued coroutines are run depth-first; previously pending coroutines  in qemu_aio_coroutine_enter()
298         QSIMPLEQ_PREPEND(&pending, &to->co_queue_wakeup);  in qemu_aio_coroutine_enter()
304             assert(!to->locks_held);  in qemu_aio_coroutine_enter()
314 void qemu_coroutine_enter(Coroutine *co)  in qemu_coroutine_enter()
319 void qemu_coroutine_enter_if_inactive(Coroutine *co)  in qemu_coroutine_enter_if_inactive()
328     Coroutine *self = qemu_coroutine_self();  in qemu_coroutine_yield()
329     Coroutine *to = self->caller;  in qemu_coroutine_yield()
334         fprintf(stderr, "Co-routine is yielding to no one\n");  in qemu_coroutine_yield()
338     self->caller = NULL;  in qemu_coroutine_yield()
342 bool qemu_coroutine_entered(Coroutine *co)  in qemu_coroutine_entered()
344     return co->caller;  in qemu_coroutine_entered()
347 AioContext *qemu_coroutine_get_aio_context(Coroutine *co)  in qemu_coroutine_get_aio_context()
349     return co->ctx;  in qemu_coroutine_get_aio_context()
361     global_pool_max_size -= removing_pool_size;  in qemu_coroutine_dec_pool_size()
373      * must limit the coroutine pool to a safe size to avoid running out of  in get_global_pool_hard_max_size()
381          * fixed amount for non-coroutine users like library dependencies,  in get_global_pool_hard_max_size()
382          * vhost-user, etc. Each coroutine takes up 2 VMAs so halve the  in get_global_pool_hard_max_size()
386             return (max_map_count - 5000) / 2;  in get_global_pool_hard_max_size()
388             /* Disable the global pool but threads still have local pools */  in get_global_pool_hard_max_size()