xref: /qemu/block/io.c (revision 85c97ca7a10b93216bc95052e9dabe3a4bb8736a)
161007b31SStefan Hajnoczi /*
261007b31SStefan Hajnoczi  * Block layer I/O functions
361007b31SStefan Hajnoczi  *
461007b31SStefan Hajnoczi  * Copyright (c) 2003 Fabrice Bellard
561007b31SStefan Hajnoczi  *
661007b31SStefan Hajnoczi  * Permission is hereby granted, free of charge, to any person obtaining a copy
761007b31SStefan Hajnoczi  * of this software and associated documentation files (the "Software"), to deal
861007b31SStefan Hajnoczi  * in the Software without restriction, including without limitation the rights
961007b31SStefan Hajnoczi  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
1061007b31SStefan Hajnoczi  * copies of the Software, and to permit persons to whom the Software is
1161007b31SStefan Hajnoczi  * furnished to do so, subject to the following conditions:
1261007b31SStefan Hajnoczi  *
1361007b31SStefan Hajnoczi  * The above copyright notice and this permission notice shall be included in
1461007b31SStefan Hajnoczi  * all copies or substantial portions of the Software.
1561007b31SStefan Hajnoczi  *
1661007b31SStefan Hajnoczi  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
1761007b31SStefan Hajnoczi  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
1861007b31SStefan Hajnoczi  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
1961007b31SStefan Hajnoczi  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
2061007b31SStefan Hajnoczi  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
2161007b31SStefan Hajnoczi  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
2261007b31SStefan Hajnoczi  * THE SOFTWARE.
2361007b31SStefan Hajnoczi  */
2461007b31SStefan Hajnoczi 
2580c71a24SPeter Maydell #include "qemu/osdep.h"
2661007b31SStefan Hajnoczi #include "trace.h"
277f0e9da6SMax Reitz #include "sysemu/block-backend.h"
2861007b31SStefan Hajnoczi #include "block/blockjob.h"
2961007b31SStefan Hajnoczi #include "block/block_int.h"
30f348b6d1SVeronia Bahaa #include "qemu/cutils.h"
31da34e65cSMarkus Armbruster #include "qapi/error.h"
32d49b6836SMarkus Armbruster #include "qemu/error-report.h"
3361007b31SStefan Hajnoczi 
3461007b31SStefan Hajnoczi #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
3561007b31SStefan Hajnoczi 
36b15404e0SEric Blake static BlockAIOCB *bdrv_co_aio_prw_vector(BdrvChild *child,
37b15404e0SEric Blake                                           int64_t offset,
3861007b31SStefan Hajnoczi                                           QEMUIOVector *qiov,
3961007b31SStefan Hajnoczi                                           BdrvRequestFlags flags,
4061007b31SStefan Hajnoczi                                           BlockCompletionFunc *cb,
4161007b31SStefan Hajnoczi                                           void *opaque,
4261007b31SStefan Hajnoczi                                           bool is_write);
4361007b31SStefan Hajnoczi static void coroutine_fn bdrv_co_do_rw(void *opaque);
44d05aa8bbSEric Blake static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
45d05aa8bbSEric Blake     int64_t offset, int count, BdrvRequestFlags flags);
4661007b31SStefan Hajnoczi 
47c2066af0SKevin Wolf static void bdrv_parent_drained_begin(BlockDriverState *bs)
4861007b31SStefan Hajnoczi {
49c2066af0SKevin Wolf     BdrvChild *c;
5027ccdd52SKevin Wolf 
51c2066af0SKevin Wolf     QLIST_FOREACH(c, &bs->parents, next_parent) {
52c2066af0SKevin Wolf         if (c->role->drained_begin) {
53c2066af0SKevin Wolf             c->role->drained_begin(c);
54c2066af0SKevin Wolf         }
55ce0f1412SPaolo Bonzini     }
56ce0f1412SPaolo Bonzini }
57ce0f1412SPaolo Bonzini 
58c2066af0SKevin Wolf static void bdrv_parent_drained_end(BlockDriverState *bs)
59ce0f1412SPaolo Bonzini {
60c2066af0SKevin Wolf     BdrvChild *c;
6127ccdd52SKevin Wolf 
62c2066af0SKevin Wolf     QLIST_FOREACH(c, &bs->parents, next_parent) {
63c2066af0SKevin Wolf         if (c->role->drained_end) {
64c2066af0SKevin Wolf             c->role->drained_end(c);
6527ccdd52SKevin Wolf         }
66c2066af0SKevin Wolf     }
6761007b31SStefan Hajnoczi }
6861007b31SStefan Hajnoczi 
69d9e0dfa2SEric Blake static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src)
70d9e0dfa2SEric Blake {
71d9e0dfa2SEric Blake     dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer);
72d9e0dfa2SEric Blake     dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer);
73d9e0dfa2SEric Blake     dst->opt_mem_alignment = MAX(dst->opt_mem_alignment,
74d9e0dfa2SEric Blake                                  src->opt_mem_alignment);
75d9e0dfa2SEric Blake     dst->min_mem_alignment = MAX(dst->min_mem_alignment,
76d9e0dfa2SEric Blake                                  src->min_mem_alignment);
77d9e0dfa2SEric Blake     dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov);
78d9e0dfa2SEric Blake }
79d9e0dfa2SEric Blake 
8061007b31SStefan Hajnoczi void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
8161007b31SStefan Hajnoczi {
8261007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
8361007b31SStefan Hajnoczi     Error *local_err = NULL;
8461007b31SStefan Hajnoczi 
8561007b31SStefan Hajnoczi     memset(&bs->bl, 0, sizeof(bs->bl));
8661007b31SStefan Hajnoczi 
8761007b31SStefan Hajnoczi     if (!drv) {
8861007b31SStefan Hajnoczi         return;
8961007b31SStefan Hajnoczi     }
9061007b31SStefan Hajnoczi 
9179ba8c98SEric Blake     /* Default alignment based on whether driver has byte interface */
92a5b8dd2cSEric Blake     bs->bl.request_alignment = drv->bdrv_co_preadv ? 1 : 512;
9379ba8c98SEric Blake 
9461007b31SStefan Hajnoczi     /* Take some limits from the children as a default */
9561007b31SStefan Hajnoczi     if (bs->file) {
969a4f4c31SKevin Wolf         bdrv_refresh_limits(bs->file->bs, &local_err);
9761007b31SStefan Hajnoczi         if (local_err) {
9861007b31SStefan Hajnoczi             error_propagate(errp, local_err);
9961007b31SStefan Hajnoczi             return;
10061007b31SStefan Hajnoczi         }
101d9e0dfa2SEric Blake         bdrv_merge_limits(&bs->bl, &bs->file->bs->bl);
10261007b31SStefan Hajnoczi     } else {
1034196d2f0SDenis V. Lunev         bs->bl.min_mem_alignment = 512;
104459b4e66SDenis V. Lunev         bs->bl.opt_mem_alignment = getpagesize();
105bd44feb7SStefan Hajnoczi 
106bd44feb7SStefan Hajnoczi         /* Safe default since most protocols use readv()/writev()/etc */
107bd44feb7SStefan Hajnoczi         bs->bl.max_iov = IOV_MAX;
10861007b31SStefan Hajnoczi     }
10961007b31SStefan Hajnoczi 
110760e0063SKevin Wolf     if (bs->backing) {
111760e0063SKevin Wolf         bdrv_refresh_limits(bs->backing->bs, &local_err);
11261007b31SStefan Hajnoczi         if (local_err) {
11361007b31SStefan Hajnoczi             error_propagate(errp, local_err);
11461007b31SStefan Hajnoczi             return;
11561007b31SStefan Hajnoczi         }
116d9e0dfa2SEric Blake         bdrv_merge_limits(&bs->bl, &bs->backing->bs->bl);
11761007b31SStefan Hajnoczi     }
11861007b31SStefan Hajnoczi 
11961007b31SStefan Hajnoczi     /* Then let the driver override it */
12061007b31SStefan Hajnoczi     if (drv->bdrv_refresh_limits) {
12161007b31SStefan Hajnoczi         drv->bdrv_refresh_limits(bs, errp);
12261007b31SStefan Hajnoczi     }
12361007b31SStefan Hajnoczi }
12461007b31SStefan Hajnoczi 
12561007b31SStefan Hajnoczi /**
12661007b31SStefan Hajnoczi  * The copy-on-read flag is actually a reference count so multiple users may
12761007b31SStefan Hajnoczi  * use the feature without worrying about clobbering its previous state.
12861007b31SStefan Hajnoczi  * Copy-on-read stays enabled until all users have called to disable it.
12961007b31SStefan Hajnoczi  */
13061007b31SStefan Hajnoczi void bdrv_enable_copy_on_read(BlockDriverState *bs)
13161007b31SStefan Hajnoczi {
13261007b31SStefan Hajnoczi     bs->copy_on_read++;
13361007b31SStefan Hajnoczi }
13461007b31SStefan Hajnoczi 
13561007b31SStefan Hajnoczi void bdrv_disable_copy_on_read(BlockDriverState *bs)
13661007b31SStefan Hajnoczi {
13761007b31SStefan Hajnoczi     assert(bs->copy_on_read > 0);
13861007b31SStefan Hajnoczi     bs->copy_on_read--;
13961007b31SStefan Hajnoczi }
14061007b31SStefan Hajnoczi 
14161007b31SStefan Hajnoczi /* Check if any requests are in-flight (including throttled requests) */
142439db28cSKevin Wolf bool bdrv_requests_pending(BlockDriverState *bs)
14361007b31SStefan Hajnoczi {
14437a639a7SKevin Wolf     BdrvChild *child;
14537a639a7SKevin Wolf 
14699723548SPaolo Bonzini     if (atomic_read(&bs->in_flight)) {
14761007b31SStefan Hajnoczi         return true;
14861007b31SStefan Hajnoczi     }
14937a639a7SKevin Wolf 
15037a639a7SKevin Wolf     QLIST_FOREACH(child, &bs->children, next) {
15137a639a7SKevin Wolf         if (bdrv_requests_pending(child->bs)) {
15261007b31SStefan Hajnoczi             return true;
15361007b31SStefan Hajnoczi         }
15461007b31SStefan Hajnoczi     }
15537a639a7SKevin Wolf 
15661007b31SStefan Hajnoczi     return false;
15761007b31SStefan Hajnoczi }
15861007b31SStefan Hajnoczi 
159d42cf288SPaolo Bonzini static bool bdrv_drain_recurse(BlockDriverState *bs)
16067da1dc5SFam Zheng {
16167da1dc5SFam Zheng     BdrvChild *child;
162d42cf288SPaolo Bonzini     bool waited;
163d42cf288SPaolo Bonzini 
16488b062c2SPaolo Bonzini     waited = BDRV_POLL_WHILE(bs, atomic_read(&bs->in_flight) > 0);
16567da1dc5SFam Zheng 
16667da1dc5SFam Zheng     if (bs->drv && bs->drv->bdrv_drain) {
16767da1dc5SFam Zheng         bs->drv->bdrv_drain(bs);
16867da1dc5SFam Zheng     }
169d42cf288SPaolo Bonzini 
17067da1dc5SFam Zheng     QLIST_FOREACH(child, &bs->children, next) {
171d42cf288SPaolo Bonzini         waited |= bdrv_drain_recurse(child->bs);
17267da1dc5SFam Zheng     }
173d42cf288SPaolo Bonzini 
174d42cf288SPaolo Bonzini     return waited;
17567da1dc5SFam Zheng }
17667da1dc5SFam Zheng 
177a77fd4bbSFam Zheng typedef struct {
178a77fd4bbSFam Zheng     Coroutine *co;
179a77fd4bbSFam Zheng     BlockDriverState *bs;
180a77fd4bbSFam Zheng     bool done;
181a77fd4bbSFam Zheng } BdrvCoDrainData;
182a77fd4bbSFam Zheng 
183a77fd4bbSFam Zheng static void bdrv_co_drain_bh_cb(void *opaque)
184a77fd4bbSFam Zheng {
185a77fd4bbSFam Zheng     BdrvCoDrainData *data = opaque;
186a77fd4bbSFam Zheng     Coroutine *co = data->co;
18799723548SPaolo Bonzini     BlockDriverState *bs = data->bs;
188a77fd4bbSFam Zheng 
18999723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
190d42cf288SPaolo Bonzini     bdrv_drained_begin(bs);
191a77fd4bbSFam Zheng     data->done = true;
1921919631eSPaolo Bonzini     aio_co_wake(co);
193a77fd4bbSFam Zheng }
194a77fd4bbSFam Zheng 
195b6e84c97SPaolo Bonzini static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs)
196a77fd4bbSFam Zheng {
197a77fd4bbSFam Zheng     BdrvCoDrainData data;
198a77fd4bbSFam Zheng 
199a77fd4bbSFam Zheng     /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
200a77fd4bbSFam Zheng      * other coroutines run if they were queued from
201a77fd4bbSFam Zheng      * qemu_co_queue_run_restart(). */
202a77fd4bbSFam Zheng 
203a77fd4bbSFam Zheng     assert(qemu_in_coroutine());
204a77fd4bbSFam Zheng     data = (BdrvCoDrainData) {
205a77fd4bbSFam Zheng         .co = qemu_coroutine_self(),
206a77fd4bbSFam Zheng         .bs = bs,
207a77fd4bbSFam Zheng         .done = false,
208a77fd4bbSFam Zheng     };
20999723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
210fffb6e12SPaolo Bonzini     aio_bh_schedule_oneshot(bdrv_get_aio_context(bs),
211fffb6e12SPaolo Bonzini                             bdrv_co_drain_bh_cb, &data);
212a77fd4bbSFam Zheng 
213a77fd4bbSFam Zheng     qemu_coroutine_yield();
214a77fd4bbSFam Zheng     /* If we are resumed from some other event (such as an aio completion or a
215a77fd4bbSFam Zheng      * timer callback), it is a bug in the caller that should be fixed. */
216a77fd4bbSFam Zheng     assert(data.done);
217a77fd4bbSFam Zheng }
218a77fd4bbSFam Zheng 
2196820643fSKevin Wolf void bdrv_drained_begin(BlockDriverState *bs)
2206820643fSKevin Wolf {
221d42cf288SPaolo Bonzini     if (qemu_in_coroutine()) {
222d42cf288SPaolo Bonzini         bdrv_co_yield_to_drain(bs);
223d42cf288SPaolo Bonzini         return;
224d42cf288SPaolo Bonzini     }
225d42cf288SPaolo Bonzini 
2266820643fSKevin Wolf     if (!bs->quiesce_counter++) {
2276820643fSKevin Wolf         aio_disable_external(bdrv_get_aio_context(bs));
2286820643fSKevin Wolf         bdrv_parent_drained_begin(bs);
2296820643fSKevin Wolf     }
2306820643fSKevin Wolf 
2316820643fSKevin Wolf     bdrv_drain_recurse(bs);
2326820643fSKevin Wolf }
2336820643fSKevin Wolf 
2346820643fSKevin Wolf void bdrv_drained_end(BlockDriverState *bs)
2356820643fSKevin Wolf {
2366820643fSKevin Wolf     assert(bs->quiesce_counter > 0);
2376820643fSKevin Wolf     if (--bs->quiesce_counter > 0) {
2386820643fSKevin Wolf         return;
2396820643fSKevin Wolf     }
2406820643fSKevin Wolf 
2416820643fSKevin Wolf     bdrv_parent_drained_end(bs);
2426820643fSKevin Wolf     aio_enable_external(bdrv_get_aio_context(bs));
2436820643fSKevin Wolf }
2446820643fSKevin Wolf 
24561007b31SStefan Hajnoczi /*
24667da1dc5SFam Zheng  * Wait for pending requests to complete on a single BlockDriverState subtree,
24767da1dc5SFam Zheng  * and suspend block driver's internal I/O until next request arrives.
24861007b31SStefan Hajnoczi  *
24961007b31SStefan Hajnoczi  * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState
25061007b31SStefan Hajnoczi  * AioContext.
2517a63f3cdSStefan Hajnoczi  *
2527a63f3cdSStefan Hajnoczi  * Only this BlockDriverState's AioContext is run, so in-flight requests must
2537a63f3cdSStefan Hajnoczi  * not depend on events in other AioContexts.  In that case, use
2547a63f3cdSStefan Hajnoczi  * bdrv_drain_all() instead.
25561007b31SStefan Hajnoczi  */
256b6e84c97SPaolo Bonzini void coroutine_fn bdrv_co_drain(BlockDriverState *bs)
257b6e84c97SPaolo Bonzini {
2586820643fSKevin Wolf     assert(qemu_in_coroutine());
2596820643fSKevin Wolf     bdrv_drained_begin(bs);
2606820643fSKevin Wolf     bdrv_drained_end(bs);
261b6e84c97SPaolo Bonzini }
262b6e84c97SPaolo Bonzini 
26361007b31SStefan Hajnoczi void bdrv_drain(BlockDriverState *bs)
26461007b31SStefan Hajnoczi {
2656820643fSKevin Wolf     bdrv_drained_begin(bs);
2666820643fSKevin Wolf     bdrv_drained_end(bs);
26761007b31SStefan Hajnoczi }
26861007b31SStefan Hajnoczi 
26961007b31SStefan Hajnoczi /*
27061007b31SStefan Hajnoczi  * Wait for pending requests to complete across all BlockDriverStates
27161007b31SStefan Hajnoczi  *
27261007b31SStefan Hajnoczi  * This function does not flush data to disk, use bdrv_flush_all() for that
27361007b31SStefan Hajnoczi  * after calling this function.
274c0778f66SAlberto Garcia  *
275c0778f66SAlberto Garcia  * This pauses all block jobs and disables external clients. It must
276c0778f66SAlberto Garcia  * be paired with bdrv_drain_all_end().
277c0778f66SAlberto Garcia  *
278c0778f66SAlberto Garcia  * NOTE: no new block jobs or BlockDriverStates can be created between
279c0778f66SAlberto Garcia  * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls.
28061007b31SStefan Hajnoczi  */
281c0778f66SAlberto Garcia void bdrv_drain_all_begin(void)
28261007b31SStefan Hajnoczi {
28361007b31SStefan Hajnoczi     /* Always run first iteration so any pending completion BHs run */
28499723548SPaolo Bonzini     bool waited = true;
2857c8eece4SKevin Wolf     BlockDriverState *bs;
28688be7b4bSKevin Wolf     BdrvNextIterator it;
287eb1364ceSAlberto Garcia     BlockJob *job = NULL;
288f406c03cSAlexander Yarygin     GSList *aio_ctxs = NULL, *ctx;
28961007b31SStefan Hajnoczi 
290eb1364ceSAlberto Garcia     while ((job = block_job_next(job))) {
291eb1364ceSAlberto Garcia         AioContext *aio_context = blk_get_aio_context(job->blk);
292eb1364ceSAlberto Garcia 
293eb1364ceSAlberto Garcia         aio_context_acquire(aio_context);
294eb1364ceSAlberto Garcia         block_job_pause(job);
295eb1364ceSAlberto Garcia         aio_context_release(aio_context);
296eb1364ceSAlberto Garcia     }
297eb1364ceSAlberto Garcia 
29888be7b4bSKevin Wolf     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
29961007b31SStefan Hajnoczi         AioContext *aio_context = bdrv_get_aio_context(bs);
30061007b31SStefan Hajnoczi 
30161007b31SStefan Hajnoczi         aio_context_acquire(aio_context);
302c2066af0SKevin Wolf         bdrv_parent_drained_begin(bs);
303c0778f66SAlberto Garcia         aio_disable_external(aio_context);
30461007b31SStefan Hajnoczi         aio_context_release(aio_context);
305f406c03cSAlexander Yarygin 
306764ba3aeSAlberto Garcia         if (!g_slist_find(aio_ctxs, aio_context)) {
307f406c03cSAlexander Yarygin             aio_ctxs = g_slist_prepend(aio_ctxs, aio_context);
308f406c03cSAlexander Yarygin         }
30961007b31SStefan Hajnoczi     }
31061007b31SStefan Hajnoczi 
3117a63f3cdSStefan Hajnoczi     /* Note that completion of an asynchronous I/O operation can trigger any
3127a63f3cdSStefan Hajnoczi      * number of other I/O operations on other devices---for example a
3137a63f3cdSStefan Hajnoczi      * coroutine can submit an I/O request to another device in response to
3147a63f3cdSStefan Hajnoczi      * request completion.  Therefore we must keep looping until there was no
3157a63f3cdSStefan Hajnoczi      * more activity rather than simply draining each device independently.
3167a63f3cdSStefan Hajnoczi      */
31799723548SPaolo Bonzini     while (waited) {
31899723548SPaolo Bonzini         waited = false;
319f406c03cSAlexander Yarygin 
320f406c03cSAlexander Yarygin         for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) {
321f406c03cSAlexander Yarygin             AioContext *aio_context = ctx->data;
32261007b31SStefan Hajnoczi 
32361007b31SStefan Hajnoczi             aio_context_acquire(aio_context);
32488be7b4bSKevin Wolf             for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
325f406c03cSAlexander Yarygin                 if (aio_context == bdrv_get_aio_context(bs)) {
326d42cf288SPaolo Bonzini                     waited |= bdrv_drain_recurse(bs);
327f406c03cSAlexander Yarygin                 }
328f406c03cSAlexander Yarygin             }
32961007b31SStefan Hajnoczi             aio_context_release(aio_context);
33061007b31SStefan Hajnoczi         }
33161007b31SStefan Hajnoczi     }
33261007b31SStefan Hajnoczi 
333c0778f66SAlberto Garcia     g_slist_free(aio_ctxs);
334c0778f66SAlberto Garcia }
335c0778f66SAlberto Garcia 
336c0778f66SAlberto Garcia void bdrv_drain_all_end(void)
337c0778f66SAlberto Garcia {
338c0778f66SAlberto Garcia     BlockDriverState *bs;
339c0778f66SAlberto Garcia     BdrvNextIterator it;
340c0778f66SAlberto Garcia     BlockJob *job = NULL;
341c0778f66SAlberto Garcia 
34288be7b4bSKevin Wolf     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
34361007b31SStefan Hajnoczi         AioContext *aio_context = bdrv_get_aio_context(bs);
34461007b31SStefan Hajnoczi 
34561007b31SStefan Hajnoczi         aio_context_acquire(aio_context);
346c0778f66SAlberto Garcia         aio_enable_external(aio_context);
347c2066af0SKevin Wolf         bdrv_parent_drained_end(bs);
34861007b31SStefan Hajnoczi         aio_context_release(aio_context);
34961007b31SStefan Hajnoczi     }
350eb1364ceSAlberto Garcia 
351eb1364ceSAlberto Garcia     while ((job = block_job_next(job))) {
352eb1364ceSAlberto Garcia         AioContext *aio_context = blk_get_aio_context(job->blk);
353eb1364ceSAlberto Garcia 
354eb1364ceSAlberto Garcia         aio_context_acquire(aio_context);
355eb1364ceSAlberto Garcia         block_job_resume(job);
356eb1364ceSAlberto Garcia         aio_context_release(aio_context);
357eb1364ceSAlberto Garcia     }
35861007b31SStefan Hajnoczi }
35961007b31SStefan Hajnoczi 
360c0778f66SAlberto Garcia void bdrv_drain_all(void)
361c0778f66SAlberto Garcia {
362c0778f66SAlberto Garcia     bdrv_drain_all_begin();
363c0778f66SAlberto Garcia     bdrv_drain_all_end();
364c0778f66SAlberto Garcia }
365c0778f66SAlberto Garcia 
36661007b31SStefan Hajnoczi /**
36761007b31SStefan Hajnoczi  * Remove an active request from the tracked requests list
36861007b31SStefan Hajnoczi  *
36961007b31SStefan Hajnoczi  * This function should be called when a tracked request is completing.
37061007b31SStefan Hajnoczi  */
37161007b31SStefan Hajnoczi static void tracked_request_end(BdrvTrackedRequest *req)
37261007b31SStefan Hajnoczi {
37361007b31SStefan Hajnoczi     if (req->serialising) {
37461007b31SStefan Hajnoczi         req->bs->serialising_in_flight--;
37561007b31SStefan Hajnoczi     }
37661007b31SStefan Hajnoczi 
37761007b31SStefan Hajnoczi     QLIST_REMOVE(req, list);
37861007b31SStefan Hajnoczi     qemu_co_queue_restart_all(&req->wait_queue);
37961007b31SStefan Hajnoczi }
38061007b31SStefan Hajnoczi 
38161007b31SStefan Hajnoczi /**
38261007b31SStefan Hajnoczi  * Add an active request to the tracked requests list
38361007b31SStefan Hajnoczi  */
38461007b31SStefan Hajnoczi static void tracked_request_begin(BdrvTrackedRequest *req,
38561007b31SStefan Hajnoczi                                   BlockDriverState *bs,
38661007b31SStefan Hajnoczi                                   int64_t offset,
387ebde595cSFam Zheng                                   unsigned int bytes,
388ebde595cSFam Zheng                                   enum BdrvTrackedRequestType type)
38961007b31SStefan Hajnoczi {
39061007b31SStefan Hajnoczi     *req = (BdrvTrackedRequest){
39161007b31SStefan Hajnoczi         .bs = bs,
39261007b31SStefan Hajnoczi         .offset         = offset,
39361007b31SStefan Hajnoczi         .bytes          = bytes,
394ebde595cSFam Zheng         .type           = type,
39561007b31SStefan Hajnoczi         .co             = qemu_coroutine_self(),
39661007b31SStefan Hajnoczi         .serialising    = false,
39761007b31SStefan Hajnoczi         .overlap_offset = offset,
39861007b31SStefan Hajnoczi         .overlap_bytes  = bytes,
39961007b31SStefan Hajnoczi     };
40061007b31SStefan Hajnoczi 
40161007b31SStefan Hajnoczi     qemu_co_queue_init(&req->wait_queue);
40261007b31SStefan Hajnoczi 
40361007b31SStefan Hajnoczi     QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
40461007b31SStefan Hajnoczi }
40561007b31SStefan Hajnoczi 
40661007b31SStefan Hajnoczi static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align)
40761007b31SStefan Hajnoczi {
40861007b31SStefan Hajnoczi     int64_t overlap_offset = req->offset & ~(align - 1);
40961007b31SStefan Hajnoczi     unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align)
41061007b31SStefan Hajnoczi                                - overlap_offset;
41161007b31SStefan Hajnoczi 
41261007b31SStefan Hajnoczi     if (!req->serialising) {
41361007b31SStefan Hajnoczi         req->bs->serialising_in_flight++;
41461007b31SStefan Hajnoczi         req->serialising = true;
41561007b31SStefan Hajnoczi     }
41661007b31SStefan Hajnoczi 
41761007b31SStefan Hajnoczi     req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
41861007b31SStefan Hajnoczi     req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
41961007b31SStefan Hajnoczi }
42061007b31SStefan Hajnoczi 
42161007b31SStefan Hajnoczi /**
422244483e6SKevin Wolf  * Round a region to cluster boundaries (sector-based)
42361007b31SStefan Hajnoczi  */
424244483e6SKevin Wolf void bdrv_round_sectors_to_clusters(BlockDriverState *bs,
42561007b31SStefan Hajnoczi                                     int64_t sector_num, int nb_sectors,
42661007b31SStefan Hajnoczi                                     int64_t *cluster_sector_num,
42761007b31SStefan Hajnoczi                                     int *cluster_nb_sectors)
42861007b31SStefan Hajnoczi {
42961007b31SStefan Hajnoczi     BlockDriverInfo bdi;
43061007b31SStefan Hajnoczi 
43161007b31SStefan Hajnoczi     if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
43261007b31SStefan Hajnoczi         *cluster_sector_num = sector_num;
43361007b31SStefan Hajnoczi         *cluster_nb_sectors = nb_sectors;
43461007b31SStefan Hajnoczi     } else {
43561007b31SStefan Hajnoczi         int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE;
43661007b31SStefan Hajnoczi         *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c);
43761007b31SStefan Hajnoczi         *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num +
43861007b31SStefan Hajnoczi                                             nb_sectors, c);
43961007b31SStefan Hajnoczi     }
44061007b31SStefan Hajnoczi }
44161007b31SStefan Hajnoczi 
442244483e6SKevin Wolf /**
443244483e6SKevin Wolf  * Round a region to cluster boundaries
444244483e6SKevin Wolf  */
445244483e6SKevin Wolf void bdrv_round_to_clusters(BlockDriverState *bs,
446244483e6SKevin Wolf                             int64_t offset, unsigned int bytes,
447244483e6SKevin Wolf                             int64_t *cluster_offset,
448244483e6SKevin Wolf                             unsigned int *cluster_bytes)
449244483e6SKevin Wolf {
450244483e6SKevin Wolf     BlockDriverInfo bdi;
451244483e6SKevin Wolf 
452244483e6SKevin Wolf     if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
453244483e6SKevin Wolf         *cluster_offset = offset;
454244483e6SKevin Wolf         *cluster_bytes = bytes;
455244483e6SKevin Wolf     } else {
456244483e6SKevin Wolf         int64_t c = bdi.cluster_size;
457244483e6SKevin Wolf         *cluster_offset = QEMU_ALIGN_DOWN(offset, c);
458244483e6SKevin Wolf         *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c);
459244483e6SKevin Wolf     }
460244483e6SKevin Wolf }
461244483e6SKevin Wolf 
46261007b31SStefan Hajnoczi static int bdrv_get_cluster_size(BlockDriverState *bs)
46361007b31SStefan Hajnoczi {
46461007b31SStefan Hajnoczi     BlockDriverInfo bdi;
46561007b31SStefan Hajnoczi     int ret;
46661007b31SStefan Hajnoczi 
46761007b31SStefan Hajnoczi     ret = bdrv_get_info(bs, &bdi);
46861007b31SStefan Hajnoczi     if (ret < 0 || bdi.cluster_size == 0) {
469a5b8dd2cSEric Blake         return bs->bl.request_alignment;
47061007b31SStefan Hajnoczi     } else {
47161007b31SStefan Hajnoczi         return bdi.cluster_size;
47261007b31SStefan Hajnoczi     }
47361007b31SStefan Hajnoczi }
47461007b31SStefan Hajnoczi 
47561007b31SStefan Hajnoczi static bool tracked_request_overlaps(BdrvTrackedRequest *req,
47661007b31SStefan Hajnoczi                                      int64_t offset, unsigned int bytes)
47761007b31SStefan Hajnoczi {
47861007b31SStefan Hajnoczi     /*        aaaa   bbbb */
47961007b31SStefan Hajnoczi     if (offset >= req->overlap_offset + req->overlap_bytes) {
48061007b31SStefan Hajnoczi         return false;
48161007b31SStefan Hajnoczi     }
48261007b31SStefan Hajnoczi     /* bbbb   aaaa        */
48361007b31SStefan Hajnoczi     if (req->overlap_offset >= offset + bytes) {
48461007b31SStefan Hajnoczi         return false;
48561007b31SStefan Hajnoczi     }
48661007b31SStefan Hajnoczi     return true;
48761007b31SStefan Hajnoczi }
48861007b31SStefan Hajnoczi 
48999723548SPaolo Bonzini void bdrv_inc_in_flight(BlockDriverState *bs)
49099723548SPaolo Bonzini {
49199723548SPaolo Bonzini     atomic_inc(&bs->in_flight);
49299723548SPaolo Bonzini }
49399723548SPaolo Bonzini 
494c9d1a561SPaolo Bonzini static void dummy_bh_cb(void *opaque)
495c9d1a561SPaolo Bonzini {
496c9d1a561SPaolo Bonzini }
497c9d1a561SPaolo Bonzini 
498c9d1a561SPaolo Bonzini void bdrv_wakeup(BlockDriverState *bs)
499c9d1a561SPaolo Bonzini {
500c9d1a561SPaolo Bonzini     if (bs->wakeup) {
501c9d1a561SPaolo Bonzini         aio_bh_schedule_oneshot(qemu_get_aio_context(), dummy_bh_cb, NULL);
502c9d1a561SPaolo Bonzini     }
503c9d1a561SPaolo Bonzini }
504c9d1a561SPaolo Bonzini 
50599723548SPaolo Bonzini void bdrv_dec_in_flight(BlockDriverState *bs)
50699723548SPaolo Bonzini {
50799723548SPaolo Bonzini     atomic_dec(&bs->in_flight);
508c9d1a561SPaolo Bonzini     bdrv_wakeup(bs);
50999723548SPaolo Bonzini }
51099723548SPaolo Bonzini 
51161007b31SStefan Hajnoczi static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self)
51261007b31SStefan Hajnoczi {
51361007b31SStefan Hajnoczi     BlockDriverState *bs = self->bs;
51461007b31SStefan Hajnoczi     BdrvTrackedRequest *req;
51561007b31SStefan Hajnoczi     bool retry;
51661007b31SStefan Hajnoczi     bool waited = false;
51761007b31SStefan Hajnoczi 
51861007b31SStefan Hajnoczi     if (!bs->serialising_in_flight) {
51961007b31SStefan Hajnoczi         return false;
52061007b31SStefan Hajnoczi     }
52161007b31SStefan Hajnoczi 
52261007b31SStefan Hajnoczi     do {
52361007b31SStefan Hajnoczi         retry = false;
52461007b31SStefan Hajnoczi         QLIST_FOREACH(req, &bs->tracked_requests, list) {
52561007b31SStefan Hajnoczi             if (req == self || (!req->serialising && !self->serialising)) {
52661007b31SStefan Hajnoczi                 continue;
52761007b31SStefan Hajnoczi             }
52861007b31SStefan Hajnoczi             if (tracked_request_overlaps(req, self->overlap_offset,
52961007b31SStefan Hajnoczi                                          self->overlap_bytes))
53061007b31SStefan Hajnoczi             {
53161007b31SStefan Hajnoczi                 /* Hitting this means there was a reentrant request, for
53261007b31SStefan Hajnoczi                  * example, a block driver issuing nested requests.  This must
53361007b31SStefan Hajnoczi                  * never happen since it means deadlock.
53461007b31SStefan Hajnoczi                  */
53561007b31SStefan Hajnoczi                 assert(qemu_coroutine_self() != req->co);
53661007b31SStefan Hajnoczi 
53761007b31SStefan Hajnoczi                 /* If the request is already (indirectly) waiting for us, or
53861007b31SStefan Hajnoczi                  * will wait for us as soon as it wakes up, then just go on
53961007b31SStefan Hajnoczi                  * (instead of producing a deadlock in the former case). */
54061007b31SStefan Hajnoczi                 if (!req->waiting_for) {
54161007b31SStefan Hajnoczi                     self->waiting_for = req;
5421ace7ceaSPaolo Bonzini                     qemu_co_queue_wait(&req->wait_queue, NULL);
54361007b31SStefan Hajnoczi                     self->waiting_for = NULL;
54461007b31SStefan Hajnoczi                     retry = true;
54561007b31SStefan Hajnoczi                     waited = true;
54661007b31SStefan Hajnoczi                     break;
54761007b31SStefan Hajnoczi                 }
54861007b31SStefan Hajnoczi             }
54961007b31SStefan Hajnoczi         }
55061007b31SStefan Hajnoczi     } while (retry);
55161007b31SStefan Hajnoczi 
55261007b31SStefan Hajnoczi     return waited;
55361007b31SStefan Hajnoczi }
55461007b31SStefan Hajnoczi 
55561007b31SStefan Hajnoczi static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset,
55661007b31SStefan Hajnoczi                                    size_t size)
55761007b31SStefan Hajnoczi {
55861007b31SStefan Hajnoczi     if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) {
55961007b31SStefan Hajnoczi         return -EIO;
56061007b31SStefan Hajnoczi     }
56161007b31SStefan Hajnoczi 
56261007b31SStefan Hajnoczi     if (!bdrv_is_inserted(bs)) {
56361007b31SStefan Hajnoczi         return -ENOMEDIUM;
56461007b31SStefan Hajnoczi     }
56561007b31SStefan Hajnoczi 
56661007b31SStefan Hajnoczi     if (offset < 0) {
56761007b31SStefan Hajnoczi         return -EIO;
56861007b31SStefan Hajnoczi     }
56961007b31SStefan Hajnoczi 
57061007b31SStefan Hajnoczi     return 0;
57161007b31SStefan Hajnoczi }
57261007b31SStefan Hajnoczi 
57361007b31SStefan Hajnoczi typedef struct RwCo {
574e293b7a3SKevin Wolf     BdrvChild *child;
57561007b31SStefan Hajnoczi     int64_t offset;
57661007b31SStefan Hajnoczi     QEMUIOVector *qiov;
57761007b31SStefan Hajnoczi     bool is_write;
57861007b31SStefan Hajnoczi     int ret;
57961007b31SStefan Hajnoczi     BdrvRequestFlags flags;
58061007b31SStefan Hajnoczi } RwCo;
58161007b31SStefan Hajnoczi 
58261007b31SStefan Hajnoczi static void coroutine_fn bdrv_rw_co_entry(void *opaque)
58361007b31SStefan Hajnoczi {
58461007b31SStefan Hajnoczi     RwCo *rwco = opaque;
58561007b31SStefan Hajnoczi 
58661007b31SStefan Hajnoczi     if (!rwco->is_write) {
587a03ef88fSKevin Wolf         rwco->ret = bdrv_co_preadv(rwco->child, rwco->offset,
58861007b31SStefan Hajnoczi                                    rwco->qiov->size, rwco->qiov,
58961007b31SStefan Hajnoczi                                    rwco->flags);
59061007b31SStefan Hajnoczi     } else {
591a03ef88fSKevin Wolf         rwco->ret = bdrv_co_pwritev(rwco->child, rwco->offset,
59261007b31SStefan Hajnoczi                                     rwco->qiov->size, rwco->qiov,
59361007b31SStefan Hajnoczi                                     rwco->flags);
59461007b31SStefan Hajnoczi     }
59561007b31SStefan Hajnoczi }
59661007b31SStefan Hajnoczi 
59761007b31SStefan Hajnoczi /*
59861007b31SStefan Hajnoczi  * Process a vectored synchronous request using coroutines
59961007b31SStefan Hajnoczi  */
600e293b7a3SKevin Wolf static int bdrv_prwv_co(BdrvChild *child, int64_t offset,
60161007b31SStefan Hajnoczi                         QEMUIOVector *qiov, bool is_write,
60261007b31SStefan Hajnoczi                         BdrvRequestFlags flags)
60361007b31SStefan Hajnoczi {
60461007b31SStefan Hajnoczi     Coroutine *co;
60561007b31SStefan Hajnoczi     RwCo rwco = {
606e293b7a3SKevin Wolf         .child = child,
60761007b31SStefan Hajnoczi         .offset = offset,
60861007b31SStefan Hajnoczi         .qiov = qiov,
60961007b31SStefan Hajnoczi         .is_write = is_write,
61061007b31SStefan Hajnoczi         .ret = NOT_DONE,
61161007b31SStefan Hajnoczi         .flags = flags,
61261007b31SStefan Hajnoczi     };
61361007b31SStefan Hajnoczi 
61461007b31SStefan Hajnoczi     if (qemu_in_coroutine()) {
61561007b31SStefan Hajnoczi         /* Fast-path if already in coroutine context */
61661007b31SStefan Hajnoczi         bdrv_rw_co_entry(&rwco);
61761007b31SStefan Hajnoczi     } else {
6180b8b8753SPaolo Bonzini         co = qemu_coroutine_create(bdrv_rw_co_entry, &rwco);
6190b8b8753SPaolo Bonzini         qemu_coroutine_enter(co);
62088b062c2SPaolo Bonzini         BDRV_POLL_WHILE(child->bs, rwco.ret == NOT_DONE);
62161007b31SStefan Hajnoczi     }
62261007b31SStefan Hajnoczi     return rwco.ret;
62361007b31SStefan Hajnoczi }
62461007b31SStefan Hajnoczi 
62561007b31SStefan Hajnoczi /*
62661007b31SStefan Hajnoczi  * Process a synchronous request using coroutines
62761007b31SStefan Hajnoczi  */
628e293b7a3SKevin Wolf static int bdrv_rw_co(BdrvChild *child, int64_t sector_num, uint8_t *buf,
62961007b31SStefan Hajnoczi                       int nb_sectors, bool is_write, BdrvRequestFlags flags)
63061007b31SStefan Hajnoczi {
63161007b31SStefan Hajnoczi     QEMUIOVector qiov;
63261007b31SStefan Hajnoczi     struct iovec iov = {
63361007b31SStefan Hajnoczi         .iov_base = (void *)buf,
63461007b31SStefan Hajnoczi         .iov_len = nb_sectors * BDRV_SECTOR_SIZE,
63561007b31SStefan Hajnoczi     };
63661007b31SStefan Hajnoczi 
63761007b31SStefan Hajnoczi     if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
63861007b31SStefan Hajnoczi         return -EINVAL;
63961007b31SStefan Hajnoczi     }
64061007b31SStefan Hajnoczi 
64161007b31SStefan Hajnoczi     qemu_iovec_init_external(&qiov, &iov, 1);
642e293b7a3SKevin Wolf     return bdrv_prwv_co(child, sector_num << BDRV_SECTOR_BITS,
64361007b31SStefan Hajnoczi                         &qiov, is_write, flags);
64461007b31SStefan Hajnoczi }
64561007b31SStefan Hajnoczi 
64661007b31SStefan Hajnoczi /* return < 0 if error. See bdrv_write() for the return codes */
647fbcbbf4eSKevin Wolf int bdrv_read(BdrvChild *child, int64_t sector_num,
64861007b31SStefan Hajnoczi               uint8_t *buf, int nb_sectors)
64961007b31SStefan Hajnoczi {
650e293b7a3SKevin Wolf     return bdrv_rw_co(child, sector_num, buf, nb_sectors, false, 0);
65161007b31SStefan Hajnoczi }
65261007b31SStefan Hajnoczi 
65361007b31SStefan Hajnoczi /* Return < 0 if error. Important errors are:
65461007b31SStefan Hajnoczi   -EIO         generic I/O error (may happen for all errors)
65561007b31SStefan Hajnoczi   -ENOMEDIUM   No media inserted.
65661007b31SStefan Hajnoczi   -EINVAL      Invalid sector number or nb_sectors
65761007b31SStefan Hajnoczi   -EACCES      Trying to write a read-only device
65861007b31SStefan Hajnoczi */
65918d51c4bSKevin Wolf int bdrv_write(BdrvChild *child, int64_t sector_num,
66061007b31SStefan Hajnoczi                const uint8_t *buf, int nb_sectors)
66161007b31SStefan Hajnoczi {
662e293b7a3SKevin Wolf     return bdrv_rw_co(child, sector_num, (uint8_t *)buf, nb_sectors, true, 0);
66361007b31SStefan Hajnoczi }
66461007b31SStefan Hajnoczi 
665720ff280SKevin Wolf int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset,
66674021bc4SEric Blake                        int count, BdrvRequestFlags flags)
66761007b31SStefan Hajnoczi {
66874021bc4SEric Blake     QEMUIOVector qiov;
66974021bc4SEric Blake     struct iovec iov = {
67074021bc4SEric Blake         .iov_base = NULL,
67174021bc4SEric Blake         .iov_len = count,
67274021bc4SEric Blake     };
67374021bc4SEric Blake 
67474021bc4SEric Blake     qemu_iovec_init_external(&qiov, &iov, 1);
675e293b7a3SKevin Wolf     return bdrv_prwv_co(child, offset, &qiov, true,
67661007b31SStefan Hajnoczi                         BDRV_REQ_ZERO_WRITE | flags);
67761007b31SStefan Hajnoczi }
67861007b31SStefan Hajnoczi 
67961007b31SStefan Hajnoczi /*
68074021bc4SEric Blake  * Completely zero out a block device with the help of bdrv_pwrite_zeroes.
68161007b31SStefan Hajnoczi  * The operation is sped up by checking the block status and only writing
68261007b31SStefan Hajnoczi  * zeroes to the device if they currently do not return zeroes. Optional
68374021bc4SEric Blake  * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
684465fe887SEric Blake  * BDRV_REQ_FUA).
68561007b31SStefan Hajnoczi  *
68661007b31SStefan Hajnoczi  * Returns < 0 on error, 0 on success. For error codes see bdrv_write().
68761007b31SStefan Hajnoczi  */
688720ff280SKevin Wolf int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags)
68961007b31SStefan Hajnoczi {
69061007b31SStefan Hajnoczi     int64_t target_sectors, ret, nb_sectors, sector_num = 0;
691720ff280SKevin Wolf     BlockDriverState *bs = child->bs;
69267a0fd2aSFam Zheng     BlockDriverState *file;
69361007b31SStefan Hajnoczi     int n;
69461007b31SStefan Hajnoczi 
69561007b31SStefan Hajnoczi     target_sectors = bdrv_nb_sectors(bs);
69661007b31SStefan Hajnoczi     if (target_sectors < 0) {
69761007b31SStefan Hajnoczi         return target_sectors;
69861007b31SStefan Hajnoczi     }
69961007b31SStefan Hajnoczi 
70061007b31SStefan Hajnoczi     for (;;) {
70161007b31SStefan Hajnoczi         nb_sectors = MIN(target_sectors - sector_num, BDRV_REQUEST_MAX_SECTORS);
70261007b31SStefan Hajnoczi         if (nb_sectors <= 0) {
70361007b31SStefan Hajnoczi             return 0;
70461007b31SStefan Hajnoczi         }
70567a0fd2aSFam Zheng         ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n, &file);
70661007b31SStefan Hajnoczi         if (ret < 0) {
70761007b31SStefan Hajnoczi             error_report("error getting block status at sector %" PRId64 ": %s",
70861007b31SStefan Hajnoczi                          sector_num, strerror(-ret));
70961007b31SStefan Hajnoczi             return ret;
71061007b31SStefan Hajnoczi         }
71161007b31SStefan Hajnoczi         if (ret & BDRV_BLOCK_ZERO) {
71261007b31SStefan Hajnoczi             sector_num += n;
71361007b31SStefan Hajnoczi             continue;
71461007b31SStefan Hajnoczi         }
715720ff280SKevin Wolf         ret = bdrv_pwrite_zeroes(child, sector_num << BDRV_SECTOR_BITS,
71674021bc4SEric Blake                                  n << BDRV_SECTOR_BITS, flags);
71761007b31SStefan Hajnoczi         if (ret < 0) {
71861007b31SStefan Hajnoczi             error_report("error writing zeroes at sector %" PRId64 ": %s",
71961007b31SStefan Hajnoczi                          sector_num, strerror(-ret));
72061007b31SStefan Hajnoczi             return ret;
72161007b31SStefan Hajnoczi         }
72261007b31SStefan Hajnoczi         sector_num += n;
72361007b31SStefan Hajnoczi     }
72461007b31SStefan Hajnoczi }
72561007b31SStefan Hajnoczi 
726cf2ab8fcSKevin Wolf int bdrv_preadv(BdrvChild *child, int64_t offset, QEMUIOVector *qiov)
727f1e84741SKevin Wolf {
728f1e84741SKevin Wolf     int ret;
729f1e84741SKevin Wolf 
730e293b7a3SKevin Wolf     ret = bdrv_prwv_co(child, offset, qiov, false, 0);
731f1e84741SKevin Wolf     if (ret < 0) {
732f1e84741SKevin Wolf         return ret;
733f1e84741SKevin Wolf     }
734f1e84741SKevin Wolf 
735f1e84741SKevin Wolf     return qiov->size;
736f1e84741SKevin Wolf }
737f1e84741SKevin Wolf 
738cf2ab8fcSKevin Wolf int bdrv_pread(BdrvChild *child, int64_t offset, void *buf, int bytes)
73961007b31SStefan Hajnoczi {
74061007b31SStefan Hajnoczi     QEMUIOVector qiov;
74161007b31SStefan Hajnoczi     struct iovec iov = {
74261007b31SStefan Hajnoczi         .iov_base = (void *)buf,
74361007b31SStefan Hajnoczi         .iov_len = bytes,
74461007b31SStefan Hajnoczi     };
74561007b31SStefan Hajnoczi 
74661007b31SStefan Hajnoczi     if (bytes < 0) {
74761007b31SStefan Hajnoczi         return -EINVAL;
74861007b31SStefan Hajnoczi     }
74961007b31SStefan Hajnoczi 
75061007b31SStefan Hajnoczi     qemu_iovec_init_external(&qiov, &iov, 1);
751cf2ab8fcSKevin Wolf     return bdrv_preadv(child, offset, &qiov);
75261007b31SStefan Hajnoczi }
75361007b31SStefan Hajnoczi 
754d9ca2ea2SKevin Wolf int bdrv_pwritev(BdrvChild *child, int64_t offset, QEMUIOVector *qiov)
75561007b31SStefan Hajnoczi {
75661007b31SStefan Hajnoczi     int ret;
75761007b31SStefan Hajnoczi 
758e293b7a3SKevin Wolf     ret = bdrv_prwv_co(child, offset, qiov, true, 0);
75961007b31SStefan Hajnoczi     if (ret < 0) {
76061007b31SStefan Hajnoczi         return ret;
76161007b31SStefan Hajnoczi     }
76261007b31SStefan Hajnoczi 
76361007b31SStefan Hajnoczi     return qiov->size;
76461007b31SStefan Hajnoczi }
76561007b31SStefan Hajnoczi 
766d9ca2ea2SKevin Wolf int bdrv_pwrite(BdrvChild *child, int64_t offset, const void *buf, int bytes)
76761007b31SStefan Hajnoczi {
76861007b31SStefan Hajnoczi     QEMUIOVector qiov;
76961007b31SStefan Hajnoczi     struct iovec iov = {
77061007b31SStefan Hajnoczi         .iov_base   = (void *) buf,
77161007b31SStefan Hajnoczi         .iov_len    = bytes,
77261007b31SStefan Hajnoczi     };
77361007b31SStefan Hajnoczi 
77461007b31SStefan Hajnoczi     if (bytes < 0) {
77561007b31SStefan Hajnoczi         return -EINVAL;
77661007b31SStefan Hajnoczi     }
77761007b31SStefan Hajnoczi 
77861007b31SStefan Hajnoczi     qemu_iovec_init_external(&qiov, &iov, 1);
779d9ca2ea2SKevin Wolf     return bdrv_pwritev(child, offset, &qiov);
78061007b31SStefan Hajnoczi }
78161007b31SStefan Hajnoczi 
78261007b31SStefan Hajnoczi /*
78361007b31SStefan Hajnoczi  * Writes to the file and ensures that no writes are reordered across this
78461007b31SStefan Hajnoczi  * request (acts as a barrier)
78561007b31SStefan Hajnoczi  *
78661007b31SStefan Hajnoczi  * Returns 0 on success, -errno in error cases.
78761007b31SStefan Hajnoczi  */
788d9ca2ea2SKevin Wolf int bdrv_pwrite_sync(BdrvChild *child, int64_t offset,
78961007b31SStefan Hajnoczi                      const void *buf, int count)
79061007b31SStefan Hajnoczi {
79161007b31SStefan Hajnoczi     int ret;
79261007b31SStefan Hajnoczi 
793d9ca2ea2SKevin Wolf     ret = bdrv_pwrite(child, offset, buf, count);
79461007b31SStefan Hajnoczi     if (ret < 0) {
79561007b31SStefan Hajnoczi         return ret;
79661007b31SStefan Hajnoczi     }
79761007b31SStefan Hajnoczi 
798d9ca2ea2SKevin Wolf     ret = bdrv_flush(child->bs);
799855a6a93SKevin Wolf     if (ret < 0) {
800855a6a93SKevin Wolf         return ret;
80161007b31SStefan Hajnoczi     }
80261007b31SStefan Hajnoczi 
80361007b31SStefan Hajnoczi     return 0;
80461007b31SStefan Hajnoczi }
80561007b31SStefan Hajnoczi 
80608844473SKevin Wolf typedef struct CoroutineIOCompletion {
80708844473SKevin Wolf     Coroutine *coroutine;
80808844473SKevin Wolf     int ret;
80908844473SKevin Wolf } CoroutineIOCompletion;
81008844473SKevin Wolf 
81108844473SKevin Wolf static void bdrv_co_io_em_complete(void *opaque, int ret)
81208844473SKevin Wolf {
81308844473SKevin Wolf     CoroutineIOCompletion *co = opaque;
81408844473SKevin Wolf 
81508844473SKevin Wolf     co->ret = ret;
816b9e413ddSPaolo Bonzini     aio_co_wake(co->coroutine);
81708844473SKevin Wolf }
81808844473SKevin Wolf 
819166fe960SKevin Wolf static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
820166fe960SKevin Wolf                                            uint64_t offset, uint64_t bytes,
821166fe960SKevin Wolf                                            QEMUIOVector *qiov, int flags)
822166fe960SKevin Wolf {
823166fe960SKevin Wolf     BlockDriver *drv = bs->drv;
8243fb06697SKevin Wolf     int64_t sector_num;
8253fb06697SKevin Wolf     unsigned int nb_sectors;
8263fb06697SKevin Wolf 
827fa166538SEric Blake     assert(!(flags & ~BDRV_REQ_MASK));
828fa166538SEric Blake 
8293fb06697SKevin Wolf     if (drv->bdrv_co_preadv) {
8303fb06697SKevin Wolf         return drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
8313fb06697SKevin Wolf     }
8323fb06697SKevin Wolf 
8333fb06697SKevin Wolf     sector_num = offset >> BDRV_SECTOR_BITS;
8343fb06697SKevin Wolf     nb_sectors = bytes >> BDRV_SECTOR_BITS;
835166fe960SKevin Wolf 
836166fe960SKevin Wolf     assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
837166fe960SKevin Wolf     assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
838166fe960SKevin Wolf     assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
839166fe960SKevin Wolf 
84008844473SKevin Wolf     if (drv->bdrv_co_readv) {
841166fe960SKevin Wolf         return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
84208844473SKevin Wolf     } else {
84308844473SKevin Wolf         BlockAIOCB *acb;
84408844473SKevin Wolf         CoroutineIOCompletion co = {
84508844473SKevin Wolf             .coroutine = qemu_coroutine_self(),
84608844473SKevin Wolf         };
84708844473SKevin Wolf 
84808844473SKevin Wolf         acb = bs->drv->bdrv_aio_readv(bs, sector_num, qiov, nb_sectors,
84908844473SKevin Wolf                                       bdrv_co_io_em_complete, &co);
85008844473SKevin Wolf         if (acb == NULL) {
85108844473SKevin Wolf             return -EIO;
85208844473SKevin Wolf         } else {
85308844473SKevin Wolf             qemu_coroutine_yield();
85408844473SKevin Wolf             return co.ret;
85508844473SKevin Wolf         }
85608844473SKevin Wolf     }
857166fe960SKevin Wolf }
858166fe960SKevin Wolf 
85978a07294SKevin Wolf static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
86078a07294SKevin Wolf                                             uint64_t offset, uint64_t bytes,
86178a07294SKevin Wolf                                             QEMUIOVector *qiov, int flags)
86278a07294SKevin Wolf {
86378a07294SKevin Wolf     BlockDriver *drv = bs->drv;
8643fb06697SKevin Wolf     int64_t sector_num;
8653fb06697SKevin Wolf     unsigned int nb_sectors;
86678a07294SKevin Wolf     int ret;
86778a07294SKevin Wolf 
868fa166538SEric Blake     assert(!(flags & ~BDRV_REQ_MASK));
869fa166538SEric Blake 
8703fb06697SKevin Wolf     if (drv->bdrv_co_pwritev) {
871515c2f43SKevin Wolf         ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov,
872515c2f43SKevin Wolf                                    flags & bs->supported_write_flags);
873515c2f43SKevin Wolf         flags &= ~bs->supported_write_flags;
8743fb06697SKevin Wolf         goto emulate_flags;
8753fb06697SKevin Wolf     }
8763fb06697SKevin Wolf 
8773fb06697SKevin Wolf     sector_num = offset >> BDRV_SECTOR_BITS;
8783fb06697SKevin Wolf     nb_sectors = bytes >> BDRV_SECTOR_BITS;
8793fb06697SKevin Wolf 
88078a07294SKevin Wolf     assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
88178a07294SKevin Wolf     assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
88278a07294SKevin Wolf     assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
88378a07294SKevin Wolf 
88478a07294SKevin Wolf     if (drv->bdrv_co_writev_flags) {
88578a07294SKevin Wolf         ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov,
8864df863f3SEric Blake                                         flags & bs->supported_write_flags);
8874df863f3SEric Blake         flags &= ~bs->supported_write_flags;
88808844473SKevin Wolf     } else if (drv->bdrv_co_writev) {
8894df863f3SEric Blake         assert(!bs->supported_write_flags);
89078a07294SKevin Wolf         ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov);
89108844473SKevin Wolf     } else {
89208844473SKevin Wolf         BlockAIOCB *acb;
89308844473SKevin Wolf         CoroutineIOCompletion co = {
89408844473SKevin Wolf             .coroutine = qemu_coroutine_self(),
89508844473SKevin Wolf         };
89608844473SKevin Wolf 
89708844473SKevin Wolf         acb = bs->drv->bdrv_aio_writev(bs, sector_num, qiov, nb_sectors,
89808844473SKevin Wolf                                        bdrv_co_io_em_complete, &co);
89908844473SKevin Wolf         if (acb == NULL) {
9003fb06697SKevin Wolf             ret = -EIO;
90108844473SKevin Wolf         } else {
90208844473SKevin Wolf             qemu_coroutine_yield();
9033fb06697SKevin Wolf             ret = co.ret;
90408844473SKevin Wolf         }
90578a07294SKevin Wolf     }
90678a07294SKevin Wolf 
9073fb06697SKevin Wolf emulate_flags:
9084df863f3SEric Blake     if (ret == 0 && (flags & BDRV_REQ_FUA)) {
90978a07294SKevin Wolf         ret = bdrv_co_flush(bs);
91078a07294SKevin Wolf     }
91178a07294SKevin Wolf 
91278a07294SKevin Wolf     return ret;
91378a07294SKevin Wolf }
91478a07294SKevin Wolf 
91529a298afSPavel Butsykin static int coroutine_fn
91629a298afSPavel Butsykin bdrv_driver_pwritev_compressed(BlockDriverState *bs, uint64_t offset,
91729a298afSPavel Butsykin                                uint64_t bytes, QEMUIOVector *qiov)
91829a298afSPavel Butsykin {
91929a298afSPavel Butsykin     BlockDriver *drv = bs->drv;
92029a298afSPavel Butsykin 
92129a298afSPavel Butsykin     if (!drv->bdrv_co_pwritev_compressed) {
92229a298afSPavel Butsykin         return -ENOTSUP;
92329a298afSPavel Butsykin     }
92429a298afSPavel Butsykin 
92529a298afSPavel Butsykin     return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov);
92629a298afSPavel Butsykin }
92729a298afSPavel Butsykin 
928*85c97ca7SKevin Wolf static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child,
929244483e6SKevin Wolf         int64_t offset, unsigned int bytes, QEMUIOVector *qiov)
93061007b31SStefan Hajnoczi {
931*85c97ca7SKevin Wolf     BlockDriverState *bs = child->bs;
932*85c97ca7SKevin Wolf 
93361007b31SStefan Hajnoczi     /* Perform I/O through a temporary buffer so that users who scribble over
93461007b31SStefan Hajnoczi      * their read buffer while the operation is in progress do not end up
93561007b31SStefan Hajnoczi      * modifying the image file.  This is critical for zero-copy guest I/O
93661007b31SStefan Hajnoczi      * where anything might happen inside guest memory.
93761007b31SStefan Hajnoczi      */
93861007b31SStefan Hajnoczi     void *bounce_buffer;
93961007b31SStefan Hajnoczi 
94061007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
94161007b31SStefan Hajnoczi     struct iovec iov;
94261007b31SStefan Hajnoczi     QEMUIOVector bounce_qiov;
943244483e6SKevin Wolf     int64_t cluster_offset;
944244483e6SKevin Wolf     unsigned int cluster_bytes;
94561007b31SStefan Hajnoczi     size_t skip_bytes;
94661007b31SStefan Hajnoczi     int ret;
94761007b31SStefan Hajnoczi 
94861007b31SStefan Hajnoczi     /* Cover entire cluster so no additional backing file I/O is required when
94961007b31SStefan Hajnoczi      * allocating cluster in the image file.
95061007b31SStefan Hajnoczi      */
951244483e6SKevin Wolf     bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes);
95261007b31SStefan Hajnoczi 
953244483e6SKevin Wolf     trace_bdrv_co_do_copy_on_readv(bs, offset, bytes,
954244483e6SKevin Wolf                                    cluster_offset, cluster_bytes);
95561007b31SStefan Hajnoczi 
956244483e6SKevin Wolf     iov.iov_len = cluster_bytes;
95761007b31SStefan Hajnoczi     iov.iov_base = bounce_buffer = qemu_try_blockalign(bs, iov.iov_len);
95861007b31SStefan Hajnoczi     if (bounce_buffer == NULL) {
95961007b31SStefan Hajnoczi         ret = -ENOMEM;
96061007b31SStefan Hajnoczi         goto err;
96161007b31SStefan Hajnoczi     }
96261007b31SStefan Hajnoczi 
96361007b31SStefan Hajnoczi     qemu_iovec_init_external(&bounce_qiov, &iov, 1);
96461007b31SStefan Hajnoczi 
965244483e6SKevin Wolf     ret = bdrv_driver_preadv(bs, cluster_offset, cluster_bytes,
966166fe960SKevin Wolf                              &bounce_qiov, 0);
96761007b31SStefan Hajnoczi     if (ret < 0) {
96861007b31SStefan Hajnoczi         goto err;
96961007b31SStefan Hajnoczi     }
97061007b31SStefan Hajnoczi 
971c1499a5eSEric Blake     if (drv->bdrv_co_pwrite_zeroes &&
97261007b31SStefan Hajnoczi         buffer_is_zero(bounce_buffer, iov.iov_len)) {
973a604fa2bSEric Blake         /* FIXME: Should we (perhaps conditionally) be setting
974a604fa2bSEric Blake          * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy
975a604fa2bSEric Blake          * that still correctly reads as zero? */
976244483e6SKevin Wolf         ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, cluster_bytes, 0);
97761007b31SStefan Hajnoczi     } else {
97861007b31SStefan Hajnoczi         /* This does not change the data on the disk, it is not necessary
97961007b31SStefan Hajnoczi          * to flush even in cache=writethrough mode.
98061007b31SStefan Hajnoczi          */
981244483e6SKevin Wolf         ret = bdrv_driver_pwritev(bs, cluster_offset, cluster_bytes,
98278a07294SKevin Wolf                                   &bounce_qiov, 0);
98361007b31SStefan Hajnoczi     }
98461007b31SStefan Hajnoczi 
98561007b31SStefan Hajnoczi     if (ret < 0) {
98661007b31SStefan Hajnoczi         /* It might be okay to ignore write errors for guest requests.  If this
98761007b31SStefan Hajnoczi          * is a deliberate copy-on-read then we don't want to ignore the error.
98861007b31SStefan Hajnoczi          * Simply report it in all cases.
98961007b31SStefan Hajnoczi          */
99061007b31SStefan Hajnoczi         goto err;
99161007b31SStefan Hajnoczi     }
99261007b31SStefan Hajnoczi 
993244483e6SKevin Wolf     skip_bytes = offset - cluster_offset;
994244483e6SKevin Wolf     qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes, bytes);
99561007b31SStefan Hajnoczi 
99661007b31SStefan Hajnoczi err:
99761007b31SStefan Hajnoczi     qemu_vfree(bounce_buffer);
99861007b31SStefan Hajnoczi     return ret;
99961007b31SStefan Hajnoczi }
100061007b31SStefan Hajnoczi 
100161007b31SStefan Hajnoczi /*
100261007b31SStefan Hajnoczi  * Forwards an already correctly aligned request to the BlockDriver. This
10031a62d0acSEric Blake  * handles copy on read, zeroing after EOF, and fragmentation of large
10041a62d0acSEric Blake  * reads; any other features must be implemented by the caller.
100561007b31SStefan Hajnoczi  */
1006*85c97ca7SKevin Wolf static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,
100761007b31SStefan Hajnoczi     BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
100861007b31SStefan Hajnoczi     int64_t align, QEMUIOVector *qiov, int flags)
100961007b31SStefan Hajnoczi {
1010*85c97ca7SKevin Wolf     BlockDriverState *bs = child->bs;
1011c9d20029SKevin Wolf     int64_t total_bytes, max_bytes;
10121a62d0acSEric Blake     int ret = 0;
10131a62d0acSEric Blake     uint64_t bytes_remaining = bytes;
10141a62d0acSEric Blake     int max_transfer;
101561007b31SStefan Hajnoczi 
101649c07526SKevin Wolf     assert(is_power_of_2(align));
101749c07526SKevin Wolf     assert((offset & (align - 1)) == 0);
101849c07526SKevin Wolf     assert((bytes & (align - 1)) == 0);
101961007b31SStefan Hajnoczi     assert(!qiov || bytes == qiov->size);
1020abb06c5aSDaniel P. Berrange     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
10211a62d0acSEric Blake     max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
10221a62d0acSEric Blake                                    align);
1023a604fa2bSEric Blake 
1024a604fa2bSEric Blake     /* TODO: We would need a per-BDS .supported_read_flags and
1025a604fa2bSEric Blake      * potential fallback support, if we ever implement any read flags
1026a604fa2bSEric Blake      * to pass through to drivers.  For now, there aren't any
1027a604fa2bSEric Blake      * passthrough flags.  */
1028a604fa2bSEric Blake     assert(!(flags & ~(BDRV_REQ_NO_SERIALISING | BDRV_REQ_COPY_ON_READ)));
102961007b31SStefan Hajnoczi 
103061007b31SStefan Hajnoczi     /* Handle Copy on Read and associated serialisation */
103161007b31SStefan Hajnoczi     if (flags & BDRV_REQ_COPY_ON_READ) {
103261007b31SStefan Hajnoczi         /* If we touch the same cluster it counts as an overlap.  This
103361007b31SStefan Hajnoczi          * guarantees that allocating writes will be serialized and not race
103461007b31SStefan Hajnoczi          * with each other for the same cluster.  For example, in copy-on-read
103561007b31SStefan Hajnoczi          * it ensures that the CoR read and write operations are atomic and
103661007b31SStefan Hajnoczi          * guest writes cannot interleave between them. */
103761007b31SStefan Hajnoczi         mark_request_serialising(req, bdrv_get_cluster_size(bs));
103861007b31SStefan Hajnoczi     }
103961007b31SStefan Hajnoczi 
104061408b25SFam Zheng     if (!(flags & BDRV_REQ_NO_SERIALISING)) {
104161007b31SStefan Hajnoczi         wait_serialising_requests(req);
104261408b25SFam Zheng     }
104361007b31SStefan Hajnoczi 
104461007b31SStefan Hajnoczi     if (flags & BDRV_REQ_COPY_ON_READ) {
104549c07526SKevin Wolf         int64_t start_sector = offset >> BDRV_SECTOR_BITS;
104649c07526SKevin Wolf         int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
104749c07526SKevin Wolf         unsigned int nb_sectors = end_sector - start_sector;
104861007b31SStefan Hajnoczi         int pnum;
104961007b31SStefan Hajnoczi 
105049c07526SKevin Wolf         ret = bdrv_is_allocated(bs, start_sector, nb_sectors, &pnum);
105161007b31SStefan Hajnoczi         if (ret < 0) {
105261007b31SStefan Hajnoczi             goto out;
105361007b31SStefan Hajnoczi         }
105461007b31SStefan Hajnoczi 
105561007b31SStefan Hajnoczi         if (!ret || pnum != nb_sectors) {
1056*85c97ca7SKevin Wolf             ret = bdrv_co_do_copy_on_readv(child, offset, bytes, qiov);
105761007b31SStefan Hajnoczi             goto out;
105861007b31SStefan Hajnoczi         }
105961007b31SStefan Hajnoczi     }
106061007b31SStefan Hajnoczi 
10611a62d0acSEric Blake     /* Forward the request to the BlockDriver, possibly fragmenting it */
106249c07526SKevin Wolf     total_bytes = bdrv_getlength(bs);
106349c07526SKevin Wolf     if (total_bytes < 0) {
106449c07526SKevin Wolf         ret = total_bytes;
106561007b31SStefan Hajnoczi         goto out;
106661007b31SStefan Hajnoczi     }
106761007b31SStefan Hajnoczi 
106849c07526SKevin Wolf     max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align);
10691a62d0acSEric Blake     if (bytes <= max_bytes && bytes <= max_transfer) {
1070166fe960SKevin Wolf         ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0);
10711a62d0acSEric Blake         goto out;
107261007b31SStefan Hajnoczi     }
107361007b31SStefan Hajnoczi 
10741a62d0acSEric Blake     while (bytes_remaining) {
10751a62d0acSEric Blake         int num;
10761a62d0acSEric Blake 
10771a62d0acSEric Blake         if (max_bytes) {
10781a62d0acSEric Blake             QEMUIOVector local_qiov;
10791a62d0acSEric Blake 
10801a62d0acSEric Blake             num = MIN(bytes_remaining, MIN(max_bytes, max_transfer));
10811a62d0acSEric Blake             assert(num);
10821a62d0acSEric Blake             qemu_iovec_init(&local_qiov, qiov->niov);
10831a62d0acSEric Blake             qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num);
10841a62d0acSEric Blake 
10851a62d0acSEric Blake             ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining,
10861a62d0acSEric Blake                                      num, &local_qiov, 0);
10871a62d0acSEric Blake             max_bytes -= num;
10881a62d0acSEric Blake             qemu_iovec_destroy(&local_qiov);
10891a62d0acSEric Blake         } else {
10901a62d0acSEric Blake             num = bytes_remaining;
10911a62d0acSEric Blake             ret = qemu_iovec_memset(qiov, bytes - bytes_remaining, 0,
10921a62d0acSEric Blake                                     bytes_remaining);
10931a62d0acSEric Blake         }
10941a62d0acSEric Blake         if (ret < 0) {
10951a62d0acSEric Blake             goto out;
10961a62d0acSEric Blake         }
10971a62d0acSEric Blake         bytes_remaining -= num;
109861007b31SStefan Hajnoczi     }
109961007b31SStefan Hajnoczi 
110061007b31SStefan Hajnoczi out:
11011a62d0acSEric Blake     return ret < 0 ? ret : 0;
110261007b31SStefan Hajnoczi }
110361007b31SStefan Hajnoczi 
110461007b31SStefan Hajnoczi /*
110561007b31SStefan Hajnoczi  * Handle a read request in coroutine context
110661007b31SStefan Hajnoczi  */
1107a03ef88fSKevin Wolf int coroutine_fn bdrv_co_preadv(BdrvChild *child,
110861007b31SStefan Hajnoczi     int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
110961007b31SStefan Hajnoczi     BdrvRequestFlags flags)
111061007b31SStefan Hajnoczi {
1111a03ef88fSKevin Wolf     BlockDriverState *bs = child->bs;
111261007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
111361007b31SStefan Hajnoczi     BdrvTrackedRequest req;
111461007b31SStefan Hajnoczi 
1115a5b8dd2cSEric Blake     uint64_t align = bs->bl.request_alignment;
111661007b31SStefan Hajnoczi     uint8_t *head_buf = NULL;
111761007b31SStefan Hajnoczi     uint8_t *tail_buf = NULL;
111861007b31SStefan Hajnoczi     QEMUIOVector local_qiov;
111961007b31SStefan Hajnoczi     bool use_local_qiov = false;
112061007b31SStefan Hajnoczi     int ret;
112161007b31SStefan Hajnoczi 
112261007b31SStefan Hajnoczi     if (!drv) {
112361007b31SStefan Hajnoczi         return -ENOMEDIUM;
112461007b31SStefan Hajnoczi     }
112561007b31SStefan Hajnoczi 
112661007b31SStefan Hajnoczi     ret = bdrv_check_byte_request(bs, offset, bytes);
112761007b31SStefan Hajnoczi     if (ret < 0) {
112861007b31SStefan Hajnoczi         return ret;
112961007b31SStefan Hajnoczi     }
113061007b31SStefan Hajnoczi 
113199723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
113299723548SPaolo Bonzini 
11339568b511SWen Congyang     /* Don't do copy-on-read if we read data before write operation */
113461408b25SFam Zheng     if (bs->copy_on_read && !(flags & BDRV_REQ_NO_SERIALISING)) {
113561007b31SStefan Hajnoczi         flags |= BDRV_REQ_COPY_ON_READ;
113661007b31SStefan Hajnoczi     }
113761007b31SStefan Hajnoczi 
113861007b31SStefan Hajnoczi     /* Align read if necessary by padding qiov */
113961007b31SStefan Hajnoczi     if (offset & (align - 1)) {
114061007b31SStefan Hajnoczi         head_buf = qemu_blockalign(bs, align);
114161007b31SStefan Hajnoczi         qemu_iovec_init(&local_qiov, qiov->niov + 2);
114261007b31SStefan Hajnoczi         qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
114361007b31SStefan Hajnoczi         qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
114461007b31SStefan Hajnoczi         use_local_qiov = true;
114561007b31SStefan Hajnoczi 
114661007b31SStefan Hajnoczi         bytes += offset & (align - 1);
114761007b31SStefan Hajnoczi         offset = offset & ~(align - 1);
114861007b31SStefan Hajnoczi     }
114961007b31SStefan Hajnoczi 
115061007b31SStefan Hajnoczi     if ((offset + bytes) & (align - 1)) {
115161007b31SStefan Hajnoczi         if (!use_local_qiov) {
115261007b31SStefan Hajnoczi             qemu_iovec_init(&local_qiov, qiov->niov + 1);
115361007b31SStefan Hajnoczi             qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
115461007b31SStefan Hajnoczi             use_local_qiov = true;
115561007b31SStefan Hajnoczi         }
115661007b31SStefan Hajnoczi         tail_buf = qemu_blockalign(bs, align);
115761007b31SStefan Hajnoczi         qemu_iovec_add(&local_qiov, tail_buf,
115861007b31SStefan Hajnoczi                        align - ((offset + bytes) & (align - 1)));
115961007b31SStefan Hajnoczi 
116061007b31SStefan Hajnoczi         bytes = ROUND_UP(bytes, align);
116161007b31SStefan Hajnoczi     }
116261007b31SStefan Hajnoczi 
1163ebde595cSFam Zheng     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
1164*85c97ca7SKevin Wolf     ret = bdrv_aligned_preadv(child, &req, offset, bytes, align,
116561007b31SStefan Hajnoczi                               use_local_qiov ? &local_qiov : qiov,
116661007b31SStefan Hajnoczi                               flags);
116761007b31SStefan Hajnoczi     tracked_request_end(&req);
116899723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
116961007b31SStefan Hajnoczi 
117061007b31SStefan Hajnoczi     if (use_local_qiov) {
117161007b31SStefan Hajnoczi         qemu_iovec_destroy(&local_qiov);
117261007b31SStefan Hajnoczi         qemu_vfree(head_buf);
117361007b31SStefan Hajnoczi         qemu_vfree(tail_buf);
117461007b31SStefan Hajnoczi     }
117561007b31SStefan Hajnoczi 
117661007b31SStefan Hajnoczi     return ret;
117761007b31SStefan Hajnoczi }
117861007b31SStefan Hajnoczi 
1179adad6496SKevin Wolf static int coroutine_fn bdrv_co_do_readv(BdrvChild *child,
118061007b31SStefan Hajnoczi     int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
118161007b31SStefan Hajnoczi     BdrvRequestFlags flags)
118261007b31SStefan Hajnoczi {
118361007b31SStefan Hajnoczi     if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
118461007b31SStefan Hajnoczi         return -EINVAL;
118561007b31SStefan Hajnoczi     }
118661007b31SStefan Hajnoczi 
1187a03ef88fSKevin Wolf     return bdrv_co_preadv(child, sector_num << BDRV_SECTOR_BITS,
118861007b31SStefan Hajnoczi                           nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
118961007b31SStefan Hajnoczi }
119061007b31SStefan Hajnoczi 
119128b04a8fSKevin Wolf int coroutine_fn bdrv_co_readv(BdrvChild *child, int64_t sector_num,
119261007b31SStefan Hajnoczi                                int nb_sectors, QEMUIOVector *qiov)
119361007b31SStefan Hajnoczi {
119428b04a8fSKevin Wolf     trace_bdrv_co_readv(child->bs, sector_num, nb_sectors);
119561007b31SStefan Hajnoczi 
1196adad6496SKevin Wolf     return bdrv_co_do_readv(child, sector_num, nb_sectors, qiov, 0);
119761007b31SStefan Hajnoczi }
119861007b31SStefan Hajnoczi 
11995def6b80SEric Blake /* Maximum buffer for write zeroes fallback, in bytes */
12005def6b80SEric Blake #define MAX_WRITE_ZEROES_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS)
120161007b31SStefan Hajnoczi 
1202d05aa8bbSEric Blake static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
1203d05aa8bbSEric Blake     int64_t offset, int count, BdrvRequestFlags flags)
120461007b31SStefan Hajnoczi {
120561007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
120661007b31SStefan Hajnoczi     QEMUIOVector qiov;
120761007b31SStefan Hajnoczi     struct iovec iov = {0};
120861007b31SStefan Hajnoczi     int ret = 0;
1209465fe887SEric Blake     bool need_flush = false;
1210443668caSDenis V. Lunev     int head = 0;
1211443668caSDenis V. Lunev     int tail = 0;
121261007b31SStefan Hajnoczi 
1213cf081fcaSEric Blake     int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, INT_MAX);
1214a5b8dd2cSEric Blake     int alignment = MAX(bs->bl.pwrite_zeroes_alignment,
1215a5b8dd2cSEric Blake                         bs->bl.request_alignment);
1216b2f95feeSEric Blake     int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer,
1217b2f95feeSEric Blake                                     MAX_WRITE_ZEROES_BOUNCE_BUFFER);
1218cf081fcaSEric Blake 
1219b8d0a980SEric Blake     assert(alignment % bs->bl.request_alignment == 0);
1220b8d0a980SEric Blake     head = offset % alignment;
1221b8d0a980SEric Blake     tail = (offset + count) % alignment;
1222b8d0a980SEric Blake     max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment);
1223b8d0a980SEric Blake     assert(max_write_zeroes >= bs->bl.request_alignment);
122461007b31SStefan Hajnoczi 
1225d05aa8bbSEric Blake     while (count > 0 && !ret) {
1226d05aa8bbSEric Blake         int num = count;
122761007b31SStefan Hajnoczi 
122861007b31SStefan Hajnoczi         /* Align request.  Block drivers can expect the "bulk" of the request
1229443668caSDenis V. Lunev          * to be aligned, and that unaligned requests do not cross cluster
1230443668caSDenis V. Lunev          * boundaries.
123161007b31SStefan Hajnoczi          */
1232443668caSDenis V. Lunev         if (head) {
1233b2f95feeSEric Blake             /* Make a small request up to the first aligned sector. For
1234b2f95feeSEric Blake              * convenience, limit this request to max_transfer even if
1235b2f95feeSEric Blake              * we don't need to fall back to writes.  */
1236b2f95feeSEric Blake             num = MIN(MIN(count, max_transfer), alignment - head);
1237b2f95feeSEric Blake             head = (head + num) % alignment;
1238b2f95feeSEric Blake             assert(num < max_write_zeroes);
1239d05aa8bbSEric Blake         } else if (tail && num > alignment) {
1240443668caSDenis V. Lunev             /* Shorten the request to the last aligned sector.  */
1241443668caSDenis V. Lunev             num -= tail;
124261007b31SStefan Hajnoczi         }
124361007b31SStefan Hajnoczi 
124461007b31SStefan Hajnoczi         /* limit request size */
124561007b31SStefan Hajnoczi         if (num > max_write_zeroes) {
124661007b31SStefan Hajnoczi             num = max_write_zeroes;
124761007b31SStefan Hajnoczi         }
124861007b31SStefan Hajnoczi 
124961007b31SStefan Hajnoczi         ret = -ENOTSUP;
125061007b31SStefan Hajnoczi         /* First try the efficient write zeroes operation */
1251d05aa8bbSEric Blake         if (drv->bdrv_co_pwrite_zeroes) {
1252d05aa8bbSEric Blake             ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num,
1253d05aa8bbSEric Blake                                              flags & bs->supported_zero_flags);
1254d05aa8bbSEric Blake             if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) &&
1255d05aa8bbSEric Blake                 !(bs->supported_zero_flags & BDRV_REQ_FUA)) {
1256d05aa8bbSEric Blake                 need_flush = true;
1257d05aa8bbSEric Blake             }
1258465fe887SEric Blake         } else {
1259465fe887SEric Blake             assert(!bs->supported_zero_flags);
126061007b31SStefan Hajnoczi         }
126161007b31SStefan Hajnoczi 
126261007b31SStefan Hajnoczi         if (ret == -ENOTSUP) {
126361007b31SStefan Hajnoczi             /* Fall back to bounce buffer if write zeroes is unsupported */
1264465fe887SEric Blake             BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;
1265465fe887SEric Blake 
1266465fe887SEric Blake             if ((flags & BDRV_REQ_FUA) &&
1267465fe887SEric Blake                 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1268465fe887SEric Blake                 /* No need for bdrv_driver_pwrite() to do a fallback
1269465fe887SEric Blake                  * flush on each chunk; use just one at the end */
1270465fe887SEric Blake                 write_flags &= ~BDRV_REQ_FUA;
1271465fe887SEric Blake                 need_flush = true;
1272465fe887SEric Blake             }
12735def6b80SEric Blake             num = MIN(num, max_transfer);
1274d05aa8bbSEric Blake             iov.iov_len = num;
127561007b31SStefan Hajnoczi             if (iov.iov_base == NULL) {
1276d05aa8bbSEric Blake                 iov.iov_base = qemu_try_blockalign(bs, num);
127761007b31SStefan Hajnoczi                 if (iov.iov_base == NULL) {
127861007b31SStefan Hajnoczi                     ret = -ENOMEM;
127961007b31SStefan Hajnoczi                     goto fail;
128061007b31SStefan Hajnoczi                 }
1281d05aa8bbSEric Blake                 memset(iov.iov_base, 0, num);
128261007b31SStefan Hajnoczi             }
128361007b31SStefan Hajnoczi             qemu_iovec_init_external(&qiov, &iov, 1);
128461007b31SStefan Hajnoczi 
1285d05aa8bbSEric Blake             ret = bdrv_driver_pwritev(bs, offset, num, &qiov, write_flags);
128661007b31SStefan Hajnoczi 
128761007b31SStefan Hajnoczi             /* Keep bounce buffer around if it is big enough for all
128861007b31SStefan Hajnoczi              * all future requests.
128961007b31SStefan Hajnoczi              */
12905def6b80SEric Blake             if (num < max_transfer) {
129161007b31SStefan Hajnoczi                 qemu_vfree(iov.iov_base);
129261007b31SStefan Hajnoczi                 iov.iov_base = NULL;
129361007b31SStefan Hajnoczi             }
129461007b31SStefan Hajnoczi         }
129561007b31SStefan Hajnoczi 
1296d05aa8bbSEric Blake         offset += num;
1297d05aa8bbSEric Blake         count -= num;
129861007b31SStefan Hajnoczi     }
129961007b31SStefan Hajnoczi 
130061007b31SStefan Hajnoczi fail:
1301465fe887SEric Blake     if (ret == 0 && need_flush) {
1302465fe887SEric Blake         ret = bdrv_co_flush(bs);
1303465fe887SEric Blake     }
130461007b31SStefan Hajnoczi     qemu_vfree(iov.iov_base);
130561007b31SStefan Hajnoczi     return ret;
130661007b31SStefan Hajnoczi }
130761007b31SStefan Hajnoczi 
130861007b31SStefan Hajnoczi /*
130904ed95f4SEric Blake  * Forwards an already correctly aligned write request to the BlockDriver,
131004ed95f4SEric Blake  * after possibly fragmenting it.
131161007b31SStefan Hajnoczi  */
1312*85c97ca7SKevin Wolf static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child,
131361007b31SStefan Hajnoczi     BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
1314cff86b38SEric Blake     int64_t align, QEMUIOVector *qiov, int flags)
131561007b31SStefan Hajnoczi {
1316*85c97ca7SKevin Wolf     BlockDriverState *bs = child->bs;
131761007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
131861007b31SStefan Hajnoczi     bool waited;
131961007b31SStefan Hajnoczi     int ret;
132061007b31SStefan Hajnoczi 
13219896c876SKevin Wolf     int64_t start_sector = offset >> BDRV_SECTOR_BITS;
13229896c876SKevin Wolf     int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
132304ed95f4SEric Blake     uint64_t bytes_remaining = bytes;
132404ed95f4SEric Blake     int max_transfer;
132561007b31SStefan Hajnoczi 
1326cff86b38SEric Blake     assert(is_power_of_2(align));
1327cff86b38SEric Blake     assert((offset & (align - 1)) == 0);
1328cff86b38SEric Blake     assert((bytes & (align - 1)) == 0);
132961007b31SStefan Hajnoczi     assert(!qiov || bytes == qiov->size);
1330abb06c5aSDaniel P. Berrange     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1331fa166538SEric Blake     assert(!(flags & ~BDRV_REQ_MASK));
133204ed95f4SEric Blake     max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
133304ed95f4SEric Blake                                    align);
133461007b31SStefan Hajnoczi 
133561007b31SStefan Hajnoczi     waited = wait_serialising_requests(req);
133661007b31SStefan Hajnoczi     assert(!waited || !req->serialising);
133761007b31SStefan Hajnoczi     assert(req->overlap_offset <= offset);
133861007b31SStefan Hajnoczi     assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
133961007b31SStefan Hajnoczi 
134061007b31SStefan Hajnoczi     ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req);
134161007b31SStefan Hajnoczi 
134261007b31SStefan Hajnoczi     if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
1343c1499a5eSEric Blake         !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes &&
134461007b31SStefan Hajnoczi         qemu_iovec_is_zero(qiov)) {
134561007b31SStefan Hajnoczi         flags |= BDRV_REQ_ZERO_WRITE;
134661007b31SStefan Hajnoczi         if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
134761007b31SStefan Hajnoczi             flags |= BDRV_REQ_MAY_UNMAP;
134861007b31SStefan Hajnoczi         }
134961007b31SStefan Hajnoczi     }
135061007b31SStefan Hajnoczi 
135161007b31SStefan Hajnoczi     if (ret < 0) {
135261007b31SStefan Hajnoczi         /* Do nothing, write notifier decided to fail this request */
135361007b31SStefan Hajnoczi     } else if (flags & BDRV_REQ_ZERO_WRITE) {
13549a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
13559896c876SKevin Wolf         ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags);
13563ea1a091SPavel Butsykin     } else if (flags & BDRV_REQ_WRITE_COMPRESSED) {
13573ea1a091SPavel Butsykin         ret = bdrv_driver_pwritev_compressed(bs, offset, bytes, qiov);
135804ed95f4SEric Blake     } else if (bytes <= max_transfer) {
13599a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV);
136078a07294SKevin Wolf         ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, flags);
136104ed95f4SEric Blake     } else {
136204ed95f4SEric Blake         bdrv_debug_event(bs, BLKDBG_PWRITEV);
136304ed95f4SEric Blake         while (bytes_remaining) {
136404ed95f4SEric Blake             int num = MIN(bytes_remaining, max_transfer);
136504ed95f4SEric Blake             QEMUIOVector local_qiov;
136604ed95f4SEric Blake             int local_flags = flags;
136704ed95f4SEric Blake 
136804ed95f4SEric Blake             assert(num);
136904ed95f4SEric Blake             if (num < bytes_remaining && (flags & BDRV_REQ_FUA) &&
137004ed95f4SEric Blake                 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
137104ed95f4SEric Blake                 /* If FUA is going to be emulated by flush, we only
137204ed95f4SEric Blake                  * need to flush on the last iteration */
137304ed95f4SEric Blake                 local_flags &= ~BDRV_REQ_FUA;
137404ed95f4SEric Blake             }
137504ed95f4SEric Blake             qemu_iovec_init(&local_qiov, qiov->niov);
137604ed95f4SEric Blake             qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num);
137704ed95f4SEric Blake 
137804ed95f4SEric Blake             ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining,
137904ed95f4SEric Blake                                       num, &local_qiov, local_flags);
138004ed95f4SEric Blake             qemu_iovec_destroy(&local_qiov);
138104ed95f4SEric Blake             if (ret < 0) {
138204ed95f4SEric Blake                 break;
138304ed95f4SEric Blake             }
138404ed95f4SEric Blake             bytes_remaining -= num;
138504ed95f4SEric Blake         }
138661007b31SStefan Hajnoczi     }
13879a4f4c31SKevin Wolf     bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
138861007b31SStefan Hajnoczi 
13893ff2f67aSEvgeny Yakovlev     ++bs->write_gen;
13909896c876SKevin Wolf     bdrv_set_dirty(bs, start_sector, end_sector - start_sector);
139161007b31SStefan Hajnoczi 
139253d8f9d8SMax Reitz     if (bs->wr_highest_offset < offset + bytes) {
139353d8f9d8SMax Reitz         bs->wr_highest_offset = offset + bytes;
139453d8f9d8SMax Reitz     }
139561007b31SStefan Hajnoczi 
139661007b31SStefan Hajnoczi     if (ret >= 0) {
13979896c876SKevin Wolf         bs->total_sectors = MAX(bs->total_sectors, end_sector);
139804ed95f4SEric Blake         ret = 0;
139961007b31SStefan Hajnoczi     }
140061007b31SStefan Hajnoczi 
140161007b31SStefan Hajnoczi     return ret;
140261007b31SStefan Hajnoczi }
140361007b31SStefan Hajnoczi 
1404*85c97ca7SKevin Wolf static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child,
14059eeb6dd1SFam Zheng                                                 int64_t offset,
14069eeb6dd1SFam Zheng                                                 unsigned int bytes,
14079eeb6dd1SFam Zheng                                                 BdrvRequestFlags flags,
14089eeb6dd1SFam Zheng                                                 BdrvTrackedRequest *req)
14099eeb6dd1SFam Zheng {
1410*85c97ca7SKevin Wolf     BlockDriverState *bs = child->bs;
14119eeb6dd1SFam Zheng     uint8_t *buf = NULL;
14129eeb6dd1SFam Zheng     QEMUIOVector local_qiov;
14139eeb6dd1SFam Zheng     struct iovec iov;
1414a5b8dd2cSEric Blake     uint64_t align = bs->bl.request_alignment;
14159eeb6dd1SFam Zheng     unsigned int head_padding_bytes, tail_padding_bytes;
14169eeb6dd1SFam Zheng     int ret = 0;
14179eeb6dd1SFam Zheng 
14189eeb6dd1SFam Zheng     head_padding_bytes = offset & (align - 1);
14199eeb6dd1SFam Zheng     tail_padding_bytes = align - ((offset + bytes) & (align - 1));
14209eeb6dd1SFam Zheng 
14219eeb6dd1SFam Zheng 
14229eeb6dd1SFam Zheng     assert(flags & BDRV_REQ_ZERO_WRITE);
14239eeb6dd1SFam Zheng     if (head_padding_bytes || tail_padding_bytes) {
14249eeb6dd1SFam Zheng         buf = qemu_blockalign(bs, align);
14259eeb6dd1SFam Zheng         iov = (struct iovec) {
14269eeb6dd1SFam Zheng             .iov_base   = buf,
14279eeb6dd1SFam Zheng             .iov_len    = align,
14289eeb6dd1SFam Zheng         };
14299eeb6dd1SFam Zheng         qemu_iovec_init_external(&local_qiov, &iov, 1);
14309eeb6dd1SFam Zheng     }
14319eeb6dd1SFam Zheng     if (head_padding_bytes) {
14329eeb6dd1SFam Zheng         uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes);
14339eeb6dd1SFam Zheng 
14349eeb6dd1SFam Zheng         /* RMW the unaligned part before head. */
14359eeb6dd1SFam Zheng         mark_request_serialising(req, align);
14369eeb6dd1SFam Zheng         wait_serialising_requests(req);
14379a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1438*85c97ca7SKevin Wolf         ret = bdrv_aligned_preadv(child, req, offset & ~(align - 1), align,
14399eeb6dd1SFam Zheng                                   align, &local_qiov, 0);
14409eeb6dd1SFam Zheng         if (ret < 0) {
14419eeb6dd1SFam Zheng             goto fail;
14429eeb6dd1SFam Zheng         }
14439a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
14449eeb6dd1SFam Zheng 
14459eeb6dd1SFam Zheng         memset(buf + head_padding_bytes, 0, zero_bytes);
1446*85c97ca7SKevin Wolf         ret = bdrv_aligned_pwritev(child, req, offset & ~(align - 1), align,
1447cff86b38SEric Blake                                    align, &local_qiov,
14489eeb6dd1SFam Zheng                                    flags & ~BDRV_REQ_ZERO_WRITE);
14499eeb6dd1SFam Zheng         if (ret < 0) {
14509eeb6dd1SFam Zheng             goto fail;
14519eeb6dd1SFam Zheng         }
14529eeb6dd1SFam Zheng         offset += zero_bytes;
14539eeb6dd1SFam Zheng         bytes -= zero_bytes;
14549eeb6dd1SFam Zheng     }
14559eeb6dd1SFam Zheng 
14569eeb6dd1SFam Zheng     assert(!bytes || (offset & (align - 1)) == 0);
14579eeb6dd1SFam Zheng     if (bytes >= align) {
14589eeb6dd1SFam Zheng         /* Write the aligned part in the middle. */
14599eeb6dd1SFam Zheng         uint64_t aligned_bytes = bytes & ~(align - 1);
1460*85c97ca7SKevin Wolf         ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align,
14619eeb6dd1SFam Zheng                                    NULL, flags);
14629eeb6dd1SFam Zheng         if (ret < 0) {
14639eeb6dd1SFam Zheng             goto fail;
14649eeb6dd1SFam Zheng         }
14659eeb6dd1SFam Zheng         bytes -= aligned_bytes;
14669eeb6dd1SFam Zheng         offset += aligned_bytes;
14679eeb6dd1SFam Zheng     }
14689eeb6dd1SFam Zheng 
14699eeb6dd1SFam Zheng     assert(!bytes || (offset & (align - 1)) == 0);
14709eeb6dd1SFam Zheng     if (bytes) {
14719eeb6dd1SFam Zheng         assert(align == tail_padding_bytes + bytes);
14729eeb6dd1SFam Zheng         /* RMW the unaligned part after tail. */
14739eeb6dd1SFam Zheng         mark_request_serialising(req, align);
14749eeb6dd1SFam Zheng         wait_serialising_requests(req);
14759a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1476*85c97ca7SKevin Wolf         ret = bdrv_aligned_preadv(child, req, offset, align,
14779eeb6dd1SFam Zheng                                   align, &local_qiov, 0);
14789eeb6dd1SFam Zheng         if (ret < 0) {
14799eeb6dd1SFam Zheng             goto fail;
14809eeb6dd1SFam Zheng         }
14819a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
14829eeb6dd1SFam Zheng 
14839eeb6dd1SFam Zheng         memset(buf, 0, bytes);
1484*85c97ca7SKevin Wolf         ret = bdrv_aligned_pwritev(child, req, offset, align, align,
14859eeb6dd1SFam Zheng                                    &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE);
14869eeb6dd1SFam Zheng     }
14879eeb6dd1SFam Zheng fail:
14889eeb6dd1SFam Zheng     qemu_vfree(buf);
14899eeb6dd1SFam Zheng     return ret;
14909eeb6dd1SFam Zheng 
14919eeb6dd1SFam Zheng }
14929eeb6dd1SFam Zheng 
149361007b31SStefan Hajnoczi /*
149461007b31SStefan Hajnoczi  * Handle a write request in coroutine context
149561007b31SStefan Hajnoczi  */
1496a03ef88fSKevin Wolf int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
149761007b31SStefan Hajnoczi     int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
149861007b31SStefan Hajnoczi     BdrvRequestFlags flags)
149961007b31SStefan Hajnoczi {
1500a03ef88fSKevin Wolf     BlockDriverState *bs = child->bs;
150161007b31SStefan Hajnoczi     BdrvTrackedRequest req;
1502a5b8dd2cSEric Blake     uint64_t align = bs->bl.request_alignment;
150361007b31SStefan Hajnoczi     uint8_t *head_buf = NULL;
150461007b31SStefan Hajnoczi     uint8_t *tail_buf = NULL;
150561007b31SStefan Hajnoczi     QEMUIOVector local_qiov;
150661007b31SStefan Hajnoczi     bool use_local_qiov = false;
150761007b31SStefan Hajnoczi     int ret;
150861007b31SStefan Hajnoczi 
150961007b31SStefan Hajnoczi     if (!bs->drv) {
151061007b31SStefan Hajnoczi         return -ENOMEDIUM;
151161007b31SStefan Hajnoczi     }
151261007b31SStefan Hajnoczi     if (bs->read_only) {
1513eaf5fe2dSPaolo Bonzini         return -EPERM;
151461007b31SStefan Hajnoczi     }
151504c01a5cSKevin Wolf     assert(!(bs->open_flags & BDRV_O_INACTIVE));
151661007b31SStefan Hajnoczi 
151761007b31SStefan Hajnoczi     ret = bdrv_check_byte_request(bs, offset, bytes);
151861007b31SStefan Hajnoczi     if (ret < 0) {
151961007b31SStefan Hajnoczi         return ret;
152061007b31SStefan Hajnoczi     }
152161007b31SStefan Hajnoczi 
152299723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
152361007b31SStefan Hajnoczi     /*
152461007b31SStefan Hajnoczi      * Align write if necessary by performing a read-modify-write cycle.
152561007b31SStefan Hajnoczi      * Pad qiov with the read parts and be sure to have a tracked request not
152661007b31SStefan Hajnoczi      * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle.
152761007b31SStefan Hajnoczi      */
1528ebde595cSFam Zheng     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
152961007b31SStefan Hajnoczi 
15309eeb6dd1SFam Zheng     if (!qiov) {
1531*85c97ca7SKevin Wolf         ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req);
15329eeb6dd1SFam Zheng         goto out;
15339eeb6dd1SFam Zheng     }
15349eeb6dd1SFam Zheng 
153561007b31SStefan Hajnoczi     if (offset & (align - 1)) {
153661007b31SStefan Hajnoczi         QEMUIOVector head_qiov;
153761007b31SStefan Hajnoczi         struct iovec head_iov;
153861007b31SStefan Hajnoczi 
153961007b31SStefan Hajnoczi         mark_request_serialising(&req, align);
154061007b31SStefan Hajnoczi         wait_serialising_requests(&req);
154161007b31SStefan Hajnoczi 
154261007b31SStefan Hajnoczi         head_buf = qemu_blockalign(bs, align);
154361007b31SStefan Hajnoczi         head_iov = (struct iovec) {
154461007b31SStefan Hajnoczi             .iov_base   = head_buf,
154561007b31SStefan Hajnoczi             .iov_len    = align,
154661007b31SStefan Hajnoczi         };
154761007b31SStefan Hajnoczi         qemu_iovec_init_external(&head_qiov, &head_iov, 1);
154861007b31SStefan Hajnoczi 
15499a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1550*85c97ca7SKevin Wolf         ret = bdrv_aligned_preadv(child, &req, offset & ~(align - 1), align,
155161007b31SStefan Hajnoczi                                   align, &head_qiov, 0);
155261007b31SStefan Hajnoczi         if (ret < 0) {
155361007b31SStefan Hajnoczi             goto fail;
155461007b31SStefan Hajnoczi         }
15559a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
155661007b31SStefan Hajnoczi 
155761007b31SStefan Hajnoczi         qemu_iovec_init(&local_qiov, qiov->niov + 2);
155861007b31SStefan Hajnoczi         qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
155961007b31SStefan Hajnoczi         qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
156061007b31SStefan Hajnoczi         use_local_qiov = true;
156161007b31SStefan Hajnoczi 
156261007b31SStefan Hajnoczi         bytes += offset & (align - 1);
156361007b31SStefan Hajnoczi         offset = offset & ~(align - 1);
1564117bc3faSPeter Lieven 
1565117bc3faSPeter Lieven         /* We have read the tail already if the request is smaller
1566117bc3faSPeter Lieven          * than one aligned block.
1567117bc3faSPeter Lieven          */
1568117bc3faSPeter Lieven         if (bytes < align) {
1569117bc3faSPeter Lieven             qemu_iovec_add(&local_qiov, head_buf + bytes, align - bytes);
1570117bc3faSPeter Lieven             bytes = align;
1571117bc3faSPeter Lieven         }
157261007b31SStefan Hajnoczi     }
157361007b31SStefan Hajnoczi 
157461007b31SStefan Hajnoczi     if ((offset + bytes) & (align - 1)) {
157561007b31SStefan Hajnoczi         QEMUIOVector tail_qiov;
157661007b31SStefan Hajnoczi         struct iovec tail_iov;
157761007b31SStefan Hajnoczi         size_t tail_bytes;
157861007b31SStefan Hajnoczi         bool waited;
157961007b31SStefan Hajnoczi 
158061007b31SStefan Hajnoczi         mark_request_serialising(&req, align);
158161007b31SStefan Hajnoczi         waited = wait_serialising_requests(&req);
158261007b31SStefan Hajnoczi         assert(!waited || !use_local_qiov);
158361007b31SStefan Hajnoczi 
158461007b31SStefan Hajnoczi         tail_buf = qemu_blockalign(bs, align);
158561007b31SStefan Hajnoczi         tail_iov = (struct iovec) {
158661007b31SStefan Hajnoczi             .iov_base   = tail_buf,
158761007b31SStefan Hajnoczi             .iov_len    = align,
158861007b31SStefan Hajnoczi         };
158961007b31SStefan Hajnoczi         qemu_iovec_init_external(&tail_qiov, &tail_iov, 1);
159061007b31SStefan Hajnoczi 
15919a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1592*85c97ca7SKevin Wolf         ret = bdrv_aligned_preadv(child, &req, (offset + bytes) & ~(align - 1),
1593*85c97ca7SKevin Wolf                                   align, align, &tail_qiov, 0);
159461007b31SStefan Hajnoczi         if (ret < 0) {
159561007b31SStefan Hajnoczi             goto fail;
159661007b31SStefan Hajnoczi         }
15979a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
159861007b31SStefan Hajnoczi 
159961007b31SStefan Hajnoczi         if (!use_local_qiov) {
160061007b31SStefan Hajnoczi             qemu_iovec_init(&local_qiov, qiov->niov + 1);
160161007b31SStefan Hajnoczi             qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
160261007b31SStefan Hajnoczi             use_local_qiov = true;
160361007b31SStefan Hajnoczi         }
160461007b31SStefan Hajnoczi 
160561007b31SStefan Hajnoczi         tail_bytes = (offset + bytes) & (align - 1);
160661007b31SStefan Hajnoczi         qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes);
160761007b31SStefan Hajnoczi 
160861007b31SStefan Hajnoczi         bytes = ROUND_UP(bytes, align);
160961007b31SStefan Hajnoczi     }
161061007b31SStefan Hajnoczi 
1611*85c97ca7SKevin Wolf     ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align,
161261007b31SStefan Hajnoczi                                use_local_qiov ? &local_qiov : qiov,
161361007b31SStefan Hajnoczi                                flags);
161461007b31SStefan Hajnoczi 
161561007b31SStefan Hajnoczi fail:
161661007b31SStefan Hajnoczi 
161761007b31SStefan Hajnoczi     if (use_local_qiov) {
161861007b31SStefan Hajnoczi         qemu_iovec_destroy(&local_qiov);
161961007b31SStefan Hajnoczi     }
162061007b31SStefan Hajnoczi     qemu_vfree(head_buf);
162161007b31SStefan Hajnoczi     qemu_vfree(tail_buf);
16229eeb6dd1SFam Zheng out:
16239eeb6dd1SFam Zheng     tracked_request_end(&req);
162499723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
162561007b31SStefan Hajnoczi     return ret;
162661007b31SStefan Hajnoczi }
162761007b31SStefan Hajnoczi 
1628adad6496SKevin Wolf static int coroutine_fn bdrv_co_do_writev(BdrvChild *child,
162961007b31SStefan Hajnoczi     int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
163061007b31SStefan Hajnoczi     BdrvRequestFlags flags)
163161007b31SStefan Hajnoczi {
163261007b31SStefan Hajnoczi     if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
163361007b31SStefan Hajnoczi         return -EINVAL;
163461007b31SStefan Hajnoczi     }
163561007b31SStefan Hajnoczi 
1636a03ef88fSKevin Wolf     return bdrv_co_pwritev(child, sector_num << BDRV_SECTOR_BITS,
163761007b31SStefan Hajnoczi                            nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
163861007b31SStefan Hajnoczi }
163961007b31SStefan Hajnoczi 
164025ec177dSKevin Wolf int coroutine_fn bdrv_co_writev(BdrvChild *child, int64_t sector_num,
164161007b31SStefan Hajnoczi     int nb_sectors, QEMUIOVector *qiov)
164261007b31SStefan Hajnoczi {
164325ec177dSKevin Wolf     trace_bdrv_co_writev(child->bs, sector_num, nb_sectors);
164461007b31SStefan Hajnoczi 
1645adad6496SKevin Wolf     return bdrv_co_do_writev(child, sector_num, nb_sectors, qiov, 0);
164661007b31SStefan Hajnoczi }
164761007b31SStefan Hajnoczi 
1648a03ef88fSKevin Wolf int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset,
1649a03ef88fSKevin Wolf                                        int count, BdrvRequestFlags flags)
165061007b31SStefan Hajnoczi {
1651a03ef88fSKevin Wolf     trace_bdrv_co_pwrite_zeroes(child->bs, offset, count, flags);
165261007b31SStefan Hajnoczi 
1653a03ef88fSKevin Wolf     if (!(child->bs->open_flags & BDRV_O_UNMAP)) {
165461007b31SStefan Hajnoczi         flags &= ~BDRV_REQ_MAY_UNMAP;
165561007b31SStefan Hajnoczi     }
165661007b31SStefan Hajnoczi 
1657a03ef88fSKevin Wolf     return bdrv_co_pwritev(child, offset, count, NULL,
165861007b31SStefan Hajnoczi                            BDRV_REQ_ZERO_WRITE | flags);
165961007b31SStefan Hajnoczi }
166061007b31SStefan Hajnoczi 
16614085f5c7SJohn Snow /*
16624085f5c7SJohn Snow  * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not.
16634085f5c7SJohn Snow  */
16644085f5c7SJohn Snow int bdrv_flush_all(void)
16654085f5c7SJohn Snow {
16664085f5c7SJohn Snow     BdrvNextIterator it;
16674085f5c7SJohn Snow     BlockDriverState *bs = NULL;
16684085f5c7SJohn Snow     int result = 0;
16694085f5c7SJohn Snow 
16704085f5c7SJohn Snow     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
16714085f5c7SJohn Snow         AioContext *aio_context = bdrv_get_aio_context(bs);
16724085f5c7SJohn Snow         int ret;
16734085f5c7SJohn Snow 
16744085f5c7SJohn Snow         aio_context_acquire(aio_context);
16754085f5c7SJohn Snow         ret = bdrv_flush(bs);
16764085f5c7SJohn Snow         if (ret < 0 && !result) {
16774085f5c7SJohn Snow             result = ret;
16784085f5c7SJohn Snow         }
16794085f5c7SJohn Snow         aio_context_release(aio_context);
16804085f5c7SJohn Snow     }
16814085f5c7SJohn Snow 
16824085f5c7SJohn Snow     return result;
16834085f5c7SJohn Snow }
16844085f5c7SJohn Snow 
16854085f5c7SJohn Snow 
168661007b31SStefan Hajnoczi typedef struct BdrvCoGetBlockStatusData {
168761007b31SStefan Hajnoczi     BlockDriverState *bs;
168861007b31SStefan Hajnoczi     BlockDriverState *base;
168967a0fd2aSFam Zheng     BlockDriverState **file;
169061007b31SStefan Hajnoczi     int64_t sector_num;
169161007b31SStefan Hajnoczi     int nb_sectors;
169261007b31SStefan Hajnoczi     int *pnum;
169361007b31SStefan Hajnoczi     int64_t ret;
169461007b31SStefan Hajnoczi     bool done;
169561007b31SStefan Hajnoczi } BdrvCoGetBlockStatusData;
169661007b31SStefan Hajnoczi 
169761007b31SStefan Hajnoczi /*
169861007b31SStefan Hajnoczi  * Returns the allocation status of the specified sectors.
169961007b31SStefan Hajnoczi  * Drivers not implementing the functionality are assumed to not support
170061007b31SStefan Hajnoczi  * backing files, hence all their sectors are reported as allocated.
170161007b31SStefan Hajnoczi  *
170261007b31SStefan Hajnoczi  * If 'sector_num' is beyond the end of the disk image the return value is 0
170361007b31SStefan Hajnoczi  * and 'pnum' is set to 0.
170461007b31SStefan Hajnoczi  *
170561007b31SStefan Hajnoczi  * 'pnum' is set to the number of sectors (including and immediately following
170661007b31SStefan Hajnoczi  * the specified sector) that are known to be in the same
170761007b31SStefan Hajnoczi  * allocated/unallocated state.
170861007b31SStefan Hajnoczi  *
170961007b31SStefan Hajnoczi  * 'nb_sectors' is the max value 'pnum' should be set to.  If nb_sectors goes
171061007b31SStefan Hajnoczi  * beyond the end of the disk image it will be clamped.
171167a0fd2aSFam Zheng  *
171267a0fd2aSFam Zheng  * If returned value is positive and BDRV_BLOCK_OFFSET_VALID bit is set, 'file'
171367a0fd2aSFam Zheng  * points to the BDS which the sector range is allocated in.
171461007b31SStefan Hajnoczi  */
171561007b31SStefan Hajnoczi static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
171661007b31SStefan Hajnoczi                                                      int64_t sector_num,
171767a0fd2aSFam Zheng                                                      int nb_sectors, int *pnum,
171867a0fd2aSFam Zheng                                                      BlockDriverState **file)
171961007b31SStefan Hajnoczi {
172061007b31SStefan Hajnoczi     int64_t total_sectors;
172161007b31SStefan Hajnoczi     int64_t n;
172261007b31SStefan Hajnoczi     int64_t ret, ret2;
172361007b31SStefan Hajnoczi 
172461007b31SStefan Hajnoczi     total_sectors = bdrv_nb_sectors(bs);
172561007b31SStefan Hajnoczi     if (total_sectors < 0) {
172661007b31SStefan Hajnoczi         return total_sectors;
172761007b31SStefan Hajnoczi     }
172861007b31SStefan Hajnoczi 
172961007b31SStefan Hajnoczi     if (sector_num >= total_sectors) {
173061007b31SStefan Hajnoczi         *pnum = 0;
173161007b31SStefan Hajnoczi         return 0;
173261007b31SStefan Hajnoczi     }
173361007b31SStefan Hajnoczi 
173461007b31SStefan Hajnoczi     n = total_sectors - sector_num;
173561007b31SStefan Hajnoczi     if (n < nb_sectors) {
173661007b31SStefan Hajnoczi         nb_sectors = n;
173761007b31SStefan Hajnoczi     }
173861007b31SStefan Hajnoczi 
173961007b31SStefan Hajnoczi     if (!bs->drv->bdrv_co_get_block_status) {
174061007b31SStefan Hajnoczi         *pnum = nb_sectors;
174161007b31SStefan Hajnoczi         ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
174261007b31SStefan Hajnoczi         if (bs->drv->protocol_name) {
174361007b31SStefan Hajnoczi             ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE);
174461007b31SStefan Hajnoczi         }
174561007b31SStefan Hajnoczi         return ret;
174661007b31SStefan Hajnoczi     }
174761007b31SStefan Hajnoczi 
174867a0fd2aSFam Zheng     *file = NULL;
174999723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
175067a0fd2aSFam Zheng     ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum,
175167a0fd2aSFam Zheng                                             file);
175261007b31SStefan Hajnoczi     if (ret < 0) {
175361007b31SStefan Hajnoczi         *pnum = 0;
175499723548SPaolo Bonzini         goto out;
175561007b31SStefan Hajnoczi     }
175661007b31SStefan Hajnoczi 
175761007b31SStefan Hajnoczi     if (ret & BDRV_BLOCK_RAW) {
175861007b31SStefan Hajnoczi         assert(ret & BDRV_BLOCK_OFFSET_VALID);
175999723548SPaolo Bonzini         ret = bdrv_get_block_status(bs->file->bs, ret >> BDRV_SECTOR_BITS,
176067a0fd2aSFam Zheng                                     *pnum, pnum, file);
176199723548SPaolo Bonzini         goto out;
176261007b31SStefan Hajnoczi     }
176361007b31SStefan Hajnoczi 
176461007b31SStefan Hajnoczi     if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
176561007b31SStefan Hajnoczi         ret |= BDRV_BLOCK_ALLOCATED;
1766a53f1a95SPaolo Bonzini     } else {
176761007b31SStefan Hajnoczi         if (bdrv_unallocated_blocks_are_zero(bs)) {
176861007b31SStefan Hajnoczi             ret |= BDRV_BLOCK_ZERO;
1769760e0063SKevin Wolf         } else if (bs->backing) {
1770760e0063SKevin Wolf             BlockDriverState *bs2 = bs->backing->bs;
177161007b31SStefan Hajnoczi             int64_t nb_sectors2 = bdrv_nb_sectors(bs2);
177261007b31SStefan Hajnoczi             if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) {
177361007b31SStefan Hajnoczi                 ret |= BDRV_BLOCK_ZERO;
177461007b31SStefan Hajnoczi             }
177561007b31SStefan Hajnoczi         }
177661007b31SStefan Hajnoczi     }
177761007b31SStefan Hajnoczi 
1778ac987b30SFam Zheng     if (*file && *file != bs &&
177961007b31SStefan Hajnoczi         (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
178061007b31SStefan Hajnoczi         (ret & BDRV_BLOCK_OFFSET_VALID)) {
178167a0fd2aSFam Zheng         BlockDriverState *file2;
178261007b31SStefan Hajnoczi         int file_pnum;
178361007b31SStefan Hajnoczi 
1784ac987b30SFam Zheng         ret2 = bdrv_co_get_block_status(*file, ret >> BDRV_SECTOR_BITS,
178567a0fd2aSFam Zheng                                         *pnum, &file_pnum, &file2);
178661007b31SStefan Hajnoczi         if (ret2 >= 0) {
178761007b31SStefan Hajnoczi             /* Ignore errors.  This is just providing extra information, it
178861007b31SStefan Hajnoczi              * is useful but not necessary.
178961007b31SStefan Hajnoczi              */
179061007b31SStefan Hajnoczi             if (!file_pnum) {
179161007b31SStefan Hajnoczi                 /* !file_pnum indicates an offset at or beyond the EOF; it is
179261007b31SStefan Hajnoczi                  * perfectly valid for the format block driver to point to such
179361007b31SStefan Hajnoczi                  * offsets, so catch it and mark everything as zero */
179461007b31SStefan Hajnoczi                 ret |= BDRV_BLOCK_ZERO;
179561007b31SStefan Hajnoczi             } else {
179661007b31SStefan Hajnoczi                 /* Limit request to the range reported by the protocol driver */
179761007b31SStefan Hajnoczi                 *pnum = file_pnum;
179861007b31SStefan Hajnoczi                 ret |= (ret2 & BDRV_BLOCK_ZERO);
179961007b31SStefan Hajnoczi             }
180061007b31SStefan Hajnoczi         }
180161007b31SStefan Hajnoczi     }
180261007b31SStefan Hajnoczi 
180399723548SPaolo Bonzini out:
180499723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
180561007b31SStefan Hajnoczi     return ret;
180661007b31SStefan Hajnoczi }
180761007b31SStefan Hajnoczi 
1808ba3f0e25SFam Zheng static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs,
1809ba3f0e25SFam Zheng         BlockDriverState *base,
1810ba3f0e25SFam Zheng         int64_t sector_num,
1811ba3f0e25SFam Zheng         int nb_sectors,
181267a0fd2aSFam Zheng         int *pnum,
181367a0fd2aSFam Zheng         BlockDriverState **file)
1814ba3f0e25SFam Zheng {
1815ba3f0e25SFam Zheng     BlockDriverState *p;
1816ba3f0e25SFam Zheng     int64_t ret = 0;
1817ba3f0e25SFam Zheng 
1818ba3f0e25SFam Zheng     assert(bs != base);
1819760e0063SKevin Wolf     for (p = bs; p != base; p = backing_bs(p)) {
182067a0fd2aSFam Zheng         ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum, file);
1821ba3f0e25SFam Zheng         if (ret < 0 || ret & BDRV_BLOCK_ALLOCATED) {
1822ba3f0e25SFam Zheng             break;
1823ba3f0e25SFam Zheng         }
1824ba3f0e25SFam Zheng         /* [sector_num, pnum] unallocated on this layer, which could be only
1825ba3f0e25SFam Zheng          * the first part of [sector_num, nb_sectors].  */
1826ba3f0e25SFam Zheng         nb_sectors = MIN(nb_sectors, *pnum);
1827ba3f0e25SFam Zheng     }
1828ba3f0e25SFam Zheng     return ret;
1829ba3f0e25SFam Zheng }
1830ba3f0e25SFam Zheng 
1831ba3f0e25SFam Zheng /* Coroutine wrapper for bdrv_get_block_status_above() */
1832ba3f0e25SFam Zheng static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque)
183361007b31SStefan Hajnoczi {
183461007b31SStefan Hajnoczi     BdrvCoGetBlockStatusData *data = opaque;
183561007b31SStefan Hajnoczi 
1836ba3f0e25SFam Zheng     data->ret = bdrv_co_get_block_status_above(data->bs, data->base,
1837ba3f0e25SFam Zheng                                                data->sector_num,
1838ba3f0e25SFam Zheng                                                data->nb_sectors,
183967a0fd2aSFam Zheng                                                data->pnum,
184067a0fd2aSFam Zheng                                                data->file);
184161007b31SStefan Hajnoczi     data->done = true;
184261007b31SStefan Hajnoczi }
184361007b31SStefan Hajnoczi 
184461007b31SStefan Hajnoczi /*
1845ba3f0e25SFam Zheng  * Synchronous wrapper around bdrv_co_get_block_status_above().
184661007b31SStefan Hajnoczi  *
1847ba3f0e25SFam Zheng  * See bdrv_co_get_block_status_above() for details.
184861007b31SStefan Hajnoczi  */
1849ba3f0e25SFam Zheng int64_t bdrv_get_block_status_above(BlockDriverState *bs,
1850ba3f0e25SFam Zheng                                     BlockDriverState *base,
1851ba3f0e25SFam Zheng                                     int64_t sector_num,
185267a0fd2aSFam Zheng                                     int nb_sectors, int *pnum,
185367a0fd2aSFam Zheng                                     BlockDriverState **file)
185461007b31SStefan Hajnoczi {
185561007b31SStefan Hajnoczi     Coroutine *co;
185661007b31SStefan Hajnoczi     BdrvCoGetBlockStatusData data = {
185761007b31SStefan Hajnoczi         .bs = bs,
1858ba3f0e25SFam Zheng         .base = base,
185967a0fd2aSFam Zheng         .file = file,
186061007b31SStefan Hajnoczi         .sector_num = sector_num,
186161007b31SStefan Hajnoczi         .nb_sectors = nb_sectors,
186261007b31SStefan Hajnoczi         .pnum = pnum,
186361007b31SStefan Hajnoczi         .done = false,
186461007b31SStefan Hajnoczi     };
186561007b31SStefan Hajnoczi 
186661007b31SStefan Hajnoczi     if (qemu_in_coroutine()) {
186761007b31SStefan Hajnoczi         /* Fast-path if already in coroutine context */
1868ba3f0e25SFam Zheng         bdrv_get_block_status_above_co_entry(&data);
186961007b31SStefan Hajnoczi     } else {
18700b8b8753SPaolo Bonzini         co = qemu_coroutine_create(bdrv_get_block_status_above_co_entry,
18710b8b8753SPaolo Bonzini                                    &data);
18720b8b8753SPaolo Bonzini         qemu_coroutine_enter(co);
187388b062c2SPaolo Bonzini         BDRV_POLL_WHILE(bs, !data.done);
187461007b31SStefan Hajnoczi     }
187561007b31SStefan Hajnoczi     return data.ret;
187661007b31SStefan Hajnoczi }
187761007b31SStefan Hajnoczi 
1878ba3f0e25SFam Zheng int64_t bdrv_get_block_status(BlockDriverState *bs,
1879ba3f0e25SFam Zheng                               int64_t sector_num,
188067a0fd2aSFam Zheng                               int nb_sectors, int *pnum,
188167a0fd2aSFam Zheng                               BlockDriverState **file)
1882ba3f0e25SFam Zheng {
1883760e0063SKevin Wolf     return bdrv_get_block_status_above(bs, backing_bs(bs),
188467a0fd2aSFam Zheng                                        sector_num, nb_sectors, pnum, file);
1885ba3f0e25SFam Zheng }
1886ba3f0e25SFam Zheng 
188761007b31SStefan Hajnoczi int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num,
188861007b31SStefan Hajnoczi                                    int nb_sectors, int *pnum)
188961007b31SStefan Hajnoczi {
189067a0fd2aSFam Zheng     BlockDriverState *file;
189167a0fd2aSFam Zheng     int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum,
189267a0fd2aSFam Zheng                                         &file);
189361007b31SStefan Hajnoczi     if (ret < 0) {
189461007b31SStefan Hajnoczi         return ret;
189561007b31SStefan Hajnoczi     }
189661007b31SStefan Hajnoczi     return !!(ret & BDRV_BLOCK_ALLOCATED);
189761007b31SStefan Hajnoczi }
189861007b31SStefan Hajnoczi 
189961007b31SStefan Hajnoczi /*
190061007b31SStefan Hajnoczi  * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
190161007b31SStefan Hajnoczi  *
190261007b31SStefan Hajnoczi  * Return true if the given sector is allocated in any image between
190361007b31SStefan Hajnoczi  * BASE and TOP (inclusive).  BASE can be NULL to check if the given
190461007b31SStefan Hajnoczi  * sector is allocated in any image of the chain.  Return false otherwise.
190561007b31SStefan Hajnoczi  *
190661007b31SStefan Hajnoczi  * 'pnum' is set to the number of sectors (including and immediately following
190761007b31SStefan Hajnoczi  *  the specified sector) that are known to be in the same
190861007b31SStefan Hajnoczi  *  allocated/unallocated state.
190961007b31SStefan Hajnoczi  *
191061007b31SStefan Hajnoczi  */
191161007b31SStefan Hajnoczi int bdrv_is_allocated_above(BlockDriverState *top,
191261007b31SStefan Hajnoczi                             BlockDriverState *base,
191361007b31SStefan Hajnoczi                             int64_t sector_num,
191461007b31SStefan Hajnoczi                             int nb_sectors, int *pnum)
191561007b31SStefan Hajnoczi {
191661007b31SStefan Hajnoczi     BlockDriverState *intermediate;
191761007b31SStefan Hajnoczi     int ret, n = nb_sectors;
191861007b31SStefan Hajnoczi 
191961007b31SStefan Hajnoczi     intermediate = top;
192061007b31SStefan Hajnoczi     while (intermediate && intermediate != base) {
192161007b31SStefan Hajnoczi         int pnum_inter;
192261007b31SStefan Hajnoczi         ret = bdrv_is_allocated(intermediate, sector_num, nb_sectors,
192361007b31SStefan Hajnoczi                                 &pnum_inter);
192461007b31SStefan Hajnoczi         if (ret < 0) {
192561007b31SStefan Hajnoczi             return ret;
192661007b31SStefan Hajnoczi         } else if (ret) {
192761007b31SStefan Hajnoczi             *pnum = pnum_inter;
192861007b31SStefan Hajnoczi             return 1;
192961007b31SStefan Hajnoczi         }
193061007b31SStefan Hajnoczi 
193161007b31SStefan Hajnoczi         /*
193261007b31SStefan Hajnoczi          * [sector_num, nb_sectors] is unallocated on top but intermediate
193361007b31SStefan Hajnoczi          * might have
193461007b31SStefan Hajnoczi          *
193561007b31SStefan Hajnoczi          * [sector_num+x, nr_sectors] allocated.
193661007b31SStefan Hajnoczi          */
193761007b31SStefan Hajnoczi         if (n > pnum_inter &&
193861007b31SStefan Hajnoczi             (intermediate == top ||
193961007b31SStefan Hajnoczi              sector_num + pnum_inter < intermediate->total_sectors)) {
194061007b31SStefan Hajnoczi             n = pnum_inter;
194161007b31SStefan Hajnoczi         }
194261007b31SStefan Hajnoczi 
1943760e0063SKevin Wolf         intermediate = backing_bs(intermediate);
194461007b31SStefan Hajnoczi     }
194561007b31SStefan Hajnoczi 
194661007b31SStefan Hajnoczi     *pnum = n;
194761007b31SStefan Hajnoczi     return 0;
194861007b31SStefan Hajnoczi }
194961007b31SStefan Hajnoczi 
19501a8ae822SKevin Wolf typedef struct BdrvVmstateCo {
19511a8ae822SKevin Wolf     BlockDriverState    *bs;
19521a8ae822SKevin Wolf     QEMUIOVector        *qiov;
19531a8ae822SKevin Wolf     int64_t             pos;
19541a8ae822SKevin Wolf     bool                is_read;
19551a8ae822SKevin Wolf     int                 ret;
19561a8ae822SKevin Wolf } BdrvVmstateCo;
19571a8ae822SKevin Wolf 
19581a8ae822SKevin Wolf static int coroutine_fn
19591a8ae822SKevin Wolf bdrv_co_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos,
19601a8ae822SKevin Wolf                    bool is_read)
19611a8ae822SKevin Wolf {
19621a8ae822SKevin Wolf     BlockDriver *drv = bs->drv;
19631a8ae822SKevin Wolf 
19641a8ae822SKevin Wolf     if (!drv) {
19651a8ae822SKevin Wolf         return -ENOMEDIUM;
19661a8ae822SKevin Wolf     } else if (drv->bdrv_load_vmstate) {
19671a8ae822SKevin Wolf         return is_read ? drv->bdrv_load_vmstate(bs, qiov, pos)
19681a8ae822SKevin Wolf                        : drv->bdrv_save_vmstate(bs, qiov, pos);
19691a8ae822SKevin Wolf     } else if (bs->file) {
19701a8ae822SKevin Wolf         return bdrv_co_rw_vmstate(bs->file->bs, qiov, pos, is_read);
19711a8ae822SKevin Wolf     }
19721a8ae822SKevin Wolf 
19731a8ae822SKevin Wolf     return -ENOTSUP;
19741a8ae822SKevin Wolf }
19751a8ae822SKevin Wolf 
19761a8ae822SKevin Wolf static void coroutine_fn bdrv_co_rw_vmstate_entry(void *opaque)
19771a8ae822SKevin Wolf {
19781a8ae822SKevin Wolf     BdrvVmstateCo *co = opaque;
19791a8ae822SKevin Wolf     co->ret = bdrv_co_rw_vmstate(co->bs, co->qiov, co->pos, co->is_read);
19801a8ae822SKevin Wolf }
19811a8ae822SKevin Wolf 
19821a8ae822SKevin Wolf static inline int
19831a8ae822SKevin Wolf bdrv_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos,
19841a8ae822SKevin Wolf                 bool is_read)
19851a8ae822SKevin Wolf {
19861a8ae822SKevin Wolf     if (qemu_in_coroutine()) {
19871a8ae822SKevin Wolf         return bdrv_co_rw_vmstate(bs, qiov, pos, is_read);
19881a8ae822SKevin Wolf     } else {
19891a8ae822SKevin Wolf         BdrvVmstateCo data = {
19901a8ae822SKevin Wolf             .bs         = bs,
19911a8ae822SKevin Wolf             .qiov       = qiov,
19921a8ae822SKevin Wolf             .pos        = pos,
19931a8ae822SKevin Wolf             .is_read    = is_read,
19941a8ae822SKevin Wolf             .ret        = -EINPROGRESS,
19951a8ae822SKevin Wolf         };
19960b8b8753SPaolo Bonzini         Coroutine *co = qemu_coroutine_create(bdrv_co_rw_vmstate_entry, &data);
19971a8ae822SKevin Wolf 
19980b8b8753SPaolo Bonzini         qemu_coroutine_enter(co);
19991a8ae822SKevin Wolf         while (data.ret == -EINPROGRESS) {
20001a8ae822SKevin Wolf             aio_poll(bdrv_get_aio_context(bs), true);
20011a8ae822SKevin Wolf         }
20021a8ae822SKevin Wolf         return data.ret;
20031a8ae822SKevin Wolf     }
20041a8ae822SKevin Wolf }
20051a8ae822SKevin Wolf 
200661007b31SStefan Hajnoczi int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
200761007b31SStefan Hajnoczi                       int64_t pos, int size)
200861007b31SStefan Hajnoczi {
200961007b31SStefan Hajnoczi     QEMUIOVector qiov;
201061007b31SStefan Hajnoczi     struct iovec iov = {
201161007b31SStefan Hajnoczi         .iov_base   = (void *) buf,
201261007b31SStefan Hajnoczi         .iov_len    = size,
201361007b31SStefan Hajnoczi     };
2014b433d942SKevin Wolf     int ret;
201561007b31SStefan Hajnoczi 
201661007b31SStefan Hajnoczi     qemu_iovec_init_external(&qiov, &iov, 1);
2017b433d942SKevin Wolf 
2018b433d942SKevin Wolf     ret = bdrv_writev_vmstate(bs, &qiov, pos);
2019b433d942SKevin Wolf     if (ret < 0) {
2020b433d942SKevin Wolf         return ret;
2021b433d942SKevin Wolf     }
2022b433d942SKevin Wolf 
2023b433d942SKevin Wolf     return size;
202461007b31SStefan Hajnoczi }
202561007b31SStefan Hajnoczi 
202661007b31SStefan Hajnoczi int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
202761007b31SStefan Hajnoczi {
20281a8ae822SKevin Wolf     return bdrv_rw_vmstate(bs, qiov, pos, false);
202961007b31SStefan Hajnoczi }
203061007b31SStefan Hajnoczi 
203161007b31SStefan Hajnoczi int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
203261007b31SStefan Hajnoczi                       int64_t pos, int size)
203361007b31SStefan Hajnoczi {
20345ddda0b8SKevin Wolf     QEMUIOVector qiov;
20355ddda0b8SKevin Wolf     struct iovec iov = {
20365ddda0b8SKevin Wolf         .iov_base   = buf,
20375ddda0b8SKevin Wolf         .iov_len    = size,
20385ddda0b8SKevin Wolf     };
2039b433d942SKevin Wolf     int ret;
20405ddda0b8SKevin Wolf 
20415ddda0b8SKevin Wolf     qemu_iovec_init_external(&qiov, &iov, 1);
2042b433d942SKevin Wolf     ret = bdrv_readv_vmstate(bs, &qiov, pos);
2043b433d942SKevin Wolf     if (ret < 0) {
2044b433d942SKevin Wolf         return ret;
2045b433d942SKevin Wolf     }
2046b433d942SKevin Wolf 
2047b433d942SKevin Wolf     return size;
20485ddda0b8SKevin Wolf }
20495ddda0b8SKevin Wolf 
20505ddda0b8SKevin Wolf int bdrv_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
20515ddda0b8SKevin Wolf {
20521a8ae822SKevin Wolf     return bdrv_rw_vmstate(bs, qiov, pos, true);
205361007b31SStefan Hajnoczi }
205461007b31SStefan Hajnoczi 
205561007b31SStefan Hajnoczi /**************************************************************/
205661007b31SStefan Hajnoczi /* async I/Os */
205761007b31SStefan Hajnoczi 
2058ebb7af21SKevin Wolf BlockAIOCB *bdrv_aio_readv(BdrvChild *child, int64_t sector_num,
205961007b31SStefan Hajnoczi                            QEMUIOVector *qiov, int nb_sectors,
206061007b31SStefan Hajnoczi                            BlockCompletionFunc *cb, void *opaque)
206161007b31SStefan Hajnoczi {
2062ebb7af21SKevin Wolf     trace_bdrv_aio_readv(child->bs, sector_num, nb_sectors, opaque);
206361007b31SStefan Hajnoczi 
2064b15404e0SEric Blake     assert(nb_sectors << BDRV_SECTOR_BITS == qiov->size);
2065b15404e0SEric Blake     return bdrv_co_aio_prw_vector(child, sector_num << BDRV_SECTOR_BITS, qiov,
2066b15404e0SEric Blake                                   0, cb, opaque, false);
206761007b31SStefan Hajnoczi }
206861007b31SStefan Hajnoczi 
20690d1049c7SKevin Wolf BlockAIOCB *bdrv_aio_writev(BdrvChild *child, int64_t sector_num,
207061007b31SStefan Hajnoczi                             QEMUIOVector *qiov, int nb_sectors,
207161007b31SStefan Hajnoczi                             BlockCompletionFunc *cb, void *opaque)
207261007b31SStefan Hajnoczi {
20730d1049c7SKevin Wolf     trace_bdrv_aio_writev(child->bs, sector_num, nb_sectors, opaque);
207461007b31SStefan Hajnoczi 
2075b15404e0SEric Blake     assert(nb_sectors << BDRV_SECTOR_BITS == qiov->size);
2076b15404e0SEric Blake     return bdrv_co_aio_prw_vector(child, sector_num << BDRV_SECTOR_BITS, qiov,
2077b15404e0SEric Blake                                   0, cb, opaque, true);
207861007b31SStefan Hajnoczi }
207961007b31SStefan Hajnoczi 
208061007b31SStefan Hajnoczi void bdrv_aio_cancel(BlockAIOCB *acb)
208161007b31SStefan Hajnoczi {
208261007b31SStefan Hajnoczi     qemu_aio_ref(acb);
208361007b31SStefan Hajnoczi     bdrv_aio_cancel_async(acb);
208461007b31SStefan Hajnoczi     while (acb->refcnt > 1) {
208561007b31SStefan Hajnoczi         if (acb->aiocb_info->get_aio_context) {
208661007b31SStefan Hajnoczi             aio_poll(acb->aiocb_info->get_aio_context(acb), true);
208761007b31SStefan Hajnoczi         } else if (acb->bs) {
20882f47da5fSPaolo Bonzini             /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so
20892f47da5fSPaolo Bonzini              * assert that we're not using an I/O thread.  Thread-safe
20902f47da5fSPaolo Bonzini              * code should use bdrv_aio_cancel_async exclusively.
20912f47da5fSPaolo Bonzini              */
20922f47da5fSPaolo Bonzini             assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context());
209361007b31SStefan Hajnoczi             aio_poll(bdrv_get_aio_context(acb->bs), true);
209461007b31SStefan Hajnoczi         } else {
209561007b31SStefan Hajnoczi             abort();
209661007b31SStefan Hajnoczi         }
209761007b31SStefan Hajnoczi     }
209861007b31SStefan Hajnoczi     qemu_aio_unref(acb);
209961007b31SStefan Hajnoczi }
210061007b31SStefan Hajnoczi 
210161007b31SStefan Hajnoczi /* Async version of aio cancel. The caller is not blocked if the acb implements
210261007b31SStefan Hajnoczi  * cancel_async, otherwise we do nothing and let the request normally complete.
210361007b31SStefan Hajnoczi  * In either case the completion callback must be called. */
210461007b31SStefan Hajnoczi void bdrv_aio_cancel_async(BlockAIOCB *acb)
210561007b31SStefan Hajnoczi {
210661007b31SStefan Hajnoczi     if (acb->aiocb_info->cancel_async) {
210761007b31SStefan Hajnoczi         acb->aiocb_info->cancel_async(acb);
210861007b31SStefan Hajnoczi     }
210961007b31SStefan Hajnoczi }
211061007b31SStefan Hajnoczi 
211161007b31SStefan Hajnoczi /**************************************************************/
211261007b31SStefan Hajnoczi /* async block device emulation */
211361007b31SStefan Hajnoczi 
211441574268SEric Blake typedef struct BlockRequest {
211541574268SEric Blake     union {
211641574268SEric Blake         /* Used during read, write, trim */
211741574268SEric Blake         struct {
2118b15404e0SEric Blake             int64_t offset;
2119b15404e0SEric Blake             int bytes;
212041574268SEric Blake             int flags;
212141574268SEric Blake             QEMUIOVector *qiov;
212241574268SEric Blake         };
212341574268SEric Blake         /* Used during ioctl */
212441574268SEric Blake         struct {
212541574268SEric Blake             int req;
212641574268SEric Blake             void *buf;
212741574268SEric Blake         };
212841574268SEric Blake     };
212941574268SEric Blake     BlockCompletionFunc *cb;
213041574268SEric Blake     void *opaque;
213141574268SEric Blake 
213241574268SEric Blake     int error;
213341574268SEric Blake } BlockRequest;
213441574268SEric Blake 
213561007b31SStefan Hajnoczi typedef struct BlockAIOCBCoroutine {
213661007b31SStefan Hajnoczi     BlockAIOCB common;
2137adad6496SKevin Wolf     BdrvChild *child;
213861007b31SStefan Hajnoczi     BlockRequest req;
213961007b31SStefan Hajnoczi     bool is_write;
214061007b31SStefan Hajnoczi     bool need_bh;
214161007b31SStefan Hajnoczi     bool *done;
214261007b31SStefan Hajnoczi } BlockAIOCBCoroutine;
214361007b31SStefan Hajnoczi 
214461007b31SStefan Hajnoczi static const AIOCBInfo bdrv_em_co_aiocb_info = {
214561007b31SStefan Hajnoczi     .aiocb_size         = sizeof(BlockAIOCBCoroutine),
214661007b31SStefan Hajnoczi };
214761007b31SStefan Hajnoczi 
214861007b31SStefan Hajnoczi static void bdrv_co_complete(BlockAIOCBCoroutine *acb)
214961007b31SStefan Hajnoczi {
215061007b31SStefan Hajnoczi     if (!acb->need_bh) {
215199723548SPaolo Bonzini         bdrv_dec_in_flight(acb->common.bs);
215261007b31SStefan Hajnoczi         acb->common.cb(acb->common.opaque, acb->req.error);
215361007b31SStefan Hajnoczi         qemu_aio_unref(acb);
215461007b31SStefan Hajnoczi     }
215561007b31SStefan Hajnoczi }
215661007b31SStefan Hajnoczi 
215761007b31SStefan Hajnoczi static void bdrv_co_em_bh(void *opaque)
215861007b31SStefan Hajnoczi {
215961007b31SStefan Hajnoczi     BlockAIOCBCoroutine *acb = opaque;
216061007b31SStefan Hajnoczi 
216161007b31SStefan Hajnoczi     assert(!acb->need_bh);
216261007b31SStefan Hajnoczi     bdrv_co_complete(acb);
216361007b31SStefan Hajnoczi }
216461007b31SStefan Hajnoczi 
216561007b31SStefan Hajnoczi static void bdrv_co_maybe_schedule_bh(BlockAIOCBCoroutine *acb)
216661007b31SStefan Hajnoczi {
216761007b31SStefan Hajnoczi     acb->need_bh = false;
216861007b31SStefan Hajnoczi     if (acb->req.error != -EINPROGRESS) {
216961007b31SStefan Hajnoczi         BlockDriverState *bs = acb->common.bs;
217061007b31SStefan Hajnoczi 
2171fffb6e12SPaolo Bonzini         aio_bh_schedule_oneshot(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb);
217261007b31SStefan Hajnoczi     }
217361007b31SStefan Hajnoczi }
217461007b31SStefan Hajnoczi 
217561007b31SStefan Hajnoczi /* Invoke bdrv_co_do_readv/bdrv_co_do_writev */
217661007b31SStefan Hajnoczi static void coroutine_fn bdrv_co_do_rw(void *opaque)
217761007b31SStefan Hajnoczi {
217861007b31SStefan Hajnoczi     BlockAIOCBCoroutine *acb = opaque;
217961007b31SStefan Hajnoczi 
218061007b31SStefan Hajnoczi     if (!acb->is_write) {
2181b15404e0SEric Blake         acb->req.error = bdrv_co_preadv(acb->child, acb->req.offset,
2182b15404e0SEric Blake             acb->req.qiov->size, acb->req.qiov, acb->req.flags);
218361007b31SStefan Hajnoczi     } else {
2184b15404e0SEric Blake         acb->req.error = bdrv_co_pwritev(acb->child, acb->req.offset,
2185b15404e0SEric Blake             acb->req.qiov->size, acb->req.qiov, acb->req.flags);
218661007b31SStefan Hajnoczi     }
218761007b31SStefan Hajnoczi 
218861007b31SStefan Hajnoczi     bdrv_co_complete(acb);
218961007b31SStefan Hajnoczi }
219061007b31SStefan Hajnoczi 
2191b15404e0SEric Blake static BlockAIOCB *bdrv_co_aio_prw_vector(BdrvChild *child,
2192b15404e0SEric Blake                                           int64_t offset,
219361007b31SStefan Hajnoczi                                           QEMUIOVector *qiov,
219461007b31SStefan Hajnoczi                                           BdrvRequestFlags flags,
219561007b31SStefan Hajnoczi                                           BlockCompletionFunc *cb,
219661007b31SStefan Hajnoczi                                           void *opaque,
219761007b31SStefan Hajnoczi                                           bool is_write)
219861007b31SStefan Hajnoczi {
219961007b31SStefan Hajnoczi     Coroutine *co;
220061007b31SStefan Hajnoczi     BlockAIOCBCoroutine *acb;
220161007b31SStefan Hajnoczi 
220299723548SPaolo Bonzini     /* Matched by bdrv_co_complete's bdrv_dec_in_flight.  */
220399723548SPaolo Bonzini     bdrv_inc_in_flight(child->bs);
220499723548SPaolo Bonzini 
2205adad6496SKevin Wolf     acb = qemu_aio_get(&bdrv_em_co_aiocb_info, child->bs, cb, opaque);
2206adad6496SKevin Wolf     acb->child = child;
220761007b31SStefan Hajnoczi     acb->need_bh = true;
220861007b31SStefan Hajnoczi     acb->req.error = -EINPROGRESS;
2209b15404e0SEric Blake     acb->req.offset = offset;
221061007b31SStefan Hajnoczi     acb->req.qiov = qiov;
221161007b31SStefan Hajnoczi     acb->req.flags = flags;
221261007b31SStefan Hajnoczi     acb->is_write = is_write;
221361007b31SStefan Hajnoczi 
22140b8b8753SPaolo Bonzini     co = qemu_coroutine_create(bdrv_co_do_rw, acb);
22150b8b8753SPaolo Bonzini     qemu_coroutine_enter(co);
221661007b31SStefan Hajnoczi 
221761007b31SStefan Hajnoczi     bdrv_co_maybe_schedule_bh(acb);
221861007b31SStefan Hajnoczi     return &acb->common;
221961007b31SStefan Hajnoczi }
222061007b31SStefan Hajnoczi 
222161007b31SStefan Hajnoczi static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque)
222261007b31SStefan Hajnoczi {
222361007b31SStefan Hajnoczi     BlockAIOCBCoroutine *acb = opaque;
222461007b31SStefan Hajnoczi     BlockDriverState *bs = acb->common.bs;
222561007b31SStefan Hajnoczi 
222661007b31SStefan Hajnoczi     acb->req.error = bdrv_co_flush(bs);
222761007b31SStefan Hajnoczi     bdrv_co_complete(acb);
222861007b31SStefan Hajnoczi }
222961007b31SStefan Hajnoczi 
223061007b31SStefan Hajnoczi BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs,
223161007b31SStefan Hajnoczi         BlockCompletionFunc *cb, void *opaque)
223261007b31SStefan Hajnoczi {
223361007b31SStefan Hajnoczi     trace_bdrv_aio_flush(bs, opaque);
223461007b31SStefan Hajnoczi 
223561007b31SStefan Hajnoczi     Coroutine *co;
223661007b31SStefan Hajnoczi     BlockAIOCBCoroutine *acb;
223761007b31SStefan Hajnoczi 
223899723548SPaolo Bonzini     /* Matched by bdrv_co_complete's bdrv_dec_in_flight.  */
223999723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
224099723548SPaolo Bonzini 
224161007b31SStefan Hajnoczi     acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
224261007b31SStefan Hajnoczi     acb->need_bh = true;
224361007b31SStefan Hajnoczi     acb->req.error = -EINPROGRESS;
224461007b31SStefan Hajnoczi 
22450b8b8753SPaolo Bonzini     co = qemu_coroutine_create(bdrv_aio_flush_co_entry, acb);
22460b8b8753SPaolo Bonzini     qemu_coroutine_enter(co);
224761007b31SStefan Hajnoczi 
224861007b31SStefan Hajnoczi     bdrv_co_maybe_schedule_bh(acb);
224961007b31SStefan Hajnoczi     return &acb->common;
225061007b31SStefan Hajnoczi }
225161007b31SStefan Hajnoczi 
225261007b31SStefan Hajnoczi /**************************************************************/
225361007b31SStefan Hajnoczi /* Coroutine block device emulation */
225461007b31SStefan Hajnoczi 
2255e293b7a3SKevin Wolf typedef struct FlushCo {
2256e293b7a3SKevin Wolf     BlockDriverState *bs;
2257e293b7a3SKevin Wolf     int ret;
2258e293b7a3SKevin Wolf } FlushCo;
2259e293b7a3SKevin Wolf 
2260e293b7a3SKevin Wolf 
226161007b31SStefan Hajnoczi static void coroutine_fn bdrv_flush_co_entry(void *opaque)
226261007b31SStefan Hajnoczi {
2263e293b7a3SKevin Wolf     FlushCo *rwco = opaque;
226461007b31SStefan Hajnoczi 
226561007b31SStefan Hajnoczi     rwco->ret = bdrv_co_flush(rwco->bs);
226661007b31SStefan Hajnoczi }
226761007b31SStefan Hajnoczi 
226861007b31SStefan Hajnoczi int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
226961007b31SStefan Hajnoczi {
227061007b31SStefan Hajnoczi     int ret;
227161007b31SStefan Hajnoczi 
22721b6bc94dSDimitris Aragiorgis     if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
22731b6bc94dSDimitris Aragiorgis         bdrv_is_sg(bs)) {
227461007b31SStefan Hajnoczi         return 0;
227561007b31SStefan Hajnoczi     }
227661007b31SStefan Hajnoczi 
227799723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
2278c32b82afSPavel Dovgalyuk 
22793ff2f67aSEvgeny Yakovlev     int current_gen = bs->write_gen;
22803ff2f67aSEvgeny Yakovlev 
22813ff2f67aSEvgeny Yakovlev     /* Wait until any previous flushes are completed */
228299723548SPaolo Bonzini     while (bs->active_flush_req) {
22831ace7ceaSPaolo Bonzini         qemu_co_queue_wait(&bs->flush_queue, NULL);
22843ff2f67aSEvgeny Yakovlev     }
22853ff2f67aSEvgeny Yakovlev 
228699723548SPaolo Bonzini     bs->active_flush_req = true;
22873ff2f67aSEvgeny Yakovlev 
2288c32b82afSPavel Dovgalyuk     /* Write back all layers by calling one driver function */
2289c32b82afSPavel Dovgalyuk     if (bs->drv->bdrv_co_flush) {
2290c32b82afSPavel Dovgalyuk         ret = bs->drv->bdrv_co_flush(bs);
2291c32b82afSPavel Dovgalyuk         goto out;
2292c32b82afSPavel Dovgalyuk     }
2293c32b82afSPavel Dovgalyuk 
229461007b31SStefan Hajnoczi     /* Write back cached data to the OS even with cache=unsafe */
229561007b31SStefan Hajnoczi     BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS);
229661007b31SStefan Hajnoczi     if (bs->drv->bdrv_co_flush_to_os) {
229761007b31SStefan Hajnoczi         ret = bs->drv->bdrv_co_flush_to_os(bs);
229861007b31SStefan Hajnoczi         if (ret < 0) {
2299cdb5e315SFam Zheng             goto out;
230061007b31SStefan Hajnoczi         }
230161007b31SStefan Hajnoczi     }
230261007b31SStefan Hajnoczi 
230361007b31SStefan Hajnoczi     /* But don't actually force it to the disk with cache=unsafe */
230461007b31SStefan Hajnoczi     if (bs->open_flags & BDRV_O_NO_FLUSH) {
230561007b31SStefan Hajnoczi         goto flush_parent;
230661007b31SStefan Hajnoczi     }
230761007b31SStefan Hajnoczi 
23083ff2f67aSEvgeny Yakovlev     /* Check if we really need to flush anything */
23093ff2f67aSEvgeny Yakovlev     if (bs->flushed_gen == current_gen) {
23103ff2f67aSEvgeny Yakovlev         goto flush_parent;
23113ff2f67aSEvgeny Yakovlev     }
23123ff2f67aSEvgeny Yakovlev 
231361007b31SStefan Hajnoczi     BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK);
231461007b31SStefan Hajnoczi     if (bs->drv->bdrv_co_flush_to_disk) {
231561007b31SStefan Hajnoczi         ret = bs->drv->bdrv_co_flush_to_disk(bs);
231661007b31SStefan Hajnoczi     } else if (bs->drv->bdrv_aio_flush) {
231761007b31SStefan Hajnoczi         BlockAIOCB *acb;
231861007b31SStefan Hajnoczi         CoroutineIOCompletion co = {
231961007b31SStefan Hajnoczi             .coroutine = qemu_coroutine_self(),
232061007b31SStefan Hajnoczi         };
232161007b31SStefan Hajnoczi 
232261007b31SStefan Hajnoczi         acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
232361007b31SStefan Hajnoczi         if (acb == NULL) {
232461007b31SStefan Hajnoczi             ret = -EIO;
232561007b31SStefan Hajnoczi         } else {
232661007b31SStefan Hajnoczi             qemu_coroutine_yield();
232761007b31SStefan Hajnoczi             ret = co.ret;
232861007b31SStefan Hajnoczi         }
232961007b31SStefan Hajnoczi     } else {
233061007b31SStefan Hajnoczi         /*
233161007b31SStefan Hajnoczi          * Some block drivers always operate in either writethrough or unsafe
233261007b31SStefan Hajnoczi          * mode and don't support bdrv_flush therefore. Usually qemu doesn't
233361007b31SStefan Hajnoczi          * know how the server works (because the behaviour is hardcoded or
233461007b31SStefan Hajnoczi          * depends on server-side configuration), so we can't ensure that
233561007b31SStefan Hajnoczi          * everything is safe on disk. Returning an error doesn't work because
233661007b31SStefan Hajnoczi          * that would break guests even if the server operates in writethrough
233761007b31SStefan Hajnoczi          * mode.
233861007b31SStefan Hajnoczi          *
233961007b31SStefan Hajnoczi          * Let's hope the user knows what he's doing.
234061007b31SStefan Hajnoczi          */
234161007b31SStefan Hajnoczi         ret = 0;
234261007b31SStefan Hajnoczi     }
23433ff2f67aSEvgeny Yakovlev 
234461007b31SStefan Hajnoczi     if (ret < 0) {
2345cdb5e315SFam Zheng         goto out;
234661007b31SStefan Hajnoczi     }
234761007b31SStefan Hajnoczi 
234861007b31SStefan Hajnoczi     /* Now flush the underlying protocol.  It will also have BDRV_O_NO_FLUSH
234961007b31SStefan Hajnoczi      * in the case of cache=unsafe, so there are no useless flushes.
235061007b31SStefan Hajnoczi      */
235161007b31SStefan Hajnoczi flush_parent:
2352cdb5e315SFam Zheng     ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0;
2353cdb5e315SFam Zheng out:
23543ff2f67aSEvgeny Yakovlev     /* Notify any pending flushes that we have completed */
2355e6af1e08SKevin Wolf     if (ret == 0) {
23563ff2f67aSEvgeny Yakovlev         bs->flushed_gen = current_gen;
2357e6af1e08SKevin Wolf     }
235899723548SPaolo Bonzini     bs->active_flush_req = false;
2359156af3acSDenis V. Lunev     /* Return value is ignored - it's ok if wait queue is empty */
2360156af3acSDenis V. Lunev     qemu_co_queue_next(&bs->flush_queue);
23613ff2f67aSEvgeny Yakovlev 
236299723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
2363cdb5e315SFam Zheng     return ret;
236461007b31SStefan Hajnoczi }
236561007b31SStefan Hajnoczi 
236661007b31SStefan Hajnoczi int bdrv_flush(BlockDriverState *bs)
236761007b31SStefan Hajnoczi {
236861007b31SStefan Hajnoczi     Coroutine *co;
2369e293b7a3SKevin Wolf     FlushCo flush_co = {
237061007b31SStefan Hajnoczi         .bs = bs,
237161007b31SStefan Hajnoczi         .ret = NOT_DONE,
237261007b31SStefan Hajnoczi     };
237361007b31SStefan Hajnoczi 
237461007b31SStefan Hajnoczi     if (qemu_in_coroutine()) {
237561007b31SStefan Hajnoczi         /* Fast-path if already in coroutine context */
2376e293b7a3SKevin Wolf         bdrv_flush_co_entry(&flush_co);
237761007b31SStefan Hajnoczi     } else {
23780b8b8753SPaolo Bonzini         co = qemu_coroutine_create(bdrv_flush_co_entry, &flush_co);
23790b8b8753SPaolo Bonzini         qemu_coroutine_enter(co);
238088b062c2SPaolo Bonzini         BDRV_POLL_WHILE(bs, flush_co.ret == NOT_DONE);
238161007b31SStefan Hajnoczi     }
238261007b31SStefan Hajnoczi 
2383e293b7a3SKevin Wolf     return flush_co.ret;
238461007b31SStefan Hajnoczi }
238561007b31SStefan Hajnoczi 
238661007b31SStefan Hajnoczi typedef struct DiscardCo {
238761007b31SStefan Hajnoczi     BlockDriverState *bs;
23880c51a893SEric Blake     int64_t offset;
23890c51a893SEric Blake     int count;
239061007b31SStefan Hajnoczi     int ret;
239161007b31SStefan Hajnoczi } DiscardCo;
23920c51a893SEric Blake static void coroutine_fn bdrv_pdiscard_co_entry(void *opaque)
239361007b31SStefan Hajnoczi {
239461007b31SStefan Hajnoczi     DiscardCo *rwco = opaque;
239561007b31SStefan Hajnoczi 
23960c51a893SEric Blake     rwco->ret = bdrv_co_pdiscard(rwco->bs, rwco->offset, rwco->count);
239761007b31SStefan Hajnoczi }
239861007b31SStefan Hajnoczi 
23999f1963b3SEric Blake int coroutine_fn bdrv_co_pdiscard(BlockDriverState *bs, int64_t offset,
24009f1963b3SEric Blake                                   int count)
240161007b31SStefan Hajnoczi {
2402b1066c87SFam Zheng     BdrvTrackedRequest req;
24039f1963b3SEric Blake     int max_pdiscard, ret;
24043482b9bcSEric Blake     int head, tail, align;
240561007b31SStefan Hajnoczi 
240661007b31SStefan Hajnoczi     if (!bs->drv) {
240761007b31SStefan Hajnoczi         return -ENOMEDIUM;
240861007b31SStefan Hajnoczi     }
240961007b31SStefan Hajnoczi 
24109f1963b3SEric Blake     ret = bdrv_check_byte_request(bs, offset, count);
241161007b31SStefan Hajnoczi     if (ret < 0) {
241261007b31SStefan Hajnoczi         return ret;
241361007b31SStefan Hajnoczi     } else if (bs->read_only) {
2414eaf5fe2dSPaolo Bonzini         return -EPERM;
241561007b31SStefan Hajnoczi     }
241604c01a5cSKevin Wolf     assert(!(bs->open_flags & BDRV_O_INACTIVE));
241761007b31SStefan Hajnoczi 
241861007b31SStefan Hajnoczi     /* Do nothing if disabled.  */
241961007b31SStefan Hajnoczi     if (!(bs->open_flags & BDRV_O_UNMAP)) {
242061007b31SStefan Hajnoczi         return 0;
242161007b31SStefan Hajnoczi     }
242261007b31SStefan Hajnoczi 
242302aefe43SEric Blake     if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) {
242461007b31SStefan Hajnoczi         return 0;
242561007b31SStefan Hajnoczi     }
242661007b31SStefan Hajnoczi 
24273482b9bcSEric Blake     /* Discard is advisory, but some devices track and coalesce
24283482b9bcSEric Blake      * unaligned requests, so we must pass everything down rather than
24293482b9bcSEric Blake      * round here.  Still, most devices will just silently ignore
24303482b9bcSEric Blake      * unaligned requests (by returning -ENOTSUP), so we must fragment
24313482b9bcSEric Blake      * the request accordingly.  */
243202aefe43SEric Blake     align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment);
2433b8d0a980SEric Blake     assert(align % bs->bl.request_alignment == 0);
2434b8d0a980SEric Blake     head = offset % align;
24353482b9bcSEric Blake     tail = (offset + count) % align;
24369f1963b3SEric Blake 
243799723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
24389f1963b3SEric Blake     tracked_request_begin(&req, bs, offset, count, BDRV_TRACKED_DISCARD);
243950824995SFam Zheng 
2440ec050f77SDenis V. Lunev     ret = notifier_with_return_list_notify(&bs->before_write_notifiers, &req);
2441ec050f77SDenis V. Lunev     if (ret < 0) {
2442ec050f77SDenis V. Lunev         goto out;
2443ec050f77SDenis V. Lunev     }
2444ec050f77SDenis V. Lunev 
24459f1963b3SEric Blake     max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT_MAX),
24469f1963b3SEric Blake                                    align);
24473482b9bcSEric Blake     assert(max_pdiscard >= bs->bl.request_alignment);
24489f1963b3SEric Blake 
24499f1963b3SEric Blake     while (count > 0) {
245061007b31SStefan Hajnoczi         int ret;
24513482b9bcSEric Blake         int num = count;
24523482b9bcSEric Blake 
24533482b9bcSEric Blake         if (head) {
24543482b9bcSEric Blake             /* Make small requests to get to alignment boundaries. */
24553482b9bcSEric Blake             num = MIN(count, align - head);
24563482b9bcSEric Blake             if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) {
24573482b9bcSEric Blake                 num %= bs->bl.request_alignment;
24583482b9bcSEric Blake             }
24593482b9bcSEric Blake             head = (head + num) % align;
24603482b9bcSEric Blake             assert(num < max_pdiscard);
24613482b9bcSEric Blake         } else if (tail) {
24623482b9bcSEric Blake             if (num > align) {
24633482b9bcSEric Blake                 /* Shorten the request to the last aligned cluster.  */
24643482b9bcSEric Blake                 num -= tail;
24653482b9bcSEric Blake             } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) &&
24663482b9bcSEric Blake                        tail > bs->bl.request_alignment) {
24673482b9bcSEric Blake                 tail %= bs->bl.request_alignment;
24683482b9bcSEric Blake                 num -= tail;
24693482b9bcSEric Blake             }
24703482b9bcSEric Blake         }
24713482b9bcSEric Blake         /* limit request size */
24723482b9bcSEric Blake         if (num > max_pdiscard) {
24733482b9bcSEric Blake             num = max_pdiscard;
24743482b9bcSEric Blake         }
247561007b31SStefan Hajnoczi 
247647a5486dSEric Blake         if (bs->drv->bdrv_co_pdiscard) {
247747a5486dSEric Blake             ret = bs->drv->bdrv_co_pdiscard(bs, offset, num);
247861007b31SStefan Hajnoczi         } else {
247961007b31SStefan Hajnoczi             BlockAIOCB *acb;
248061007b31SStefan Hajnoczi             CoroutineIOCompletion co = {
248161007b31SStefan Hajnoczi                 .coroutine = qemu_coroutine_self(),
248261007b31SStefan Hajnoczi             };
248361007b31SStefan Hajnoczi 
24844da444a0SEric Blake             acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num,
248561007b31SStefan Hajnoczi                                              bdrv_co_io_em_complete, &co);
248661007b31SStefan Hajnoczi             if (acb == NULL) {
2487b1066c87SFam Zheng                 ret = -EIO;
2488b1066c87SFam Zheng                 goto out;
248961007b31SStefan Hajnoczi             } else {
249061007b31SStefan Hajnoczi                 qemu_coroutine_yield();
249161007b31SStefan Hajnoczi                 ret = co.ret;
249261007b31SStefan Hajnoczi             }
249361007b31SStefan Hajnoczi         }
249461007b31SStefan Hajnoczi         if (ret && ret != -ENOTSUP) {
2495b1066c87SFam Zheng             goto out;
249661007b31SStefan Hajnoczi         }
249761007b31SStefan Hajnoczi 
24989f1963b3SEric Blake         offset += num;
24999f1963b3SEric Blake         count -= num;
250061007b31SStefan Hajnoczi     }
2501b1066c87SFam Zheng     ret = 0;
2502b1066c87SFam Zheng out:
25033ff2f67aSEvgeny Yakovlev     ++bs->write_gen;
2504968d8b06SDenis V. Lunev     bdrv_set_dirty(bs, req.offset >> BDRV_SECTOR_BITS,
2505968d8b06SDenis V. Lunev                    req.bytes >> BDRV_SECTOR_BITS);
2506b1066c87SFam Zheng     tracked_request_end(&req);
250799723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
2508b1066c87SFam Zheng     return ret;
250961007b31SStefan Hajnoczi }
251061007b31SStefan Hajnoczi 
25110c51a893SEric Blake int bdrv_pdiscard(BlockDriverState *bs, int64_t offset, int count)
251261007b31SStefan Hajnoczi {
251361007b31SStefan Hajnoczi     Coroutine *co;
251461007b31SStefan Hajnoczi     DiscardCo rwco = {
251561007b31SStefan Hajnoczi         .bs = bs,
25160c51a893SEric Blake         .offset = offset,
25170c51a893SEric Blake         .count = count,
251861007b31SStefan Hajnoczi         .ret = NOT_DONE,
251961007b31SStefan Hajnoczi     };
252061007b31SStefan Hajnoczi 
252161007b31SStefan Hajnoczi     if (qemu_in_coroutine()) {
252261007b31SStefan Hajnoczi         /* Fast-path if already in coroutine context */
25230c51a893SEric Blake         bdrv_pdiscard_co_entry(&rwco);
252461007b31SStefan Hajnoczi     } else {
25250c51a893SEric Blake         co = qemu_coroutine_create(bdrv_pdiscard_co_entry, &rwco);
25260b8b8753SPaolo Bonzini         qemu_coroutine_enter(co);
252788b062c2SPaolo Bonzini         BDRV_POLL_WHILE(bs, rwco.ret == NOT_DONE);
252861007b31SStefan Hajnoczi     }
252961007b31SStefan Hajnoczi 
253061007b31SStefan Hajnoczi     return rwco.ret;
253161007b31SStefan Hajnoczi }
253261007b31SStefan Hajnoczi 
253348af776aSKevin Wolf int bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf)
253461007b31SStefan Hajnoczi {
253561007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
25365c5ae76aSFam Zheng     CoroutineIOCompletion co = {
25375c5ae76aSFam Zheng         .coroutine = qemu_coroutine_self(),
25385c5ae76aSFam Zheng     };
25395c5ae76aSFam Zheng     BlockAIOCB *acb;
254061007b31SStefan Hajnoczi 
254199723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
254216a389dcSKevin Wolf     if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) {
25435c5ae76aSFam Zheng         co.ret = -ENOTSUP;
25445c5ae76aSFam Zheng         goto out;
25455c5ae76aSFam Zheng     }
25465c5ae76aSFam Zheng 
254716a389dcSKevin Wolf     if (drv->bdrv_co_ioctl) {
254816a389dcSKevin Wolf         co.ret = drv->bdrv_co_ioctl(bs, req, buf);
254916a389dcSKevin Wolf     } else {
25505c5ae76aSFam Zheng         acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
25515c5ae76aSFam Zheng         if (!acb) {
2552c8a9fd80SFam Zheng             co.ret = -ENOTSUP;
2553c8a9fd80SFam Zheng             goto out;
25545c5ae76aSFam Zheng         }
25555c5ae76aSFam Zheng         qemu_coroutine_yield();
255616a389dcSKevin Wolf     }
25575c5ae76aSFam Zheng out:
255899723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
25595c5ae76aSFam Zheng     return co.ret;
25605c5ae76aSFam Zheng }
25615c5ae76aSFam Zheng 
256261007b31SStefan Hajnoczi void *qemu_blockalign(BlockDriverState *bs, size_t size)
256361007b31SStefan Hajnoczi {
256461007b31SStefan Hajnoczi     return qemu_memalign(bdrv_opt_mem_align(bs), size);
256561007b31SStefan Hajnoczi }
256661007b31SStefan Hajnoczi 
256761007b31SStefan Hajnoczi void *qemu_blockalign0(BlockDriverState *bs, size_t size)
256861007b31SStefan Hajnoczi {
256961007b31SStefan Hajnoczi     return memset(qemu_blockalign(bs, size), 0, size);
257061007b31SStefan Hajnoczi }
257161007b31SStefan Hajnoczi 
257261007b31SStefan Hajnoczi void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
257361007b31SStefan Hajnoczi {
257461007b31SStefan Hajnoczi     size_t align = bdrv_opt_mem_align(bs);
257561007b31SStefan Hajnoczi 
257661007b31SStefan Hajnoczi     /* Ensure that NULL is never returned on success */
257761007b31SStefan Hajnoczi     assert(align > 0);
257861007b31SStefan Hajnoczi     if (size == 0) {
257961007b31SStefan Hajnoczi         size = align;
258061007b31SStefan Hajnoczi     }
258161007b31SStefan Hajnoczi 
258261007b31SStefan Hajnoczi     return qemu_try_memalign(align, size);
258361007b31SStefan Hajnoczi }
258461007b31SStefan Hajnoczi 
258561007b31SStefan Hajnoczi void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
258661007b31SStefan Hajnoczi {
258761007b31SStefan Hajnoczi     void *mem = qemu_try_blockalign(bs, size);
258861007b31SStefan Hajnoczi 
258961007b31SStefan Hajnoczi     if (mem) {
259061007b31SStefan Hajnoczi         memset(mem, 0, size);
259161007b31SStefan Hajnoczi     }
259261007b31SStefan Hajnoczi 
259361007b31SStefan Hajnoczi     return mem;
259461007b31SStefan Hajnoczi }
259561007b31SStefan Hajnoczi 
259661007b31SStefan Hajnoczi /*
259761007b31SStefan Hajnoczi  * Check if all memory in this vector is sector aligned.
259861007b31SStefan Hajnoczi  */
259961007b31SStefan Hajnoczi bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
260061007b31SStefan Hajnoczi {
260161007b31SStefan Hajnoczi     int i;
26024196d2f0SDenis V. Lunev     size_t alignment = bdrv_min_mem_align(bs);
260361007b31SStefan Hajnoczi 
260461007b31SStefan Hajnoczi     for (i = 0; i < qiov->niov; i++) {
260561007b31SStefan Hajnoczi         if ((uintptr_t) qiov->iov[i].iov_base % alignment) {
260661007b31SStefan Hajnoczi             return false;
260761007b31SStefan Hajnoczi         }
260861007b31SStefan Hajnoczi         if (qiov->iov[i].iov_len % alignment) {
260961007b31SStefan Hajnoczi             return false;
261061007b31SStefan Hajnoczi         }
261161007b31SStefan Hajnoczi     }
261261007b31SStefan Hajnoczi 
261361007b31SStefan Hajnoczi     return true;
261461007b31SStefan Hajnoczi }
261561007b31SStefan Hajnoczi 
261661007b31SStefan Hajnoczi void bdrv_add_before_write_notifier(BlockDriverState *bs,
261761007b31SStefan Hajnoczi                                     NotifierWithReturn *notifier)
261861007b31SStefan Hajnoczi {
261961007b31SStefan Hajnoczi     notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
262061007b31SStefan Hajnoczi }
262161007b31SStefan Hajnoczi 
262261007b31SStefan Hajnoczi void bdrv_io_plug(BlockDriverState *bs)
262361007b31SStefan Hajnoczi {
26246b98bd64SPaolo Bonzini     BdrvChild *child;
26256b98bd64SPaolo Bonzini 
26266b98bd64SPaolo Bonzini     QLIST_FOREACH(child, &bs->children, next) {
26276b98bd64SPaolo Bonzini         bdrv_io_plug(child->bs);
26286b98bd64SPaolo Bonzini     }
26296b98bd64SPaolo Bonzini 
26308f90b5e9SPaolo Bonzini     if (bs->io_plugged++ == 0) {
263161007b31SStefan Hajnoczi         BlockDriver *drv = bs->drv;
263261007b31SStefan Hajnoczi         if (drv && drv->bdrv_io_plug) {
263361007b31SStefan Hajnoczi             drv->bdrv_io_plug(bs);
26346b98bd64SPaolo Bonzini         }
263561007b31SStefan Hajnoczi     }
263661007b31SStefan Hajnoczi }
263761007b31SStefan Hajnoczi 
263861007b31SStefan Hajnoczi void bdrv_io_unplug(BlockDriverState *bs)
263961007b31SStefan Hajnoczi {
26406b98bd64SPaolo Bonzini     BdrvChild *child;
26416b98bd64SPaolo Bonzini 
26426b98bd64SPaolo Bonzini     assert(bs->io_plugged);
26438f90b5e9SPaolo Bonzini     if (--bs->io_plugged == 0) {
264461007b31SStefan Hajnoczi         BlockDriver *drv = bs->drv;
264561007b31SStefan Hajnoczi         if (drv && drv->bdrv_io_unplug) {
264661007b31SStefan Hajnoczi             drv->bdrv_io_unplug(bs);
264761007b31SStefan Hajnoczi         }
264861007b31SStefan Hajnoczi     }
264961007b31SStefan Hajnoczi 
26506b98bd64SPaolo Bonzini     QLIST_FOREACH(child, &bs->children, next) {
26516b98bd64SPaolo Bonzini         bdrv_io_unplug(child->bs);
26526b98bd64SPaolo Bonzini     }
26536b98bd64SPaolo Bonzini }
2654