xref: /qemu/block/io.c (revision 636cb5125823a75ef6bbc9d63f8d7890576fdfec)
161007b31SStefan Hajnoczi /*
261007b31SStefan Hajnoczi  * Block layer I/O functions
361007b31SStefan Hajnoczi  *
461007b31SStefan Hajnoczi  * Copyright (c) 2003 Fabrice Bellard
561007b31SStefan Hajnoczi  *
661007b31SStefan Hajnoczi  * Permission is hereby granted, free of charge, to any person obtaining a copy
761007b31SStefan Hajnoczi  * of this software and associated documentation files (the "Software"), to deal
861007b31SStefan Hajnoczi  * in the Software without restriction, including without limitation the rights
961007b31SStefan Hajnoczi  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
1061007b31SStefan Hajnoczi  * copies of the Software, and to permit persons to whom the Software is
1161007b31SStefan Hajnoczi  * furnished to do so, subject to the following conditions:
1261007b31SStefan Hajnoczi  *
1361007b31SStefan Hajnoczi  * The above copyright notice and this permission notice shall be included in
1461007b31SStefan Hajnoczi  * all copies or substantial portions of the Software.
1561007b31SStefan Hajnoczi  *
1661007b31SStefan Hajnoczi  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
1761007b31SStefan Hajnoczi  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
1861007b31SStefan Hajnoczi  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
1961007b31SStefan Hajnoczi  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
2061007b31SStefan Hajnoczi  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
2161007b31SStefan Hajnoczi  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
2261007b31SStefan Hajnoczi  * THE SOFTWARE.
2361007b31SStefan Hajnoczi  */
2461007b31SStefan Hajnoczi 
2580c71a24SPeter Maydell #include "qemu/osdep.h"
2661007b31SStefan Hajnoczi #include "trace.h"
277f0e9da6SMax Reitz #include "sysemu/block-backend.h"
2861007b31SStefan Hajnoczi #include "block/blockjob.h"
29f321dcb5SPaolo Bonzini #include "block/blockjob_int.h"
3061007b31SStefan Hajnoczi #include "block/block_int.h"
31f348b6d1SVeronia Bahaa #include "qemu/cutils.h"
32da34e65cSMarkus Armbruster #include "qapi/error.h"
33d49b6836SMarkus Armbruster #include "qemu/error-report.h"
3461007b31SStefan Hajnoczi 
3561007b31SStefan Hajnoczi #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
3661007b31SStefan Hajnoczi 
37cb2e2878SEric Blake /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */
38cb2e2878SEric Blake #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS)
39cb2e2878SEric Blake 
40d05aa8bbSEric Blake static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
41f5a5ca79SManos Pitsidianakis     int64_t offset, int bytes, BdrvRequestFlags flags);
4261007b31SStefan Hajnoczi 
430152bf40SKevin Wolf void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore)
4461007b31SStefan Hajnoczi {
4502d21300SKevin Wolf     BdrvChild *c, *next;
4627ccdd52SKevin Wolf 
4702d21300SKevin Wolf     QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
480152bf40SKevin Wolf         if (c == ignore) {
490152bf40SKevin Wolf             continue;
500152bf40SKevin Wolf         }
51c2066af0SKevin Wolf         if (c->role->drained_begin) {
52c2066af0SKevin Wolf             c->role->drained_begin(c);
53c2066af0SKevin Wolf         }
54ce0f1412SPaolo Bonzini     }
55ce0f1412SPaolo Bonzini }
56ce0f1412SPaolo Bonzini 
570152bf40SKevin Wolf void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore)
58ce0f1412SPaolo Bonzini {
5902d21300SKevin Wolf     BdrvChild *c, *next;
6027ccdd52SKevin Wolf 
6102d21300SKevin Wolf     QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
620152bf40SKevin Wolf         if (c == ignore) {
630152bf40SKevin Wolf             continue;
640152bf40SKevin Wolf         }
65c2066af0SKevin Wolf         if (c->role->drained_end) {
66c2066af0SKevin Wolf             c->role->drained_end(c);
6727ccdd52SKevin Wolf         }
68c2066af0SKevin Wolf     }
6961007b31SStefan Hajnoczi }
7061007b31SStefan Hajnoczi 
71d9e0dfa2SEric Blake static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src)
72d9e0dfa2SEric Blake {
73d9e0dfa2SEric Blake     dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer);
74d9e0dfa2SEric Blake     dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer);
75d9e0dfa2SEric Blake     dst->opt_mem_alignment = MAX(dst->opt_mem_alignment,
76d9e0dfa2SEric Blake                                  src->opt_mem_alignment);
77d9e0dfa2SEric Blake     dst->min_mem_alignment = MAX(dst->min_mem_alignment,
78d9e0dfa2SEric Blake                                  src->min_mem_alignment);
79d9e0dfa2SEric Blake     dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov);
80d9e0dfa2SEric Blake }
81d9e0dfa2SEric Blake 
8261007b31SStefan Hajnoczi void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
8361007b31SStefan Hajnoczi {
8461007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
8561007b31SStefan Hajnoczi     Error *local_err = NULL;
8661007b31SStefan Hajnoczi 
8761007b31SStefan Hajnoczi     memset(&bs->bl, 0, sizeof(bs->bl));
8861007b31SStefan Hajnoczi 
8961007b31SStefan Hajnoczi     if (!drv) {
9061007b31SStefan Hajnoczi         return;
9161007b31SStefan Hajnoczi     }
9261007b31SStefan Hajnoczi 
9379ba8c98SEric Blake     /* Default alignment based on whether driver has byte interface */
94a5b8dd2cSEric Blake     bs->bl.request_alignment = drv->bdrv_co_preadv ? 1 : 512;
9579ba8c98SEric Blake 
9661007b31SStefan Hajnoczi     /* Take some limits from the children as a default */
9761007b31SStefan Hajnoczi     if (bs->file) {
989a4f4c31SKevin Wolf         bdrv_refresh_limits(bs->file->bs, &local_err);
9961007b31SStefan Hajnoczi         if (local_err) {
10061007b31SStefan Hajnoczi             error_propagate(errp, local_err);
10161007b31SStefan Hajnoczi             return;
10261007b31SStefan Hajnoczi         }
103d9e0dfa2SEric Blake         bdrv_merge_limits(&bs->bl, &bs->file->bs->bl);
10461007b31SStefan Hajnoczi     } else {
1054196d2f0SDenis V. Lunev         bs->bl.min_mem_alignment = 512;
106459b4e66SDenis V. Lunev         bs->bl.opt_mem_alignment = getpagesize();
107bd44feb7SStefan Hajnoczi 
108bd44feb7SStefan Hajnoczi         /* Safe default since most protocols use readv()/writev()/etc */
109bd44feb7SStefan Hajnoczi         bs->bl.max_iov = IOV_MAX;
11061007b31SStefan Hajnoczi     }
11161007b31SStefan Hajnoczi 
112760e0063SKevin Wolf     if (bs->backing) {
113760e0063SKevin Wolf         bdrv_refresh_limits(bs->backing->bs, &local_err);
11461007b31SStefan Hajnoczi         if (local_err) {
11561007b31SStefan Hajnoczi             error_propagate(errp, local_err);
11661007b31SStefan Hajnoczi             return;
11761007b31SStefan Hajnoczi         }
118d9e0dfa2SEric Blake         bdrv_merge_limits(&bs->bl, &bs->backing->bs->bl);
11961007b31SStefan Hajnoczi     }
12061007b31SStefan Hajnoczi 
12161007b31SStefan Hajnoczi     /* Then let the driver override it */
12261007b31SStefan Hajnoczi     if (drv->bdrv_refresh_limits) {
12361007b31SStefan Hajnoczi         drv->bdrv_refresh_limits(bs, errp);
12461007b31SStefan Hajnoczi     }
12561007b31SStefan Hajnoczi }
12661007b31SStefan Hajnoczi 
12761007b31SStefan Hajnoczi /**
12861007b31SStefan Hajnoczi  * The copy-on-read flag is actually a reference count so multiple users may
12961007b31SStefan Hajnoczi  * use the feature without worrying about clobbering its previous state.
13061007b31SStefan Hajnoczi  * Copy-on-read stays enabled until all users have called to disable it.
13161007b31SStefan Hajnoczi  */
13261007b31SStefan Hajnoczi void bdrv_enable_copy_on_read(BlockDriverState *bs)
13361007b31SStefan Hajnoczi {
134d3faa13eSPaolo Bonzini     atomic_inc(&bs->copy_on_read);
13561007b31SStefan Hajnoczi }
13661007b31SStefan Hajnoczi 
13761007b31SStefan Hajnoczi void bdrv_disable_copy_on_read(BlockDriverState *bs)
13861007b31SStefan Hajnoczi {
139d3faa13eSPaolo Bonzini     int old = atomic_fetch_dec(&bs->copy_on_read);
140d3faa13eSPaolo Bonzini     assert(old >= 1);
14161007b31SStefan Hajnoczi }
14261007b31SStefan Hajnoczi 
14361124f03SPaolo Bonzini typedef struct {
14461124f03SPaolo Bonzini     Coroutine *co;
14561124f03SPaolo Bonzini     BlockDriverState *bs;
14661124f03SPaolo Bonzini     bool done;
147481cad48SManos Pitsidianakis     bool begin;
148b0165585SKevin Wolf     bool recursive;
1490152bf40SKevin Wolf     BdrvChild *parent;
15061124f03SPaolo Bonzini } BdrvCoDrainData;
15161124f03SPaolo Bonzini 
15261124f03SPaolo Bonzini static void coroutine_fn bdrv_drain_invoke_entry(void *opaque)
15361124f03SPaolo Bonzini {
15461124f03SPaolo Bonzini     BdrvCoDrainData *data = opaque;
15561124f03SPaolo Bonzini     BlockDriverState *bs = data->bs;
15661124f03SPaolo Bonzini 
157481cad48SManos Pitsidianakis     if (data->begin) {
158f8ea8dacSManos Pitsidianakis         bs->drv->bdrv_co_drain_begin(bs);
159481cad48SManos Pitsidianakis     } else {
160481cad48SManos Pitsidianakis         bs->drv->bdrv_co_drain_end(bs);
161481cad48SManos Pitsidianakis     }
16261124f03SPaolo Bonzini 
16361124f03SPaolo Bonzini     /* Set data->done before reading bs->wakeup.  */
16461124f03SPaolo Bonzini     atomic_mb_set(&data->done, true);
16561124f03SPaolo Bonzini     bdrv_wakeup(bs);
16661124f03SPaolo Bonzini }
16761124f03SPaolo Bonzini 
168db0289b9SKevin Wolf /* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */
1697b6a3d35SKevin Wolf static void bdrv_drain_invoke(BlockDriverState *bs, bool begin, bool recursive)
17061124f03SPaolo Bonzini {
171db0289b9SKevin Wolf     BdrvChild *child, *tmp;
172481cad48SManos Pitsidianakis     BdrvCoDrainData data = { .bs = bs, .done = false, .begin = begin};
17361124f03SPaolo Bonzini 
174f8ea8dacSManos Pitsidianakis     if (!bs->drv || (begin && !bs->drv->bdrv_co_drain_begin) ||
175481cad48SManos Pitsidianakis             (!begin && !bs->drv->bdrv_co_drain_end)) {
17661124f03SPaolo Bonzini         return;
17761124f03SPaolo Bonzini     }
17861124f03SPaolo Bonzini 
17961124f03SPaolo Bonzini     data.co = qemu_coroutine_create(bdrv_drain_invoke_entry, &data);
18061124f03SPaolo Bonzini     bdrv_coroutine_enter(bs, data.co);
18161124f03SPaolo Bonzini     BDRV_POLL_WHILE(bs, !data.done);
182db0289b9SKevin Wolf 
1837b6a3d35SKevin Wolf     if (recursive) {
184db0289b9SKevin Wolf         QLIST_FOREACH_SAFE(child, &bs->children, next, tmp) {
1857b6a3d35SKevin Wolf             bdrv_drain_invoke(child->bs, begin, true);
1867b6a3d35SKevin Wolf         }
187db0289b9SKevin Wolf     }
18861124f03SPaolo Bonzini }
18961124f03SPaolo Bonzini 
19099c05de9SKevin Wolf static bool bdrv_drain_recurse(BlockDriverState *bs)
19167da1dc5SFam Zheng {
192178bd438SFam Zheng     BdrvChild *child, *tmp;
193d42cf288SPaolo Bonzini     bool waited;
194d42cf288SPaolo Bonzini 
195481cad48SManos Pitsidianakis     /* Wait for drained requests to finish */
196481cad48SManos Pitsidianakis     waited = BDRV_POLL_WHILE(bs, atomic_read(&bs->in_flight) > 0);
197d42cf288SPaolo Bonzini 
198178bd438SFam Zheng     QLIST_FOREACH_SAFE(child, &bs->children, next, tmp) {
199178bd438SFam Zheng         BlockDriverState *bs = child->bs;
200178bd438SFam Zheng         bool in_main_loop =
201178bd438SFam Zheng             qemu_get_current_aio_context() == qemu_get_aio_context();
202178bd438SFam Zheng         assert(bs->refcnt > 0);
203178bd438SFam Zheng         if (in_main_loop) {
204178bd438SFam Zheng             /* In case the recursive bdrv_drain_recurse processes a
205178bd438SFam Zheng              * block_job_defer_to_main_loop BH and modifies the graph,
206178bd438SFam Zheng              * let's hold a reference to bs until we are done.
207178bd438SFam Zheng              *
208178bd438SFam Zheng              * IOThread doesn't have such a BH, and it is not safe to call
209178bd438SFam Zheng              * bdrv_unref without BQL, so skip doing it there.
210178bd438SFam Zheng              */
211178bd438SFam Zheng             bdrv_ref(bs);
212178bd438SFam Zheng         }
21399c05de9SKevin Wolf         waited |= bdrv_drain_recurse(bs);
214178bd438SFam Zheng         if (in_main_loop) {
215178bd438SFam Zheng             bdrv_unref(bs);
216178bd438SFam Zheng         }
21767da1dc5SFam Zheng     }
218d42cf288SPaolo Bonzini 
219d42cf288SPaolo Bonzini     return waited;
22067da1dc5SFam Zheng }
22167da1dc5SFam Zheng 
222b0165585SKevin Wolf static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
223b0165585SKevin Wolf                                   BdrvChild *parent);
224b0165585SKevin Wolf static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
225b0165585SKevin Wolf                                 BdrvChild *parent);
2260152bf40SKevin Wolf 
227a77fd4bbSFam Zheng static void bdrv_co_drain_bh_cb(void *opaque)
228a77fd4bbSFam Zheng {
229a77fd4bbSFam Zheng     BdrvCoDrainData *data = opaque;
230a77fd4bbSFam Zheng     Coroutine *co = data->co;
23199723548SPaolo Bonzini     BlockDriverState *bs = data->bs;
232a77fd4bbSFam Zheng 
23399723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
234481cad48SManos Pitsidianakis     if (data->begin) {
235b0165585SKevin Wolf         bdrv_do_drained_begin(bs, data->recursive, data->parent);
236481cad48SManos Pitsidianakis     } else {
237b0165585SKevin Wolf         bdrv_do_drained_end(bs, data->recursive, data->parent);
238481cad48SManos Pitsidianakis     }
239481cad48SManos Pitsidianakis 
240a77fd4bbSFam Zheng     data->done = true;
2411919631eSPaolo Bonzini     aio_co_wake(co);
242a77fd4bbSFam Zheng }
243a77fd4bbSFam Zheng 
244481cad48SManos Pitsidianakis static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
245b0165585SKevin Wolf                                                 bool begin, bool recursive,
246b0165585SKevin Wolf                                                 BdrvChild *parent)
247a77fd4bbSFam Zheng {
248a77fd4bbSFam Zheng     BdrvCoDrainData data;
249a77fd4bbSFam Zheng 
250a77fd4bbSFam Zheng     /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
251a77fd4bbSFam Zheng      * other coroutines run if they were queued from
252a77fd4bbSFam Zheng      * qemu_co_queue_run_restart(). */
253a77fd4bbSFam Zheng 
254a77fd4bbSFam Zheng     assert(qemu_in_coroutine());
255a77fd4bbSFam Zheng     data = (BdrvCoDrainData) {
256a77fd4bbSFam Zheng         .co = qemu_coroutine_self(),
257a77fd4bbSFam Zheng         .bs = bs,
258a77fd4bbSFam Zheng         .done = false,
259481cad48SManos Pitsidianakis         .begin = begin,
260b0165585SKevin Wolf         .recursive = recursive,
2610152bf40SKevin Wolf         .parent = parent,
262a77fd4bbSFam Zheng     };
26399723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
264fffb6e12SPaolo Bonzini     aio_bh_schedule_oneshot(bdrv_get_aio_context(bs),
265fffb6e12SPaolo Bonzini                             bdrv_co_drain_bh_cb, &data);
266a77fd4bbSFam Zheng 
267a77fd4bbSFam Zheng     qemu_coroutine_yield();
268a77fd4bbSFam Zheng     /* If we are resumed from some other event (such as an aio completion or a
269a77fd4bbSFam Zheng      * timer callback), it is a bug in the caller that should be fixed. */
270a77fd4bbSFam Zheng     assert(data.done);
271a77fd4bbSFam Zheng }
272a77fd4bbSFam Zheng 
273d736f119SKevin Wolf void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
274b0165585SKevin Wolf                            BdrvChild *parent)
2756820643fSKevin Wolf {
276b0165585SKevin Wolf     BdrvChild *child, *next;
277b0165585SKevin Wolf 
278d42cf288SPaolo Bonzini     if (qemu_in_coroutine()) {
279b0165585SKevin Wolf         bdrv_co_yield_to_drain(bs, true, recursive, parent);
280d42cf288SPaolo Bonzini         return;
281d42cf288SPaolo Bonzini     }
282d42cf288SPaolo Bonzini 
28360369b86SKevin Wolf     /* Stop things in parent-to-child order */
284414c2ec3SPaolo Bonzini     if (atomic_fetch_inc(&bs->quiesce_counter) == 0) {
2856820643fSKevin Wolf         aio_disable_external(bdrv_get_aio_context(bs));
2866820643fSKevin Wolf     }
2876820643fSKevin Wolf 
2880152bf40SKevin Wolf     bdrv_parent_drained_begin(bs, parent);
2897b6a3d35SKevin Wolf     bdrv_drain_invoke(bs, true, false);
29099c05de9SKevin Wolf     bdrv_drain_recurse(bs);
291b0165585SKevin Wolf 
292b0165585SKevin Wolf     if (recursive) {
293d736f119SKevin Wolf         bs->recursive_quiesce_counter++;
294b0165585SKevin Wolf         QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
295b0165585SKevin Wolf             bdrv_do_drained_begin(child->bs, true, child);
296b0165585SKevin Wolf         }
297b0165585SKevin Wolf     }
2986820643fSKevin Wolf }
2996820643fSKevin Wolf 
3000152bf40SKevin Wolf void bdrv_drained_begin(BlockDriverState *bs)
3010152bf40SKevin Wolf {
302b0165585SKevin Wolf     bdrv_do_drained_begin(bs, false, NULL);
3030152bf40SKevin Wolf }
3040152bf40SKevin Wolf 
305b0165585SKevin Wolf void bdrv_subtree_drained_begin(BlockDriverState *bs)
3066820643fSKevin Wolf {
307b0165585SKevin Wolf     bdrv_do_drained_begin(bs, true, NULL);
308b0165585SKevin Wolf }
309b0165585SKevin Wolf 
310d736f119SKevin Wolf void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
311b0165585SKevin Wolf                          BdrvChild *parent)
312b0165585SKevin Wolf {
313b0165585SKevin Wolf     BdrvChild *child, *next;
3140f115168SKevin Wolf     int old_quiesce_counter;
3150f115168SKevin Wolf 
316481cad48SManos Pitsidianakis     if (qemu_in_coroutine()) {
317b0165585SKevin Wolf         bdrv_co_yield_to_drain(bs, false, recursive, parent);
318481cad48SManos Pitsidianakis         return;
319481cad48SManos Pitsidianakis     }
3206820643fSKevin Wolf     assert(bs->quiesce_counter > 0);
3210f115168SKevin Wolf     old_quiesce_counter = atomic_fetch_dec(&bs->quiesce_counter);
3226820643fSKevin Wolf 
32360369b86SKevin Wolf     /* Re-enable things in child-to-parent order */
3247b6a3d35SKevin Wolf     bdrv_drain_invoke(bs, false, false);
3250152bf40SKevin Wolf     bdrv_parent_drained_end(bs, parent);
3260f115168SKevin Wolf     if (old_quiesce_counter == 1) {
3276820643fSKevin Wolf         aio_enable_external(bdrv_get_aio_context(bs));
3286820643fSKevin Wolf     }
329b0165585SKevin Wolf 
330b0165585SKevin Wolf     if (recursive) {
331d736f119SKevin Wolf         bs->recursive_quiesce_counter--;
332b0165585SKevin Wolf         QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
333b0165585SKevin Wolf             bdrv_do_drained_end(child->bs, true, child);
334b0165585SKevin Wolf         }
335b0165585SKevin Wolf     }
3360f115168SKevin Wolf }
3376820643fSKevin Wolf 
3380152bf40SKevin Wolf void bdrv_drained_end(BlockDriverState *bs)
3390152bf40SKevin Wolf {
340b0165585SKevin Wolf     bdrv_do_drained_end(bs, false, NULL);
341b0165585SKevin Wolf }
342b0165585SKevin Wolf 
343b0165585SKevin Wolf void bdrv_subtree_drained_end(BlockDriverState *bs)
344b0165585SKevin Wolf {
345b0165585SKevin Wolf     bdrv_do_drained_end(bs, true, NULL);
3460152bf40SKevin Wolf }
3470152bf40SKevin Wolf 
348d736f119SKevin Wolf void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent)
349d736f119SKevin Wolf {
350d736f119SKevin Wolf     int i;
351d736f119SKevin Wolf 
352d736f119SKevin Wolf     for (i = 0; i < new_parent->recursive_quiesce_counter; i++) {
353d736f119SKevin Wolf         bdrv_do_drained_begin(child->bs, true, child);
354d736f119SKevin Wolf     }
355d736f119SKevin Wolf }
356d736f119SKevin Wolf 
357d736f119SKevin Wolf void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent)
358d736f119SKevin Wolf {
359d736f119SKevin Wolf     int i;
360d736f119SKevin Wolf 
361d736f119SKevin Wolf     for (i = 0; i < old_parent->recursive_quiesce_counter; i++) {
362d736f119SKevin Wolf         bdrv_do_drained_end(child->bs, true, child);
363d736f119SKevin Wolf     }
364d736f119SKevin Wolf }
365d736f119SKevin Wolf 
36661007b31SStefan Hajnoczi /*
36767da1dc5SFam Zheng  * Wait for pending requests to complete on a single BlockDriverState subtree,
36867da1dc5SFam Zheng  * and suspend block driver's internal I/O until next request arrives.
36961007b31SStefan Hajnoczi  *
37061007b31SStefan Hajnoczi  * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState
37161007b31SStefan Hajnoczi  * AioContext.
3727a63f3cdSStefan Hajnoczi  *
3737a63f3cdSStefan Hajnoczi  * Only this BlockDriverState's AioContext is run, so in-flight requests must
3747a63f3cdSStefan Hajnoczi  * not depend on events in other AioContexts.  In that case, use
3757a63f3cdSStefan Hajnoczi  * bdrv_drain_all() instead.
37661007b31SStefan Hajnoczi  */
377b6e84c97SPaolo Bonzini void coroutine_fn bdrv_co_drain(BlockDriverState *bs)
378b6e84c97SPaolo Bonzini {
3796820643fSKevin Wolf     assert(qemu_in_coroutine());
3806820643fSKevin Wolf     bdrv_drained_begin(bs);
3816820643fSKevin Wolf     bdrv_drained_end(bs);
382b6e84c97SPaolo Bonzini }
383b6e84c97SPaolo Bonzini 
38461007b31SStefan Hajnoczi void bdrv_drain(BlockDriverState *bs)
38561007b31SStefan Hajnoczi {
3866820643fSKevin Wolf     bdrv_drained_begin(bs);
3876820643fSKevin Wolf     bdrv_drained_end(bs);
38861007b31SStefan Hajnoczi }
38961007b31SStefan Hajnoczi 
39061007b31SStefan Hajnoczi /*
39161007b31SStefan Hajnoczi  * Wait for pending requests to complete across all BlockDriverStates
39261007b31SStefan Hajnoczi  *
39361007b31SStefan Hajnoczi  * This function does not flush data to disk, use bdrv_flush_all() for that
39461007b31SStefan Hajnoczi  * after calling this function.
395c0778f66SAlberto Garcia  *
396c0778f66SAlberto Garcia  * This pauses all block jobs and disables external clients. It must
397c0778f66SAlberto Garcia  * be paired with bdrv_drain_all_end().
398c0778f66SAlberto Garcia  *
399c0778f66SAlberto Garcia  * NOTE: no new block jobs or BlockDriverStates can be created between
400c0778f66SAlberto Garcia  * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls.
40161007b31SStefan Hajnoczi  */
402c0778f66SAlberto Garcia void bdrv_drain_all_begin(void)
40361007b31SStefan Hajnoczi {
40461007b31SStefan Hajnoczi     /* Always run first iteration so any pending completion BHs run */
40599723548SPaolo Bonzini     bool waited = true;
4067c8eece4SKevin Wolf     BlockDriverState *bs;
40788be7b4bSKevin Wolf     BdrvNextIterator it;
408f406c03cSAlexander Yarygin     GSList *aio_ctxs = NULL, *ctx;
40961007b31SStefan Hajnoczi 
4109a7e86c8SKevin Wolf     /* BDRV_POLL_WHILE() for a node can only be called from its own I/O thread
4119a7e86c8SKevin Wolf      * or the main loop AioContext. We potentially use BDRV_POLL_WHILE() on
4129a7e86c8SKevin Wolf      * nodes in several different AioContexts, so make sure we're in the main
4139a7e86c8SKevin Wolf      * context. */
4149a7e86c8SKevin Wolf     assert(qemu_get_current_aio_context() == qemu_get_aio_context());
4159a7e86c8SKevin Wolf 
41688be7b4bSKevin Wolf     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
41761007b31SStefan Hajnoczi         AioContext *aio_context = bdrv_get_aio_context(bs);
41861007b31SStefan Hajnoczi 
41960369b86SKevin Wolf         /* Stop things in parent-to-child order */
42061007b31SStefan Hajnoczi         aio_context_acquire(aio_context);
421c0778f66SAlberto Garcia         aio_disable_external(aio_context);
4220152bf40SKevin Wolf         bdrv_parent_drained_begin(bs, NULL);
4237b6a3d35SKevin Wolf         bdrv_drain_invoke(bs, true, true);
42461007b31SStefan Hajnoczi         aio_context_release(aio_context);
425f406c03cSAlexander Yarygin 
426764ba3aeSAlberto Garcia         if (!g_slist_find(aio_ctxs, aio_context)) {
427f406c03cSAlexander Yarygin             aio_ctxs = g_slist_prepend(aio_ctxs, aio_context);
428f406c03cSAlexander Yarygin         }
42961007b31SStefan Hajnoczi     }
43061007b31SStefan Hajnoczi 
4317a63f3cdSStefan Hajnoczi     /* Note that completion of an asynchronous I/O operation can trigger any
4327a63f3cdSStefan Hajnoczi      * number of other I/O operations on other devices---for example a
4337a63f3cdSStefan Hajnoczi      * coroutine can submit an I/O request to another device in response to
4347a63f3cdSStefan Hajnoczi      * request completion.  Therefore we must keep looping until there was no
4357a63f3cdSStefan Hajnoczi      * more activity rather than simply draining each device independently.
4367a63f3cdSStefan Hajnoczi      */
43799723548SPaolo Bonzini     while (waited) {
43899723548SPaolo Bonzini         waited = false;
439f406c03cSAlexander Yarygin 
440f406c03cSAlexander Yarygin         for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) {
441f406c03cSAlexander Yarygin             AioContext *aio_context = ctx->data;
44261007b31SStefan Hajnoczi 
44361007b31SStefan Hajnoczi             aio_context_acquire(aio_context);
44488be7b4bSKevin Wolf             for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
445f406c03cSAlexander Yarygin                 if (aio_context == bdrv_get_aio_context(bs)) {
44699c05de9SKevin Wolf                     waited |= bdrv_drain_recurse(bs);
447f406c03cSAlexander Yarygin                 }
448f406c03cSAlexander Yarygin             }
44961007b31SStefan Hajnoczi             aio_context_release(aio_context);
45061007b31SStefan Hajnoczi         }
45161007b31SStefan Hajnoczi     }
45261007b31SStefan Hajnoczi 
453c0778f66SAlberto Garcia     g_slist_free(aio_ctxs);
454c0778f66SAlberto Garcia }
455c0778f66SAlberto Garcia 
456c0778f66SAlberto Garcia void bdrv_drain_all_end(void)
457c0778f66SAlberto Garcia {
458c0778f66SAlberto Garcia     BlockDriverState *bs;
459c0778f66SAlberto Garcia     BdrvNextIterator it;
460c0778f66SAlberto Garcia 
46188be7b4bSKevin Wolf     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
46261007b31SStefan Hajnoczi         AioContext *aio_context = bdrv_get_aio_context(bs);
46361007b31SStefan Hajnoczi 
46460369b86SKevin Wolf         /* Re-enable things in child-to-parent order */
46561007b31SStefan Hajnoczi         aio_context_acquire(aio_context);
4667b6a3d35SKevin Wolf         bdrv_drain_invoke(bs, false, true);
4670152bf40SKevin Wolf         bdrv_parent_drained_end(bs, NULL);
46860369b86SKevin Wolf         aio_enable_external(aio_context);
46961007b31SStefan Hajnoczi         aio_context_release(aio_context);
47061007b31SStefan Hajnoczi     }
47161007b31SStefan Hajnoczi }
47261007b31SStefan Hajnoczi 
473c0778f66SAlberto Garcia void bdrv_drain_all(void)
474c0778f66SAlberto Garcia {
475c0778f66SAlberto Garcia     bdrv_drain_all_begin();
476c0778f66SAlberto Garcia     bdrv_drain_all_end();
477c0778f66SAlberto Garcia }
478c0778f66SAlberto Garcia 
47961007b31SStefan Hajnoczi /**
48061007b31SStefan Hajnoczi  * Remove an active request from the tracked requests list
48161007b31SStefan Hajnoczi  *
48261007b31SStefan Hajnoczi  * This function should be called when a tracked request is completing.
48361007b31SStefan Hajnoczi  */
48461007b31SStefan Hajnoczi static void tracked_request_end(BdrvTrackedRequest *req)
48561007b31SStefan Hajnoczi {
48661007b31SStefan Hajnoczi     if (req->serialising) {
48720fc71b2SPaolo Bonzini         atomic_dec(&req->bs->serialising_in_flight);
48861007b31SStefan Hajnoczi     }
48961007b31SStefan Hajnoczi 
4903783fa3dSPaolo Bonzini     qemu_co_mutex_lock(&req->bs->reqs_lock);
49161007b31SStefan Hajnoczi     QLIST_REMOVE(req, list);
49261007b31SStefan Hajnoczi     qemu_co_queue_restart_all(&req->wait_queue);
4933783fa3dSPaolo Bonzini     qemu_co_mutex_unlock(&req->bs->reqs_lock);
49461007b31SStefan Hajnoczi }
49561007b31SStefan Hajnoczi 
49661007b31SStefan Hajnoczi /**
49761007b31SStefan Hajnoczi  * Add an active request to the tracked requests list
49861007b31SStefan Hajnoczi  */
49961007b31SStefan Hajnoczi static void tracked_request_begin(BdrvTrackedRequest *req,
50061007b31SStefan Hajnoczi                                   BlockDriverState *bs,
50161007b31SStefan Hajnoczi                                   int64_t offset,
502ebde595cSFam Zheng                                   unsigned int bytes,
503ebde595cSFam Zheng                                   enum BdrvTrackedRequestType type)
50461007b31SStefan Hajnoczi {
50561007b31SStefan Hajnoczi     *req = (BdrvTrackedRequest){
50661007b31SStefan Hajnoczi         .bs = bs,
50761007b31SStefan Hajnoczi         .offset         = offset,
50861007b31SStefan Hajnoczi         .bytes          = bytes,
509ebde595cSFam Zheng         .type           = type,
51061007b31SStefan Hajnoczi         .co             = qemu_coroutine_self(),
51161007b31SStefan Hajnoczi         .serialising    = false,
51261007b31SStefan Hajnoczi         .overlap_offset = offset,
51361007b31SStefan Hajnoczi         .overlap_bytes  = bytes,
51461007b31SStefan Hajnoczi     };
51561007b31SStefan Hajnoczi 
51661007b31SStefan Hajnoczi     qemu_co_queue_init(&req->wait_queue);
51761007b31SStefan Hajnoczi 
5183783fa3dSPaolo Bonzini     qemu_co_mutex_lock(&bs->reqs_lock);
51961007b31SStefan Hajnoczi     QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
5203783fa3dSPaolo Bonzini     qemu_co_mutex_unlock(&bs->reqs_lock);
52161007b31SStefan Hajnoczi }
52261007b31SStefan Hajnoczi 
52361007b31SStefan Hajnoczi static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align)
52461007b31SStefan Hajnoczi {
52561007b31SStefan Hajnoczi     int64_t overlap_offset = req->offset & ~(align - 1);
52661007b31SStefan Hajnoczi     unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align)
52761007b31SStefan Hajnoczi                                - overlap_offset;
52861007b31SStefan Hajnoczi 
52961007b31SStefan Hajnoczi     if (!req->serialising) {
53020fc71b2SPaolo Bonzini         atomic_inc(&req->bs->serialising_in_flight);
53161007b31SStefan Hajnoczi         req->serialising = true;
53261007b31SStefan Hajnoczi     }
53361007b31SStefan Hajnoczi 
53461007b31SStefan Hajnoczi     req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
53561007b31SStefan Hajnoczi     req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
53661007b31SStefan Hajnoczi }
53761007b31SStefan Hajnoczi 
53861007b31SStefan Hajnoczi /**
539244483e6SKevin Wolf  * Round a region to cluster boundaries
540244483e6SKevin Wolf  */
541244483e6SKevin Wolf void bdrv_round_to_clusters(BlockDriverState *bs,
5427cfd5275SEric Blake                             int64_t offset, int64_t bytes,
543244483e6SKevin Wolf                             int64_t *cluster_offset,
5447cfd5275SEric Blake                             int64_t *cluster_bytes)
545244483e6SKevin Wolf {
546244483e6SKevin Wolf     BlockDriverInfo bdi;
547244483e6SKevin Wolf 
548244483e6SKevin Wolf     if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
549244483e6SKevin Wolf         *cluster_offset = offset;
550244483e6SKevin Wolf         *cluster_bytes = bytes;
551244483e6SKevin Wolf     } else {
552244483e6SKevin Wolf         int64_t c = bdi.cluster_size;
553244483e6SKevin Wolf         *cluster_offset = QEMU_ALIGN_DOWN(offset, c);
554244483e6SKevin Wolf         *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c);
555244483e6SKevin Wolf     }
556244483e6SKevin Wolf }
557244483e6SKevin Wolf 
55861007b31SStefan Hajnoczi static int bdrv_get_cluster_size(BlockDriverState *bs)
55961007b31SStefan Hajnoczi {
56061007b31SStefan Hajnoczi     BlockDriverInfo bdi;
56161007b31SStefan Hajnoczi     int ret;
56261007b31SStefan Hajnoczi 
56361007b31SStefan Hajnoczi     ret = bdrv_get_info(bs, &bdi);
56461007b31SStefan Hajnoczi     if (ret < 0 || bdi.cluster_size == 0) {
565a5b8dd2cSEric Blake         return bs->bl.request_alignment;
56661007b31SStefan Hajnoczi     } else {
56761007b31SStefan Hajnoczi         return bdi.cluster_size;
56861007b31SStefan Hajnoczi     }
56961007b31SStefan Hajnoczi }
57061007b31SStefan Hajnoczi 
57161007b31SStefan Hajnoczi static bool tracked_request_overlaps(BdrvTrackedRequest *req,
57261007b31SStefan Hajnoczi                                      int64_t offset, unsigned int bytes)
57361007b31SStefan Hajnoczi {
57461007b31SStefan Hajnoczi     /*        aaaa   bbbb */
57561007b31SStefan Hajnoczi     if (offset >= req->overlap_offset + req->overlap_bytes) {
57661007b31SStefan Hajnoczi         return false;
57761007b31SStefan Hajnoczi     }
57861007b31SStefan Hajnoczi     /* bbbb   aaaa        */
57961007b31SStefan Hajnoczi     if (req->overlap_offset >= offset + bytes) {
58061007b31SStefan Hajnoczi         return false;
58161007b31SStefan Hajnoczi     }
58261007b31SStefan Hajnoczi     return true;
58361007b31SStefan Hajnoczi }
58461007b31SStefan Hajnoczi 
58599723548SPaolo Bonzini void bdrv_inc_in_flight(BlockDriverState *bs)
58699723548SPaolo Bonzini {
58799723548SPaolo Bonzini     atomic_inc(&bs->in_flight);
58899723548SPaolo Bonzini }
58999723548SPaolo Bonzini 
590c9d1a561SPaolo Bonzini static void dummy_bh_cb(void *opaque)
591c9d1a561SPaolo Bonzini {
592c9d1a561SPaolo Bonzini }
593c9d1a561SPaolo Bonzini 
594c9d1a561SPaolo Bonzini void bdrv_wakeup(BlockDriverState *bs)
595c9d1a561SPaolo Bonzini {
596e2a6ae7fSPaolo Bonzini     /* The barrier (or an atomic op) is in the caller.  */
597e2a6ae7fSPaolo Bonzini     if (atomic_read(&bs->wakeup)) {
598c9d1a561SPaolo Bonzini         aio_bh_schedule_oneshot(qemu_get_aio_context(), dummy_bh_cb, NULL);
599c9d1a561SPaolo Bonzini     }
600c9d1a561SPaolo Bonzini }
601c9d1a561SPaolo Bonzini 
60299723548SPaolo Bonzini void bdrv_dec_in_flight(BlockDriverState *bs)
60399723548SPaolo Bonzini {
60499723548SPaolo Bonzini     atomic_dec(&bs->in_flight);
605c9d1a561SPaolo Bonzini     bdrv_wakeup(bs);
60699723548SPaolo Bonzini }
60799723548SPaolo Bonzini 
60861007b31SStefan Hajnoczi static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self)
60961007b31SStefan Hajnoczi {
61061007b31SStefan Hajnoczi     BlockDriverState *bs = self->bs;
61161007b31SStefan Hajnoczi     BdrvTrackedRequest *req;
61261007b31SStefan Hajnoczi     bool retry;
61361007b31SStefan Hajnoczi     bool waited = false;
61461007b31SStefan Hajnoczi 
61520fc71b2SPaolo Bonzini     if (!atomic_read(&bs->serialising_in_flight)) {
61661007b31SStefan Hajnoczi         return false;
61761007b31SStefan Hajnoczi     }
61861007b31SStefan Hajnoczi 
61961007b31SStefan Hajnoczi     do {
62061007b31SStefan Hajnoczi         retry = false;
6213783fa3dSPaolo Bonzini         qemu_co_mutex_lock(&bs->reqs_lock);
62261007b31SStefan Hajnoczi         QLIST_FOREACH(req, &bs->tracked_requests, list) {
62361007b31SStefan Hajnoczi             if (req == self || (!req->serialising && !self->serialising)) {
62461007b31SStefan Hajnoczi                 continue;
62561007b31SStefan Hajnoczi             }
62661007b31SStefan Hajnoczi             if (tracked_request_overlaps(req, self->overlap_offset,
62761007b31SStefan Hajnoczi                                          self->overlap_bytes))
62861007b31SStefan Hajnoczi             {
62961007b31SStefan Hajnoczi                 /* Hitting this means there was a reentrant request, for
63061007b31SStefan Hajnoczi                  * example, a block driver issuing nested requests.  This must
63161007b31SStefan Hajnoczi                  * never happen since it means deadlock.
63261007b31SStefan Hajnoczi                  */
63361007b31SStefan Hajnoczi                 assert(qemu_coroutine_self() != req->co);
63461007b31SStefan Hajnoczi 
63561007b31SStefan Hajnoczi                 /* If the request is already (indirectly) waiting for us, or
63661007b31SStefan Hajnoczi                  * will wait for us as soon as it wakes up, then just go on
63761007b31SStefan Hajnoczi                  * (instead of producing a deadlock in the former case). */
63861007b31SStefan Hajnoczi                 if (!req->waiting_for) {
63961007b31SStefan Hajnoczi                     self->waiting_for = req;
6403783fa3dSPaolo Bonzini                     qemu_co_queue_wait(&req->wait_queue, &bs->reqs_lock);
64161007b31SStefan Hajnoczi                     self->waiting_for = NULL;
64261007b31SStefan Hajnoczi                     retry = true;
64361007b31SStefan Hajnoczi                     waited = true;
64461007b31SStefan Hajnoczi                     break;
64561007b31SStefan Hajnoczi                 }
64661007b31SStefan Hajnoczi             }
64761007b31SStefan Hajnoczi         }
6483783fa3dSPaolo Bonzini         qemu_co_mutex_unlock(&bs->reqs_lock);
64961007b31SStefan Hajnoczi     } while (retry);
65061007b31SStefan Hajnoczi 
65161007b31SStefan Hajnoczi     return waited;
65261007b31SStefan Hajnoczi }
65361007b31SStefan Hajnoczi 
65461007b31SStefan Hajnoczi static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset,
65561007b31SStefan Hajnoczi                                    size_t size)
65661007b31SStefan Hajnoczi {
65761007b31SStefan Hajnoczi     if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) {
65861007b31SStefan Hajnoczi         return -EIO;
65961007b31SStefan Hajnoczi     }
66061007b31SStefan Hajnoczi 
66161007b31SStefan Hajnoczi     if (!bdrv_is_inserted(bs)) {
66261007b31SStefan Hajnoczi         return -ENOMEDIUM;
66361007b31SStefan Hajnoczi     }
66461007b31SStefan Hajnoczi 
66561007b31SStefan Hajnoczi     if (offset < 0) {
66661007b31SStefan Hajnoczi         return -EIO;
66761007b31SStefan Hajnoczi     }
66861007b31SStefan Hajnoczi 
66961007b31SStefan Hajnoczi     return 0;
67061007b31SStefan Hajnoczi }
67161007b31SStefan Hajnoczi 
67261007b31SStefan Hajnoczi typedef struct RwCo {
673e293b7a3SKevin Wolf     BdrvChild *child;
67461007b31SStefan Hajnoczi     int64_t offset;
67561007b31SStefan Hajnoczi     QEMUIOVector *qiov;
67661007b31SStefan Hajnoczi     bool is_write;
67761007b31SStefan Hajnoczi     int ret;
67861007b31SStefan Hajnoczi     BdrvRequestFlags flags;
67961007b31SStefan Hajnoczi } RwCo;
68061007b31SStefan Hajnoczi 
68161007b31SStefan Hajnoczi static void coroutine_fn bdrv_rw_co_entry(void *opaque)
68261007b31SStefan Hajnoczi {
68361007b31SStefan Hajnoczi     RwCo *rwco = opaque;
68461007b31SStefan Hajnoczi 
68561007b31SStefan Hajnoczi     if (!rwco->is_write) {
686a03ef88fSKevin Wolf         rwco->ret = bdrv_co_preadv(rwco->child, rwco->offset,
68761007b31SStefan Hajnoczi                                    rwco->qiov->size, rwco->qiov,
68861007b31SStefan Hajnoczi                                    rwco->flags);
68961007b31SStefan Hajnoczi     } else {
690a03ef88fSKevin Wolf         rwco->ret = bdrv_co_pwritev(rwco->child, rwco->offset,
69161007b31SStefan Hajnoczi                                     rwco->qiov->size, rwco->qiov,
69261007b31SStefan Hajnoczi                                     rwco->flags);
69361007b31SStefan Hajnoczi     }
69461007b31SStefan Hajnoczi }
69561007b31SStefan Hajnoczi 
69661007b31SStefan Hajnoczi /*
69761007b31SStefan Hajnoczi  * Process a vectored synchronous request using coroutines
69861007b31SStefan Hajnoczi  */
699e293b7a3SKevin Wolf static int bdrv_prwv_co(BdrvChild *child, int64_t offset,
70061007b31SStefan Hajnoczi                         QEMUIOVector *qiov, bool is_write,
70161007b31SStefan Hajnoczi                         BdrvRequestFlags flags)
70261007b31SStefan Hajnoczi {
70361007b31SStefan Hajnoczi     Coroutine *co;
70461007b31SStefan Hajnoczi     RwCo rwco = {
705e293b7a3SKevin Wolf         .child = child,
70661007b31SStefan Hajnoczi         .offset = offset,
70761007b31SStefan Hajnoczi         .qiov = qiov,
70861007b31SStefan Hajnoczi         .is_write = is_write,
70961007b31SStefan Hajnoczi         .ret = NOT_DONE,
71061007b31SStefan Hajnoczi         .flags = flags,
71161007b31SStefan Hajnoczi     };
71261007b31SStefan Hajnoczi 
71361007b31SStefan Hajnoczi     if (qemu_in_coroutine()) {
71461007b31SStefan Hajnoczi         /* Fast-path if already in coroutine context */
71561007b31SStefan Hajnoczi         bdrv_rw_co_entry(&rwco);
71661007b31SStefan Hajnoczi     } else {
7170b8b8753SPaolo Bonzini         co = qemu_coroutine_create(bdrv_rw_co_entry, &rwco);
718e92f0e19SFam Zheng         bdrv_coroutine_enter(child->bs, co);
71988b062c2SPaolo Bonzini         BDRV_POLL_WHILE(child->bs, rwco.ret == NOT_DONE);
72061007b31SStefan Hajnoczi     }
72161007b31SStefan Hajnoczi     return rwco.ret;
72261007b31SStefan Hajnoczi }
72361007b31SStefan Hajnoczi 
72461007b31SStefan Hajnoczi /*
72561007b31SStefan Hajnoczi  * Process a synchronous request using coroutines
72661007b31SStefan Hajnoczi  */
727e293b7a3SKevin Wolf static int bdrv_rw_co(BdrvChild *child, int64_t sector_num, uint8_t *buf,
72861007b31SStefan Hajnoczi                       int nb_sectors, bool is_write, BdrvRequestFlags flags)
72961007b31SStefan Hajnoczi {
73061007b31SStefan Hajnoczi     QEMUIOVector qiov;
73161007b31SStefan Hajnoczi     struct iovec iov = {
73261007b31SStefan Hajnoczi         .iov_base = (void *)buf,
73361007b31SStefan Hajnoczi         .iov_len = nb_sectors * BDRV_SECTOR_SIZE,
73461007b31SStefan Hajnoczi     };
73561007b31SStefan Hajnoczi 
73661007b31SStefan Hajnoczi     if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
73761007b31SStefan Hajnoczi         return -EINVAL;
73861007b31SStefan Hajnoczi     }
73961007b31SStefan Hajnoczi 
74061007b31SStefan Hajnoczi     qemu_iovec_init_external(&qiov, &iov, 1);
741e293b7a3SKevin Wolf     return bdrv_prwv_co(child, sector_num << BDRV_SECTOR_BITS,
74261007b31SStefan Hajnoczi                         &qiov, is_write, flags);
74361007b31SStefan Hajnoczi }
74461007b31SStefan Hajnoczi 
74561007b31SStefan Hajnoczi /* return < 0 if error. See bdrv_write() for the return codes */
746fbcbbf4eSKevin Wolf int bdrv_read(BdrvChild *child, int64_t sector_num,
74761007b31SStefan Hajnoczi               uint8_t *buf, int nb_sectors)
74861007b31SStefan Hajnoczi {
749e293b7a3SKevin Wolf     return bdrv_rw_co(child, sector_num, buf, nb_sectors, false, 0);
75061007b31SStefan Hajnoczi }
75161007b31SStefan Hajnoczi 
75261007b31SStefan Hajnoczi /* Return < 0 if error. Important errors are:
75361007b31SStefan Hajnoczi   -EIO         generic I/O error (may happen for all errors)
75461007b31SStefan Hajnoczi   -ENOMEDIUM   No media inserted.
75561007b31SStefan Hajnoczi   -EINVAL      Invalid sector number or nb_sectors
75661007b31SStefan Hajnoczi   -EACCES      Trying to write a read-only device
75761007b31SStefan Hajnoczi */
75818d51c4bSKevin Wolf int bdrv_write(BdrvChild *child, int64_t sector_num,
75961007b31SStefan Hajnoczi                const uint8_t *buf, int nb_sectors)
76061007b31SStefan Hajnoczi {
761e293b7a3SKevin Wolf     return bdrv_rw_co(child, sector_num, (uint8_t *)buf, nb_sectors, true, 0);
76261007b31SStefan Hajnoczi }
76361007b31SStefan Hajnoczi 
764720ff280SKevin Wolf int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset,
765f5a5ca79SManos Pitsidianakis                        int bytes, BdrvRequestFlags flags)
76661007b31SStefan Hajnoczi {
76774021bc4SEric Blake     QEMUIOVector qiov;
76874021bc4SEric Blake     struct iovec iov = {
76974021bc4SEric Blake         .iov_base = NULL,
770f5a5ca79SManos Pitsidianakis         .iov_len = bytes,
77174021bc4SEric Blake     };
77274021bc4SEric Blake 
77374021bc4SEric Blake     qemu_iovec_init_external(&qiov, &iov, 1);
774e293b7a3SKevin Wolf     return bdrv_prwv_co(child, offset, &qiov, true,
77561007b31SStefan Hajnoczi                         BDRV_REQ_ZERO_WRITE | flags);
77661007b31SStefan Hajnoczi }
77761007b31SStefan Hajnoczi 
77861007b31SStefan Hajnoczi /*
77974021bc4SEric Blake  * Completely zero out a block device with the help of bdrv_pwrite_zeroes.
78061007b31SStefan Hajnoczi  * The operation is sped up by checking the block status and only writing
78161007b31SStefan Hajnoczi  * zeroes to the device if they currently do not return zeroes. Optional
78274021bc4SEric Blake  * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
783465fe887SEric Blake  * BDRV_REQ_FUA).
78461007b31SStefan Hajnoczi  *
78561007b31SStefan Hajnoczi  * Returns < 0 on error, 0 on success. For error codes see bdrv_write().
78661007b31SStefan Hajnoczi  */
787720ff280SKevin Wolf int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags)
78861007b31SStefan Hajnoczi {
789237d78f8SEric Blake     int ret;
790237d78f8SEric Blake     int64_t target_size, bytes, offset = 0;
791720ff280SKevin Wolf     BlockDriverState *bs = child->bs;
79261007b31SStefan Hajnoczi 
7937286d610SEric Blake     target_size = bdrv_getlength(bs);
7947286d610SEric Blake     if (target_size < 0) {
7957286d610SEric Blake         return target_size;
79661007b31SStefan Hajnoczi     }
79761007b31SStefan Hajnoczi 
79861007b31SStefan Hajnoczi     for (;;) {
7997286d610SEric Blake         bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES);
8007286d610SEric Blake         if (bytes <= 0) {
80161007b31SStefan Hajnoczi             return 0;
80261007b31SStefan Hajnoczi         }
803237d78f8SEric Blake         ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL);
80461007b31SStefan Hajnoczi         if (ret < 0) {
8057286d610SEric Blake             error_report("error getting block status at offset %" PRId64 ": %s",
8067286d610SEric Blake                          offset, strerror(-ret));
80761007b31SStefan Hajnoczi             return ret;
80861007b31SStefan Hajnoczi         }
80961007b31SStefan Hajnoczi         if (ret & BDRV_BLOCK_ZERO) {
810237d78f8SEric Blake             offset += bytes;
81161007b31SStefan Hajnoczi             continue;
81261007b31SStefan Hajnoczi         }
813237d78f8SEric Blake         ret = bdrv_pwrite_zeroes(child, offset, bytes, flags);
81461007b31SStefan Hajnoczi         if (ret < 0) {
8157286d610SEric Blake             error_report("error writing zeroes at offset %" PRId64 ": %s",
8167286d610SEric Blake                          offset, strerror(-ret));
81761007b31SStefan Hajnoczi             return ret;
81861007b31SStefan Hajnoczi         }
819237d78f8SEric Blake         offset += bytes;
82061007b31SStefan Hajnoczi     }
82161007b31SStefan Hajnoczi }
82261007b31SStefan Hajnoczi 
823cf2ab8fcSKevin Wolf int bdrv_preadv(BdrvChild *child, int64_t offset, QEMUIOVector *qiov)
824f1e84741SKevin Wolf {
825f1e84741SKevin Wolf     int ret;
826f1e84741SKevin Wolf 
827e293b7a3SKevin Wolf     ret = bdrv_prwv_co(child, offset, qiov, false, 0);
828f1e84741SKevin Wolf     if (ret < 0) {
829f1e84741SKevin Wolf         return ret;
830f1e84741SKevin Wolf     }
831f1e84741SKevin Wolf 
832f1e84741SKevin Wolf     return qiov->size;
833f1e84741SKevin Wolf }
834f1e84741SKevin Wolf 
835cf2ab8fcSKevin Wolf int bdrv_pread(BdrvChild *child, int64_t offset, void *buf, int bytes)
83661007b31SStefan Hajnoczi {
83761007b31SStefan Hajnoczi     QEMUIOVector qiov;
83861007b31SStefan Hajnoczi     struct iovec iov = {
83961007b31SStefan Hajnoczi         .iov_base = (void *)buf,
84061007b31SStefan Hajnoczi         .iov_len = bytes,
84161007b31SStefan Hajnoczi     };
84261007b31SStefan Hajnoczi 
84361007b31SStefan Hajnoczi     if (bytes < 0) {
84461007b31SStefan Hajnoczi         return -EINVAL;
84561007b31SStefan Hajnoczi     }
84661007b31SStefan Hajnoczi 
84761007b31SStefan Hajnoczi     qemu_iovec_init_external(&qiov, &iov, 1);
848cf2ab8fcSKevin Wolf     return bdrv_preadv(child, offset, &qiov);
84961007b31SStefan Hajnoczi }
85061007b31SStefan Hajnoczi 
851d9ca2ea2SKevin Wolf int bdrv_pwritev(BdrvChild *child, int64_t offset, QEMUIOVector *qiov)
85261007b31SStefan Hajnoczi {
85361007b31SStefan Hajnoczi     int ret;
85461007b31SStefan Hajnoczi 
855e293b7a3SKevin Wolf     ret = bdrv_prwv_co(child, offset, qiov, true, 0);
85661007b31SStefan Hajnoczi     if (ret < 0) {
85761007b31SStefan Hajnoczi         return ret;
85861007b31SStefan Hajnoczi     }
85961007b31SStefan Hajnoczi 
86061007b31SStefan Hajnoczi     return qiov->size;
86161007b31SStefan Hajnoczi }
86261007b31SStefan Hajnoczi 
863d9ca2ea2SKevin Wolf int bdrv_pwrite(BdrvChild *child, int64_t offset, const void *buf, int bytes)
86461007b31SStefan Hajnoczi {
86561007b31SStefan Hajnoczi     QEMUIOVector qiov;
86661007b31SStefan Hajnoczi     struct iovec iov = {
86761007b31SStefan Hajnoczi         .iov_base   = (void *) buf,
86861007b31SStefan Hajnoczi         .iov_len    = bytes,
86961007b31SStefan Hajnoczi     };
87061007b31SStefan Hajnoczi 
87161007b31SStefan Hajnoczi     if (bytes < 0) {
87261007b31SStefan Hajnoczi         return -EINVAL;
87361007b31SStefan Hajnoczi     }
87461007b31SStefan Hajnoczi 
87561007b31SStefan Hajnoczi     qemu_iovec_init_external(&qiov, &iov, 1);
876d9ca2ea2SKevin Wolf     return bdrv_pwritev(child, offset, &qiov);
87761007b31SStefan Hajnoczi }
87861007b31SStefan Hajnoczi 
87961007b31SStefan Hajnoczi /*
88061007b31SStefan Hajnoczi  * Writes to the file and ensures that no writes are reordered across this
88161007b31SStefan Hajnoczi  * request (acts as a barrier)
88261007b31SStefan Hajnoczi  *
88361007b31SStefan Hajnoczi  * Returns 0 on success, -errno in error cases.
88461007b31SStefan Hajnoczi  */
885d9ca2ea2SKevin Wolf int bdrv_pwrite_sync(BdrvChild *child, int64_t offset,
88661007b31SStefan Hajnoczi                      const void *buf, int count)
88761007b31SStefan Hajnoczi {
88861007b31SStefan Hajnoczi     int ret;
88961007b31SStefan Hajnoczi 
890d9ca2ea2SKevin Wolf     ret = bdrv_pwrite(child, offset, buf, count);
89161007b31SStefan Hajnoczi     if (ret < 0) {
89261007b31SStefan Hajnoczi         return ret;
89361007b31SStefan Hajnoczi     }
89461007b31SStefan Hajnoczi 
895d9ca2ea2SKevin Wolf     ret = bdrv_flush(child->bs);
896855a6a93SKevin Wolf     if (ret < 0) {
897855a6a93SKevin Wolf         return ret;
89861007b31SStefan Hajnoczi     }
89961007b31SStefan Hajnoczi 
90061007b31SStefan Hajnoczi     return 0;
90161007b31SStefan Hajnoczi }
90261007b31SStefan Hajnoczi 
90308844473SKevin Wolf typedef struct CoroutineIOCompletion {
90408844473SKevin Wolf     Coroutine *coroutine;
90508844473SKevin Wolf     int ret;
90608844473SKevin Wolf } CoroutineIOCompletion;
90708844473SKevin Wolf 
90808844473SKevin Wolf static void bdrv_co_io_em_complete(void *opaque, int ret)
90908844473SKevin Wolf {
91008844473SKevin Wolf     CoroutineIOCompletion *co = opaque;
91108844473SKevin Wolf 
91208844473SKevin Wolf     co->ret = ret;
913b9e413ddSPaolo Bonzini     aio_co_wake(co->coroutine);
91408844473SKevin Wolf }
91508844473SKevin Wolf 
916166fe960SKevin Wolf static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
917166fe960SKevin Wolf                                            uint64_t offset, uint64_t bytes,
918166fe960SKevin Wolf                                            QEMUIOVector *qiov, int flags)
919166fe960SKevin Wolf {
920166fe960SKevin Wolf     BlockDriver *drv = bs->drv;
9213fb06697SKevin Wolf     int64_t sector_num;
9223fb06697SKevin Wolf     unsigned int nb_sectors;
9233fb06697SKevin Wolf 
924fa166538SEric Blake     assert(!(flags & ~BDRV_REQ_MASK));
925fa166538SEric Blake 
926d470ad42SMax Reitz     if (!drv) {
927d470ad42SMax Reitz         return -ENOMEDIUM;
928d470ad42SMax Reitz     }
929d470ad42SMax Reitz 
9303fb06697SKevin Wolf     if (drv->bdrv_co_preadv) {
9313fb06697SKevin Wolf         return drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
9323fb06697SKevin Wolf     }
9333fb06697SKevin Wolf 
9343fb06697SKevin Wolf     sector_num = offset >> BDRV_SECTOR_BITS;
9353fb06697SKevin Wolf     nb_sectors = bytes >> BDRV_SECTOR_BITS;
936166fe960SKevin Wolf 
937166fe960SKevin Wolf     assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
938166fe960SKevin Wolf     assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
939166fe960SKevin Wolf     assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
940166fe960SKevin Wolf 
94108844473SKevin Wolf     if (drv->bdrv_co_readv) {
942166fe960SKevin Wolf         return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
94308844473SKevin Wolf     } else {
94408844473SKevin Wolf         BlockAIOCB *acb;
94508844473SKevin Wolf         CoroutineIOCompletion co = {
94608844473SKevin Wolf             .coroutine = qemu_coroutine_self(),
94708844473SKevin Wolf         };
94808844473SKevin Wolf 
94908844473SKevin Wolf         acb = bs->drv->bdrv_aio_readv(bs, sector_num, qiov, nb_sectors,
95008844473SKevin Wolf                                       bdrv_co_io_em_complete, &co);
95108844473SKevin Wolf         if (acb == NULL) {
95208844473SKevin Wolf             return -EIO;
95308844473SKevin Wolf         } else {
95408844473SKevin Wolf             qemu_coroutine_yield();
95508844473SKevin Wolf             return co.ret;
95608844473SKevin Wolf         }
95708844473SKevin Wolf     }
958166fe960SKevin Wolf }
959166fe960SKevin Wolf 
96078a07294SKevin Wolf static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
96178a07294SKevin Wolf                                             uint64_t offset, uint64_t bytes,
96278a07294SKevin Wolf                                             QEMUIOVector *qiov, int flags)
96378a07294SKevin Wolf {
96478a07294SKevin Wolf     BlockDriver *drv = bs->drv;
9653fb06697SKevin Wolf     int64_t sector_num;
9663fb06697SKevin Wolf     unsigned int nb_sectors;
96778a07294SKevin Wolf     int ret;
96878a07294SKevin Wolf 
969fa166538SEric Blake     assert(!(flags & ~BDRV_REQ_MASK));
970fa166538SEric Blake 
971d470ad42SMax Reitz     if (!drv) {
972d470ad42SMax Reitz         return -ENOMEDIUM;
973d470ad42SMax Reitz     }
974d470ad42SMax Reitz 
9753fb06697SKevin Wolf     if (drv->bdrv_co_pwritev) {
976515c2f43SKevin Wolf         ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov,
977515c2f43SKevin Wolf                                    flags & bs->supported_write_flags);
978515c2f43SKevin Wolf         flags &= ~bs->supported_write_flags;
9793fb06697SKevin Wolf         goto emulate_flags;
9803fb06697SKevin Wolf     }
9813fb06697SKevin Wolf 
9823fb06697SKevin Wolf     sector_num = offset >> BDRV_SECTOR_BITS;
9833fb06697SKevin Wolf     nb_sectors = bytes >> BDRV_SECTOR_BITS;
9843fb06697SKevin Wolf 
98578a07294SKevin Wolf     assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
98678a07294SKevin Wolf     assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
98778a07294SKevin Wolf     assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
98878a07294SKevin Wolf 
98978a07294SKevin Wolf     if (drv->bdrv_co_writev_flags) {
99078a07294SKevin Wolf         ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov,
9914df863f3SEric Blake                                         flags & bs->supported_write_flags);
9924df863f3SEric Blake         flags &= ~bs->supported_write_flags;
99308844473SKevin Wolf     } else if (drv->bdrv_co_writev) {
9944df863f3SEric Blake         assert(!bs->supported_write_flags);
99578a07294SKevin Wolf         ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov);
99608844473SKevin Wolf     } else {
99708844473SKevin Wolf         BlockAIOCB *acb;
99808844473SKevin Wolf         CoroutineIOCompletion co = {
99908844473SKevin Wolf             .coroutine = qemu_coroutine_self(),
100008844473SKevin Wolf         };
100108844473SKevin Wolf 
100208844473SKevin Wolf         acb = bs->drv->bdrv_aio_writev(bs, sector_num, qiov, nb_sectors,
100308844473SKevin Wolf                                        bdrv_co_io_em_complete, &co);
100408844473SKevin Wolf         if (acb == NULL) {
10053fb06697SKevin Wolf             ret = -EIO;
100608844473SKevin Wolf         } else {
100708844473SKevin Wolf             qemu_coroutine_yield();
10083fb06697SKevin Wolf             ret = co.ret;
100908844473SKevin Wolf         }
101078a07294SKevin Wolf     }
101178a07294SKevin Wolf 
10123fb06697SKevin Wolf emulate_flags:
10134df863f3SEric Blake     if (ret == 0 && (flags & BDRV_REQ_FUA)) {
101478a07294SKevin Wolf         ret = bdrv_co_flush(bs);
101578a07294SKevin Wolf     }
101678a07294SKevin Wolf 
101778a07294SKevin Wolf     return ret;
101878a07294SKevin Wolf }
101978a07294SKevin Wolf 
102029a298afSPavel Butsykin static int coroutine_fn
102129a298afSPavel Butsykin bdrv_driver_pwritev_compressed(BlockDriverState *bs, uint64_t offset,
102229a298afSPavel Butsykin                                uint64_t bytes, QEMUIOVector *qiov)
102329a298afSPavel Butsykin {
102429a298afSPavel Butsykin     BlockDriver *drv = bs->drv;
102529a298afSPavel Butsykin 
1026d470ad42SMax Reitz     if (!drv) {
1027d470ad42SMax Reitz         return -ENOMEDIUM;
1028d470ad42SMax Reitz     }
1029d470ad42SMax Reitz 
103029a298afSPavel Butsykin     if (!drv->bdrv_co_pwritev_compressed) {
103129a298afSPavel Butsykin         return -ENOTSUP;
103229a298afSPavel Butsykin     }
103329a298afSPavel Butsykin 
103429a298afSPavel Butsykin     return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov);
103529a298afSPavel Butsykin }
103629a298afSPavel Butsykin 
103785c97ca7SKevin Wolf static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child,
1038244483e6SKevin Wolf         int64_t offset, unsigned int bytes, QEMUIOVector *qiov)
103961007b31SStefan Hajnoczi {
104085c97ca7SKevin Wolf     BlockDriverState *bs = child->bs;
104185c97ca7SKevin Wolf 
104261007b31SStefan Hajnoczi     /* Perform I/O through a temporary buffer so that users who scribble over
104361007b31SStefan Hajnoczi      * their read buffer while the operation is in progress do not end up
104461007b31SStefan Hajnoczi      * modifying the image file.  This is critical for zero-copy guest I/O
104561007b31SStefan Hajnoczi      * where anything might happen inside guest memory.
104661007b31SStefan Hajnoczi      */
104761007b31SStefan Hajnoczi     void *bounce_buffer;
104861007b31SStefan Hajnoczi 
104961007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
105061007b31SStefan Hajnoczi     struct iovec iov;
1051cb2e2878SEric Blake     QEMUIOVector local_qiov;
1052244483e6SKevin Wolf     int64_t cluster_offset;
10537cfd5275SEric Blake     int64_t cluster_bytes;
105461007b31SStefan Hajnoczi     size_t skip_bytes;
105561007b31SStefan Hajnoczi     int ret;
1056cb2e2878SEric Blake     int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer,
1057cb2e2878SEric Blake                                     BDRV_REQUEST_MAX_BYTES);
1058cb2e2878SEric Blake     unsigned int progress = 0;
105961007b31SStefan Hajnoczi 
1060d470ad42SMax Reitz     if (!drv) {
1061d470ad42SMax Reitz         return -ENOMEDIUM;
1062d470ad42SMax Reitz     }
1063d470ad42SMax Reitz 
10641bf03e66SKevin Wolf     /* FIXME We cannot require callers to have write permissions when all they
10651bf03e66SKevin Wolf      * are doing is a read request. If we did things right, write permissions
10661bf03e66SKevin Wolf      * would be obtained anyway, but internally by the copy-on-read code. As
1067765d9df9SEric Blake      * long as it is implemented here rather than in a separate filter driver,
10681bf03e66SKevin Wolf      * the copy-on-read code doesn't have its own BdrvChild, however, for which
10691bf03e66SKevin Wolf      * it could request permissions. Therefore we have to bypass the permission
10701bf03e66SKevin Wolf      * system for the moment. */
10711bf03e66SKevin Wolf     // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
1072afa4b293SKevin Wolf 
107361007b31SStefan Hajnoczi     /* Cover entire cluster so no additional backing file I/O is required when
1074cb2e2878SEric Blake      * allocating cluster in the image file.  Note that this value may exceed
1075cb2e2878SEric Blake      * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which
1076cb2e2878SEric Blake      * is one reason we loop rather than doing it all at once.
107761007b31SStefan Hajnoczi      */
1078244483e6SKevin Wolf     bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes);
1079cb2e2878SEric Blake     skip_bytes = offset - cluster_offset;
108061007b31SStefan Hajnoczi 
1081244483e6SKevin Wolf     trace_bdrv_co_do_copy_on_readv(bs, offset, bytes,
1082244483e6SKevin Wolf                                    cluster_offset, cluster_bytes);
108361007b31SStefan Hajnoczi 
1084cb2e2878SEric Blake     bounce_buffer = qemu_try_blockalign(bs,
1085cb2e2878SEric Blake                                         MIN(MIN(max_transfer, cluster_bytes),
1086cb2e2878SEric Blake                                             MAX_BOUNCE_BUFFER));
108761007b31SStefan Hajnoczi     if (bounce_buffer == NULL) {
108861007b31SStefan Hajnoczi         ret = -ENOMEM;
108961007b31SStefan Hajnoczi         goto err;
109061007b31SStefan Hajnoczi     }
109161007b31SStefan Hajnoczi 
1092cb2e2878SEric Blake     while (cluster_bytes) {
1093cb2e2878SEric Blake         int64_t pnum;
109461007b31SStefan Hajnoczi 
1095cb2e2878SEric Blake         ret = bdrv_is_allocated(bs, cluster_offset,
1096cb2e2878SEric Blake                                 MIN(cluster_bytes, max_transfer), &pnum);
1097cb2e2878SEric Blake         if (ret < 0) {
1098cb2e2878SEric Blake             /* Safe to treat errors in querying allocation as if
1099cb2e2878SEric Blake              * unallocated; we'll probably fail again soon on the
1100cb2e2878SEric Blake              * read, but at least that will set a decent errno.
1101cb2e2878SEric Blake              */
1102cb2e2878SEric Blake             pnum = MIN(cluster_bytes, max_transfer);
1103cb2e2878SEric Blake         }
1104cb2e2878SEric Blake 
1105cb2e2878SEric Blake         assert(skip_bytes < pnum);
1106cb2e2878SEric Blake 
1107cb2e2878SEric Blake         if (ret <= 0) {
1108cb2e2878SEric Blake             /* Must copy-on-read; use the bounce buffer */
1109cb2e2878SEric Blake             iov.iov_base = bounce_buffer;
1110cb2e2878SEric Blake             iov.iov_len = pnum = MIN(pnum, MAX_BOUNCE_BUFFER);
1111cb2e2878SEric Blake             qemu_iovec_init_external(&local_qiov, &iov, 1);
1112cb2e2878SEric Blake 
1113cb2e2878SEric Blake             ret = bdrv_driver_preadv(bs, cluster_offset, pnum,
1114cb2e2878SEric Blake                                      &local_qiov, 0);
111561007b31SStefan Hajnoczi             if (ret < 0) {
111661007b31SStefan Hajnoczi                 goto err;
111761007b31SStefan Hajnoczi             }
111861007b31SStefan Hajnoczi 
1119d855ebcdSEric Blake             bdrv_debug_event(bs, BLKDBG_COR_WRITE);
1120c1499a5eSEric Blake             if (drv->bdrv_co_pwrite_zeroes &&
1121cb2e2878SEric Blake                 buffer_is_zero(bounce_buffer, pnum)) {
1122a604fa2bSEric Blake                 /* FIXME: Should we (perhaps conditionally) be setting
1123a604fa2bSEric Blake                  * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy
1124a604fa2bSEric Blake                  * that still correctly reads as zero? */
1125cb2e2878SEric Blake                 ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, pnum, 0);
112661007b31SStefan Hajnoczi             } else {
1127cb2e2878SEric Blake                 /* This does not change the data on the disk, it is not
1128cb2e2878SEric Blake                  * necessary to flush even in cache=writethrough mode.
112961007b31SStefan Hajnoczi                  */
1130cb2e2878SEric Blake                 ret = bdrv_driver_pwritev(bs, cluster_offset, pnum,
1131cb2e2878SEric Blake                                           &local_qiov, 0);
113261007b31SStefan Hajnoczi             }
113361007b31SStefan Hajnoczi 
113461007b31SStefan Hajnoczi             if (ret < 0) {
1135cb2e2878SEric Blake                 /* It might be okay to ignore write errors for guest
1136cb2e2878SEric Blake                  * requests.  If this is a deliberate copy-on-read
1137cb2e2878SEric Blake                  * then we don't want to ignore the error.  Simply
1138cb2e2878SEric Blake                  * report it in all cases.
113961007b31SStefan Hajnoczi                  */
114061007b31SStefan Hajnoczi                 goto err;
114161007b31SStefan Hajnoczi             }
114261007b31SStefan Hajnoczi 
1143cb2e2878SEric Blake             qemu_iovec_from_buf(qiov, progress, bounce_buffer + skip_bytes,
1144cb2e2878SEric Blake                                 pnum - skip_bytes);
1145cb2e2878SEric Blake         } else {
1146cb2e2878SEric Blake             /* Read directly into the destination */
1147cb2e2878SEric Blake             qemu_iovec_init(&local_qiov, qiov->niov);
1148cb2e2878SEric Blake             qemu_iovec_concat(&local_qiov, qiov, progress, pnum - skip_bytes);
1149cb2e2878SEric Blake             ret = bdrv_driver_preadv(bs, offset + progress, local_qiov.size,
1150cb2e2878SEric Blake                                      &local_qiov, 0);
1151cb2e2878SEric Blake             qemu_iovec_destroy(&local_qiov);
1152cb2e2878SEric Blake             if (ret < 0) {
1153cb2e2878SEric Blake                 goto err;
1154cb2e2878SEric Blake             }
1155cb2e2878SEric Blake         }
1156cb2e2878SEric Blake 
1157cb2e2878SEric Blake         cluster_offset += pnum;
1158cb2e2878SEric Blake         cluster_bytes -= pnum;
1159cb2e2878SEric Blake         progress += pnum - skip_bytes;
1160cb2e2878SEric Blake         skip_bytes = 0;
1161cb2e2878SEric Blake     }
1162cb2e2878SEric Blake     ret = 0;
116361007b31SStefan Hajnoczi 
116461007b31SStefan Hajnoczi err:
116561007b31SStefan Hajnoczi     qemu_vfree(bounce_buffer);
116661007b31SStefan Hajnoczi     return ret;
116761007b31SStefan Hajnoczi }
116861007b31SStefan Hajnoczi 
116961007b31SStefan Hajnoczi /*
117061007b31SStefan Hajnoczi  * Forwards an already correctly aligned request to the BlockDriver. This
11711a62d0acSEric Blake  * handles copy on read, zeroing after EOF, and fragmentation of large
11721a62d0acSEric Blake  * reads; any other features must be implemented by the caller.
117361007b31SStefan Hajnoczi  */
117485c97ca7SKevin Wolf static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,
117561007b31SStefan Hajnoczi     BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
117661007b31SStefan Hajnoczi     int64_t align, QEMUIOVector *qiov, int flags)
117761007b31SStefan Hajnoczi {
117885c97ca7SKevin Wolf     BlockDriverState *bs = child->bs;
1179c9d20029SKevin Wolf     int64_t total_bytes, max_bytes;
11801a62d0acSEric Blake     int ret = 0;
11811a62d0acSEric Blake     uint64_t bytes_remaining = bytes;
11821a62d0acSEric Blake     int max_transfer;
118361007b31SStefan Hajnoczi 
118449c07526SKevin Wolf     assert(is_power_of_2(align));
118549c07526SKevin Wolf     assert((offset & (align - 1)) == 0);
118649c07526SKevin Wolf     assert((bytes & (align - 1)) == 0);
118761007b31SStefan Hajnoczi     assert(!qiov || bytes == qiov->size);
1188abb06c5aSDaniel P. Berrange     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
11891a62d0acSEric Blake     max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
11901a62d0acSEric Blake                                    align);
1191a604fa2bSEric Blake 
1192a604fa2bSEric Blake     /* TODO: We would need a per-BDS .supported_read_flags and
1193a604fa2bSEric Blake      * potential fallback support, if we ever implement any read flags
1194a604fa2bSEric Blake      * to pass through to drivers.  For now, there aren't any
1195a604fa2bSEric Blake      * passthrough flags.  */
1196a604fa2bSEric Blake     assert(!(flags & ~(BDRV_REQ_NO_SERIALISING | BDRV_REQ_COPY_ON_READ)));
119761007b31SStefan Hajnoczi 
119861007b31SStefan Hajnoczi     /* Handle Copy on Read and associated serialisation */
119961007b31SStefan Hajnoczi     if (flags & BDRV_REQ_COPY_ON_READ) {
120061007b31SStefan Hajnoczi         /* If we touch the same cluster it counts as an overlap.  This
120161007b31SStefan Hajnoczi          * guarantees that allocating writes will be serialized and not race
120261007b31SStefan Hajnoczi          * with each other for the same cluster.  For example, in copy-on-read
120361007b31SStefan Hajnoczi          * it ensures that the CoR read and write operations are atomic and
120461007b31SStefan Hajnoczi          * guest writes cannot interleave between them. */
120561007b31SStefan Hajnoczi         mark_request_serialising(req, bdrv_get_cluster_size(bs));
120661007b31SStefan Hajnoczi     }
120761007b31SStefan Hajnoczi 
120861408b25SFam Zheng     if (!(flags & BDRV_REQ_NO_SERIALISING)) {
120961007b31SStefan Hajnoczi         wait_serialising_requests(req);
121061408b25SFam Zheng     }
121161007b31SStefan Hajnoczi 
121261007b31SStefan Hajnoczi     if (flags & BDRV_REQ_COPY_ON_READ) {
1213d6a644bbSEric Blake         int64_t pnum;
121461007b31SStefan Hajnoczi 
121588e63df2SEric Blake         ret = bdrv_is_allocated(bs, offset, bytes, &pnum);
121661007b31SStefan Hajnoczi         if (ret < 0) {
121761007b31SStefan Hajnoczi             goto out;
121861007b31SStefan Hajnoczi         }
121961007b31SStefan Hajnoczi 
122088e63df2SEric Blake         if (!ret || pnum != bytes) {
122185c97ca7SKevin Wolf             ret = bdrv_co_do_copy_on_readv(child, offset, bytes, qiov);
122261007b31SStefan Hajnoczi             goto out;
122361007b31SStefan Hajnoczi         }
122461007b31SStefan Hajnoczi     }
122561007b31SStefan Hajnoczi 
12261a62d0acSEric Blake     /* Forward the request to the BlockDriver, possibly fragmenting it */
122749c07526SKevin Wolf     total_bytes = bdrv_getlength(bs);
122849c07526SKevin Wolf     if (total_bytes < 0) {
122949c07526SKevin Wolf         ret = total_bytes;
123061007b31SStefan Hajnoczi         goto out;
123161007b31SStefan Hajnoczi     }
123261007b31SStefan Hajnoczi 
123349c07526SKevin Wolf     max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align);
12341a62d0acSEric Blake     if (bytes <= max_bytes && bytes <= max_transfer) {
1235166fe960SKevin Wolf         ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0);
12361a62d0acSEric Blake         goto out;
123761007b31SStefan Hajnoczi     }
123861007b31SStefan Hajnoczi 
12391a62d0acSEric Blake     while (bytes_remaining) {
12401a62d0acSEric Blake         int num;
12411a62d0acSEric Blake 
12421a62d0acSEric Blake         if (max_bytes) {
12431a62d0acSEric Blake             QEMUIOVector local_qiov;
12441a62d0acSEric Blake 
12451a62d0acSEric Blake             num = MIN(bytes_remaining, MIN(max_bytes, max_transfer));
12461a62d0acSEric Blake             assert(num);
12471a62d0acSEric Blake             qemu_iovec_init(&local_qiov, qiov->niov);
12481a62d0acSEric Blake             qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num);
12491a62d0acSEric Blake 
12501a62d0acSEric Blake             ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining,
12511a62d0acSEric Blake                                      num, &local_qiov, 0);
12521a62d0acSEric Blake             max_bytes -= num;
12531a62d0acSEric Blake             qemu_iovec_destroy(&local_qiov);
12541a62d0acSEric Blake         } else {
12551a62d0acSEric Blake             num = bytes_remaining;
12561a62d0acSEric Blake             ret = qemu_iovec_memset(qiov, bytes - bytes_remaining, 0,
12571a62d0acSEric Blake                                     bytes_remaining);
12581a62d0acSEric Blake         }
12591a62d0acSEric Blake         if (ret < 0) {
12601a62d0acSEric Blake             goto out;
12611a62d0acSEric Blake         }
12621a62d0acSEric Blake         bytes_remaining -= num;
126361007b31SStefan Hajnoczi     }
126461007b31SStefan Hajnoczi 
126561007b31SStefan Hajnoczi out:
12661a62d0acSEric Blake     return ret < 0 ? ret : 0;
126761007b31SStefan Hajnoczi }
126861007b31SStefan Hajnoczi 
126961007b31SStefan Hajnoczi /*
127061007b31SStefan Hajnoczi  * Handle a read request in coroutine context
127161007b31SStefan Hajnoczi  */
1272a03ef88fSKevin Wolf int coroutine_fn bdrv_co_preadv(BdrvChild *child,
127361007b31SStefan Hajnoczi     int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
127461007b31SStefan Hajnoczi     BdrvRequestFlags flags)
127561007b31SStefan Hajnoczi {
1276a03ef88fSKevin Wolf     BlockDriverState *bs = child->bs;
127761007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
127861007b31SStefan Hajnoczi     BdrvTrackedRequest req;
127961007b31SStefan Hajnoczi 
1280a5b8dd2cSEric Blake     uint64_t align = bs->bl.request_alignment;
128161007b31SStefan Hajnoczi     uint8_t *head_buf = NULL;
128261007b31SStefan Hajnoczi     uint8_t *tail_buf = NULL;
128361007b31SStefan Hajnoczi     QEMUIOVector local_qiov;
128461007b31SStefan Hajnoczi     bool use_local_qiov = false;
128561007b31SStefan Hajnoczi     int ret;
128661007b31SStefan Hajnoczi 
1287f42cf447SDaniel P. Berrange     trace_bdrv_co_preadv(child->bs, offset, bytes, flags);
1288f42cf447SDaniel P. Berrange 
128961007b31SStefan Hajnoczi     if (!drv) {
129061007b31SStefan Hajnoczi         return -ENOMEDIUM;
129161007b31SStefan Hajnoczi     }
129261007b31SStefan Hajnoczi 
129361007b31SStefan Hajnoczi     ret = bdrv_check_byte_request(bs, offset, bytes);
129461007b31SStefan Hajnoczi     if (ret < 0) {
129561007b31SStefan Hajnoczi         return ret;
129661007b31SStefan Hajnoczi     }
129761007b31SStefan Hajnoczi 
129899723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
129999723548SPaolo Bonzini 
13009568b511SWen Congyang     /* Don't do copy-on-read if we read data before write operation */
1301d3faa13eSPaolo Bonzini     if (atomic_read(&bs->copy_on_read) && !(flags & BDRV_REQ_NO_SERIALISING)) {
130261007b31SStefan Hajnoczi         flags |= BDRV_REQ_COPY_ON_READ;
130361007b31SStefan Hajnoczi     }
130461007b31SStefan Hajnoczi 
130561007b31SStefan Hajnoczi     /* Align read if necessary by padding qiov */
130661007b31SStefan Hajnoczi     if (offset & (align - 1)) {
130761007b31SStefan Hajnoczi         head_buf = qemu_blockalign(bs, align);
130861007b31SStefan Hajnoczi         qemu_iovec_init(&local_qiov, qiov->niov + 2);
130961007b31SStefan Hajnoczi         qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
131061007b31SStefan Hajnoczi         qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
131161007b31SStefan Hajnoczi         use_local_qiov = true;
131261007b31SStefan Hajnoczi 
131361007b31SStefan Hajnoczi         bytes += offset & (align - 1);
131461007b31SStefan Hajnoczi         offset = offset & ~(align - 1);
131561007b31SStefan Hajnoczi     }
131661007b31SStefan Hajnoczi 
131761007b31SStefan Hajnoczi     if ((offset + bytes) & (align - 1)) {
131861007b31SStefan Hajnoczi         if (!use_local_qiov) {
131961007b31SStefan Hajnoczi             qemu_iovec_init(&local_qiov, qiov->niov + 1);
132061007b31SStefan Hajnoczi             qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
132161007b31SStefan Hajnoczi             use_local_qiov = true;
132261007b31SStefan Hajnoczi         }
132361007b31SStefan Hajnoczi         tail_buf = qemu_blockalign(bs, align);
132461007b31SStefan Hajnoczi         qemu_iovec_add(&local_qiov, tail_buf,
132561007b31SStefan Hajnoczi                        align - ((offset + bytes) & (align - 1)));
132661007b31SStefan Hajnoczi 
132761007b31SStefan Hajnoczi         bytes = ROUND_UP(bytes, align);
132861007b31SStefan Hajnoczi     }
132961007b31SStefan Hajnoczi 
1330ebde595cSFam Zheng     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
133185c97ca7SKevin Wolf     ret = bdrv_aligned_preadv(child, &req, offset, bytes, align,
133261007b31SStefan Hajnoczi                               use_local_qiov ? &local_qiov : qiov,
133361007b31SStefan Hajnoczi                               flags);
133461007b31SStefan Hajnoczi     tracked_request_end(&req);
133599723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
133661007b31SStefan Hajnoczi 
133761007b31SStefan Hajnoczi     if (use_local_qiov) {
133861007b31SStefan Hajnoczi         qemu_iovec_destroy(&local_qiov);
133961007b31SStefan Hajnoczi         qemu_vfree(head_buf);
134061007b31SStefan Hajnoczi         qemu_vfree(tail_buf);
134161007b31SStefan Hajnoczi     }
134261007b31SStefan Hajnoczi 
134361007b31SStefan Hajnoczi     return ret;
134461007b31SStefan Hajnoczi }
134561007b31SStefan Hajnoczi 
1346adad6496SKevin Wolf static int coroutine_fn bdrv_co_do_readv(BdrvChild *child,
134761007b31SStefan Hajnoczi     int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
134861007b31SStefan Hajnoczi     BdrvRequestFlags flags)
134961007b31SStefan Hajnoczi {
135061007b31SStefan Hajnoczi     if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
135161007b31SStefan Hajnoczi         return -EINVAL;
135261007b31SStefan Hajnoczi     }
135361007b31SStefan Hajnoczi 
1354a03ef88fSKevin Wolf     return bdrv_co_preadv(child, sector_num << BDRV_SECTOR_BITS,
135561007b31SStefan Hajnoczi                           nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
135661007b31SStefan Hajnoczi }
135761007b31SStefan Hajnoczi 
135828b04a8fSKevin Wolf int coroutine_fn bdrv_co_readv(BdrvChild *child, int64_t sector_num,
135961007b31SStefan Hajnoczi                                int nb_sectors, QEMUIOVector *qiov)
136061007b31SStefan Hajnoczi {
1361adad6496SKevin Wolf     return bdrv_co_do_readv(child, sector_num, nb_sectors, qiov, 0);
136261007b31SStefan Hajnoczi }
136361007b31SStefan Hajnoczi 
1364d05aa8bbSEric Blake static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
1365f5a5ca79SManos Pitsidianakis     int64_t offset, int bytes, BdrvRequestFlags flags)
136661007b31SStefan Hajnoczi {
136761007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
136861007b31SStefan Hajnoczi     QEMUIOVector qiov;
136961007b31SStefan Hajnoczi     struct iovec iov = {0};
137061007b31SStefan Hajnoczi     int ret = 0;
1371465fe887SEric Blake     bool need_flush = false;
1372443668caSDenis V. Lunev     int head = 0;
1373443668caSDenis V. Lunev     int tail = 0;
137461007b31SStefan Hajnoczi 
1375cf081fcaSEric Blake     int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, INT_MAX);
1376a5b8dd2cSEric Blake     int alignment = MAX(bs->bl.pwrite_zeroes_alignment,
1377a5b8dd2cSEric Blake                         bs->bl.request_alignment);
1378cb2e2878SEric Blake     int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER);
1379cf081fcaSEric Blake 
1380d470ad42SMax Reitz     if (!drv) {
1381d470ad42SMax Reitz         return -ENOMEDIUM;
1382d470ad42SMax Reitz     }
1383d470ad42SMax Reitz 
1384b8d0a980SEric Blake     assert(alignment % bs->bl.request_alignment == 0);
1385b8d0a980SEric Blake     head = offset % alignment;
1386f5a5ca79SManos Pitsidianakis     tail = (offset + bytes) % alignment;
1387b8d0a980SEric Blake     max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment);
1388b8d0a980SEric Blake     assert(max_write_zeroes >= bs->bl.request_alignment);
138961007b31SStefan Hajnoczi 
1390f5a5ca79SManos Pitsidianakis     while (bytes > 0 && !ret) {
1391f5a5ca79SManos Pitsidianakis         int num = bytes;
139261007b31SStefan Hajnoczi 
139361007b31SStefan Hajnoczi         /* Align request.  Block drivers can expect the "bulk" of the request
1394443668caSDenis V. Lunev          * to be aligned, and that unaligned requests do not cross cluster
1395443668caSDenis V. Lunev          * boundaries.
139661007b31SStefan Hajnoczi          */
1397443668caSDenis V. Lunev         if (head) {
1398b2f95feeSEric Blake             /* Make a small request up to the first aligned sector. For
1399b2f95feeSEric Blake              * convenience, limit this request to max_transfer even if
1400b2f95feeSEric Blake              * we don't need to fall back to writes.  */
1401f5a5ca79SManos Pitsidianakis             num = MIN(MIN(bytes, max_transfer), alignment - head);
1402b2f95feeSEric Blake             head = (head + num) % alignment;
1403b2f95feeSEric Blake             assert(num < max_write_zeroes);
1404d05aa8bbSEric Blake         } else if (tail && num > alignment) {
1405443668caSDenis V. Lunev             /* Shorten the request to the last aligned sector.  */
1406443668caSDenis V. Lunev             num -= tail;
140761007b31SStefan Hajnoczi         }
140861007b31SStefan Hajnoczi 
140961007b31SStefan Hajnoczi         /* limit request size */
141061007b31SStefan Hajnoczi         if (num > max_write_zeroes) {
141161007b31SStefan Hajnoczi             num = max_write_zeroes;
141261007b31SStefan Hajnoczi         }
141361007b31SStefan Hajnoczi 
141461007b31SStefan Hajnoczi         ret = -ENOTSUP;
141561007b31SStefan Hajnoczi         /* First try the efficient write zeroes operation */
1416d05aa8bbSEric Blake         if (drv->bdrv_co_pwrite_zeroes) {
1417d05aa8bbSEric Blake             ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num,
1418d05aa8bbSEric Blake                                              flags & bs->supported_zero_flags);
1419d05aa8bbSEric Blake             if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) &&
1420d05aa8bbSEric Blake                 !(bs->supported_zero_flags & BDRV_REQ_FUA)) {
1421d05aa8bbSEric Blake                 need_flush = true;
1422d05aa8bbSEric Blake             }
1423465fe887SEric Blake         } else {
1424465fe887SEric Blake             assert(!bs->supported_zero_flags);
142561007b31SStefan Hajnoczi         }
142661007b31SStefan Hajnoczi 
142761007b31SStefan Hajnoczi         if (ret == -ENOTSUP) {
142861007b31SStefan Hajnoczi             /* Fall back to bounce buffer if write zeroes is unsupported */
1429465fe887SEric Blake             BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;
1430465fe887SEric Blake 
1431465fe887SEric Blake             if ((flags & BDRV_REQ_FUA) &&
1432465fe887SEric Blake                 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1433465fe887SEric Blake                 /* No need for bdrv_driver_pwrite() to do a fallback
1434465fe887SEric Blake                  * flush on each chunk; use just one at the end */
1435465fe887SEric Blake                 write_flags &= ~BDRV_REQ_FUA;
1436465fe887SEric Blake                 need_flush = true;
1437465fe887SEric Blake             }
14385def6b80SEric Blake             num = MIN(num, max_transfer);
1439d05aa8bbSEric Blake             iov.iov_len = num;
144061007b31SStefan Hajnoczi             if (iov.iov_base == NULL) {
1441d05aa8bbSEric Blake                 iov.iov_base = qemu_try_blockalign(bs, num);
144261007b31SStefan Hajnoczi                 if (iov.iov_base == NULL) {
144361007b31SStefan Hajnoczi                     ret = -ENOMEM;
144461007b31SStefan Hajnoczi                     goto fail;
144561007b31SStefan Hajnoczi                 }
1446d05aa8bbSEric Blake                 memset(iov.iov_base, 0, num);
144761007b31SStefan Hajnoczi             }
144861007b31SStefan Hajnoczi             qemu_iovec_init_external(&qiov, &iov, 1);
144961007b31SStefan Hajnoczi 
1450d05aa8bbSEric Blake             ret = bdrv_driver_pwritev(bs, offset, num, &qiov, write_flags);
145161007b31SStefan Hajnoczi 
145261007b31SStefan Hajnoczi             /* Keep bounce buffer around if it is big enough for all
145361007b31SStefan Hajnoczi              * all future requests.
145461007b31SStefan Hajnoczi              */
14555def6b80SEric Blake             if (num < max_transfer) {
145661007b31SStefan Hajnoczi                 qemu_vfree(iov.iov_base);
145761007b31SStefan Hajnoczi                 iov.iov_base = NULL;
145861007b31SStefan Hajnoczi             }
145961007b31SStefan Hajnoczi         }
146061007b31SStefan Hajnoczi 
1461d05aa8bbSEric Blake         offset += num;
1462f5a5ca79SManos Pitsidianakis         bytes -= num;
146361007b31SStefan Hajnoczi     }
146461007b31SStefan Hajnoczi 
146561007b31SStefan Hajnoczi fail:
1466465fe887SEric Blake     if (ret == 0 && need_flush) {
1467465fe887SEric Blake         ret = bdrv_co_flush(bs);
1468465fe887SEric Blake     }
146961007b31SStefan Hajnoczi     qemu_vfree(iov.iov_base);
147061007b31SStefan Hajnoczi     return ret;
147161007b31SStefan Hajnoczi }
147261007b31SStefan Hajnoczi 
147361007b31SStefan Hajnoczi /*
147404ed95f4SEric Blake  * Forwards an already correctly aligned write request to the BlockDriver,
147504ed95f4SEric Blake  * after possibly fragmenting it.
147661007b31SStefan Hajnoczi  */
147785c97ca7SKevin Wolf static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child,
147861007b31SStefan Hajnoczi     BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
1479cff86b38SEric Blake     int64_t align, QEMUIOVector *qiov, int flags)
148061007b31SStefan Hajnoczi {
148185c97ca7SKevin Wolf     BlockDriverState *bs = child->bs;
148261007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
148361007b31SStefan Hajnoczi     bool waited;
148461007b31SStefan Hajnoczi     int ret;
148561007b31SStefan Hajnoczi 
14869896c876SKevin Wolf     int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
148704ed95f4SEric Blake     uint64_t bytes_remaining = bytes;
148804ed95f4SEric Blake     int max_transfer;
148961007b31SStefan Hajnoczi 
1490d470ad42SMax Reitz     if (!drv) {
1491d470ad42SMax Reitz         return -ENOMEDIUM;
1492d470ad42SMax Reitz     }
1493d470ad42SMax Reitz 
1494d6883bc9SVladimir Sementsov-Ogievskiy     if (bdrv_has_readonly_bitmaps(bs)) {
1495d6883bc9SVladimir Sementsov-Ogievskiy         return -EPERM;
1496d6883bc9SVladimir Sementsov-Ogievskiy     }
1497d6883bc9SVladimir Sementsov-Ogievskiy 
1498cff86b38SEric Blake     assert(is_power_of_2(align));
1499cff86b38SEric Blake     assert((offset & (align - 1)) == 0);
1500cff86b38SEric Blake     assert((bytes & (align - 1)) == 0);
150161007b31SStefan Hajnoczi     assert(!qiov || bytes == qiov->size);
1502abb06c5aSDaniel P. Berrange     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1503fa166538SEric Blake     assert(!(flags & ~BDRV_REQ_MASK));
150404ed95f4SEric Blake     max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
150504ed95f4SEric Blake                                    align);
150661007b31SStefan Hajnoczi 
150761007b31SStefan Hajnoczi     waited = wait_serialising_requests(req);
150861007b31SStefan Hajnoczi     assert(!waited || !req->serialising);
150961007b31SStefan Hajnoczi     assert(req->overlap_offset <= offset);
151061007b31SStefan Hajnoczi     assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
1511362b3786SMax Reitz     assert(child->perm & BLK_PERM_WRITE);
1512362b3786SMax Reitz     assert(end_sector <= bs->total_sectors || child->perm & BLK_PERM_RESIZE);
151361007b31SStefan Hajnoczi 
151461007b31SStefan Hajnoczi     ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req);
151561007b31SStefan Hajnoczi 
151661007b31SStefan Hajnoczi     if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
1517c1499a5eSEric Blake         !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes &&
151861007b31SStefan Hajnoczi         qemu_iovec_is_zero(qiov)) {
151961007b31SStefan Hajnoczi         flags |= BDRV_REQ_ZERO_WRITE;
152061007b31SStefan Hajnoczi         if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
152161007b31SStefan Hajnoczi             flags |= BDRV_REQ_MAY_UNMAP;
152261007b31SStefan Hajnoczi         }
152361007b31SStefan Hajnoczi     }
152461007b31SStefan Hajnoczi 
152561007b31SStefan Hajnoczi     if (ret < 0) {
152661007b31SStefan Hajnoczi         /* Do nothing, write notifier decided to fail this request */
152761007b31SStefan Hajnoczi     } else if (flags & BDRV_REQ_ZERO_WRITE) {
15289a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
15299896c876SKevin Wolf         ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags);
15303ea1a091SPavel Butsykin     } else if (flags & BDRV_REQ_WRITE_COMPRESSED) {
15313ea1a091SPavel Butsykin         ret = bdrv_driver_pwritev_compressed(bs, offset, bytes, qiov);
153204ed95f4SEric Blake     } else if (bytes <= max_transfer) {
15339a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV);
153478a07294SKevin Wolf         ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, flags);
153504ed95f4SEric Blake     } else {
153604ed95f4SEric Blake         bdrv_debug_event(bs, BLKDBG_PWRITEV);
153704ed95f4SEric Blake         while (bytes_remaining) {
153804ed95f4SEric Blake             int num = MIN(bytes_remaining, max_transfer);
153904ed95f4SEric Blake             QEMUIOVector local_qiov;
154004ed95f4SEric Blake             int local_flags = flags;
154104ed95f4SEric Blake 
154204ed95f4SEric Blake             assert(num);
154304ed95f4SEric Blake             if (num < bytes_remaining && (flags & BDRV_REQ_FUA) &&
154404ed95f4SEric Blake                 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
154504ed95f4SEric Blake                 /* If FUA is going to be emulated by flush, we only
154604ed95f4SEric Blake                  * need to flush on the last iteration */
154704ed95f4SEric Blake                 local_flags &= ~BDRV_REQ_FUA;
154804ed95f4SEric Blake             }
154904ed95f4SEric Blake             qemu_iovec_init(&local_qiov, qiov->niov);
155004ed95f4SEric Blake             qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num);
155104ed95f4SEric Blake 
155204ed95f4SEric Blake             ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining,
155304ed95f4SEric Blake                                       num, &local_qiov, local_flags);
155404ed95f4SEric Blake             qemu_iovec_destroy(&local_qiov);
155504ed95f4SEric Blake             if (ret < 0) {
155604ed95f4SEric Blake                 break;
155704ed95f4SEric Blake             }
155804ed95f4SEric Blake             bytes_remaining -= num;
155904ed95f4SEric Blake         }
156061007b31SStefan Hajnoczi     }
15619a4f4c31SKevin Wolf     bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
156261007b31SStefan Hajnoczi 
156347fec599SPaolo Bonzini     atomic_inc(&bs->write_gen);
15640fdf1a4fSEric Blake     bdrv_set_dirty(bs, offset, bytes);
156561007b31SStefan Hajnoczi 
1566f7946da2SPaolo Bonzini     stat64_max(&bs->wr_highest_offset, offset + bytes);
156761007b31SStefan Hajnoczi 
156861007b31SStefan Hajnoczi     if (ret >= 0) {
15699896c876SKevin Wolf         bs->total_sectors = MAX(bs->total_sectors, end_sector);
157004ed95f4SEric Blake         ret = 0;
157161007b31SStefan Hajnoczi     }
157261007b31SStefan Hajnoczi 
157361007b31SStefan Hajnoczi     return ret;
157461007b31SStefan Hajnoczi }
157561007b31SStefan Hajnoczi 
157685c97ca7SKevin Wolf static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child,
15779eeb6dd1SFam Zheng                                                 int64_t offset,
15789eeb6dd1SFam Zheng                                                 unsigned int bytes,
15799eeb6dd1SFam Zheng                                                 BdrvRequestFlags flags,
15809eeb6dd1SFam Zheng                                                 BdrvTrackedRequest *req)
15819eeb6dd1SFam Zheng {
158285c97ca7SKevin Wolf     BlockDriverState *bs = child->bs;
15839eeb6dd1SFam Zheng     uint8_t *buf = NULL;
15849eeb6dd1SFam Zheng     QEMUIOVector local_qiov;
15859eeb6dd1SFam Zheng     struct iovec iov;
1586a5b8dd2cSEric Blake     uint64_t align = bs->bl.request_alignment;
15879eeb6dd1SFam Zheng     unsigned int head_padding_bytes, tail_padding_bytes;
15889eeb6dd1SFam Zheng     int ret = 0;
15899eeb6dd1SFam Zheng 
15909eeb6dd1SFam Zheng     head_padding_bytes = offset & (align - 1);
1591f13ce1beSDenis V. Lunev     tail_padding_bytes = (align - (offset + bytes)) & (align - 1);
15929eeb6dd1SFam Zheng 
15939eeb6dd1SFam Zheng 
15949eeb6dd1SFam Zheng     assert(flags & BDRV_REQ_ZERO_WRITE);
15959eeb6dd1SFam Zheng     if (head_padding_bytes || tail_padding_bytes) {
15969eeb6dd1SFam Zheng         buf = qemu_blockalign(bs, align);
15979eeb6dd1SFam Zheng         iov = (struct iovec) {
15989eeb6dd1SFam Zheng             .iov_base   = buf,
15999eeb6dd1SFam Zheng             .iov_len    = align,
16009eeb6dd1SFam Zheng         };
16019eeb6dd1SFam Zheng         qemu_iovec_init_external(&local_qiov, &iov, 1);
16029eeb6dd1SFam Zheng     }
16039eeb6dd1SFam Zheng     if (head_padding_bytes) {
16049eeb6dd1SFam Zheng         uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes);
16059eeb6dd1SFam Zheng 
16069eeb6dd1SFam Zheng         /* RMW the unaligned part before head. */
16079eeb6dd1SFam Zheng         mark_request_serialising(req, align);
16089eeb6dd1SFam Zheng         wait_serialising_requests(req);
16099a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
161085c97ca7SKevin Wolf         ret = bdrv_aligned_preadv(child, req, offset & ~(align - 1), align,
16119eeb6dd1SFam Zheng                                   align, &local_qiov, 0);
16129eeb6dd1SFam Zheng         if (ret < 0) {
16139eeb6dd1SFam Zheng             goto fail;
16149eeb6dd1SFam Zheng         }
16159a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
16169eeb6dd1SFam Zheng 
16179eeb6dd1SFam Zheng         memset(buf + head_padding_bytes, 0, zero_bytes);
161885c97ca7SKevin Wolf         ret = bdrv_aligned_pwritev(child, req, offset & ~(align - 1), align,
1619cff86b38SEric Blake                                    align, &local_qiov,
16209eeb6dd1SFam Zheng                                    flags & ~BDRV_REQ_ZERO_WRITE);
16219eeb6dd1SFam Zheng         if (ret < 0) {
16229eeb6dd1SFam Zheng             goto fail;
16239eeb6dd1SFam Zheng         }
16249eeb6dd1SFam Zheng         offset += zero_bytes;
16259eeb6dd1SFam Zheng         bytes -= zero_bytes;
16269eeb6dd1SFam Zheng     }
16279eeb6dd1SFam Zheng 
16289eeb6dd1SFam Zheng     assert(!bytes || (offset & (align - 1)) == 0);
16299eeb6dd1SFam Zheng     if (bytes >= align) {
16309eeb6dd1SFam Zheng         /* Write the aligned part in the middle. */
16319eeb6dd1SFam Zheng         uint64_t aligned_bytes = bytes & ~(align - 1);
163285c97ca7SKevin Wolf         ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align,
16339eeb6dd1SFam Zheng                                    NULL, flags);
16349eeb6dd1SFam Zheng         if (ret < 0) {
16359eeb6dd1SFam Zheng             goto fail;
16369eeb6dd1SFam Zheng         }
16379eeb6dd1SFam Zheng         bytes -= aligned_bytes;
16389eeb6dd1SFam Zheng         offset += aligned_bytes;
16399eeb6dd1SFam Zheng     }
16409eeb6dd1SFam Zheng 
16419eeb6dd1SFam Zheng     assert(!bytes || (offset & (align - 1)) == 0);
16429eeb6dd1SFam Zheng     if (bytes) {
16439eeb6dd1SFam Zheng         assert(align == tail_padding_bytes + bytes);
16449eeb6dd1SFam Zheng         /* RMW the unaligned part after tail. */
16459eeb6dd1SFam Zheng         mark_request_serialising(req, align);
16469eeb6dd1SFam Zheng         wait_serialising_requests(req);
16479a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
164885c97ca7SKevin Wolf         ret = bdrv_aligned_preadv(child, req, offset, align,
16499eeb6dd1SFam Zheng                                   align, &local_qiov, 0);
16509eeb6dd1SFam Zheng         if (ret < 0) {
16519eeb6dd1SFam Zheng             goto fail;
16529eeb6dd1SFam Zheng         }
16539a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
16549eeb6dd1SFam Zheng 
16559eeb6dd1SFam Zheng         memset(buf, 0, bytes);
165685c97ca7SKevin Wolf         ret = bdrv_aligned_pwritev(child, req, offset, align, align,
16579eeb6dd1SFam Zheng                                    &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE);
16589eeb6dd1SFam Zheng     }
16599eeb6dd1SFam Zheng fail:
16609eeb6dd1SFam Zheng     qemu_vfree(buf);
16619eeb6dd1SFam Zheng     return ret;
16629eeb6dd1SFam Zheng 
16639eeb6dd1SFam Zheng }
16649eeb6dd1SFam Zheng 
166561007b31SStefan Hajnoczi /*
166661007b31SStefan Hajnoczi  * Handle a write request in coroutine context
166761007b31SStefan Hajnoczi  */
1668a03ef88fSKevin Wolf int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
166961007b31SStefan Hajnoczi     int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
167061007b31SStefan Hajnoczi     BdrvRequestFlags flags)
167161007b31SStefan Hajnoczi {
1672a03ef88fSKevin Wolf     BlockDriverState *bs = child->bs;
167361007b31SStefan Hajnoczi     BdrvTrackedRequest req;
1674a5b8dd2cSEric Blake     uint64_t align = bs->bl.request_alignment;
167561007b31SStefan Hajnoczi     uint8_t *head_buf = NULL;
167661007b31SStefan Hajnoczi     uint8_t *tail_buf = NULL;
167761007b31SStefan Hajnoczi     QEMUIOVector local_qiov;
167861007b31SStefan Hajnoczi     bool use_local_qiov = false;
167961007b31SStefan Hajnoczi     int ret;
168061007b31SStefan Hajnoczi 
1681f42cf447SDaniel P. Berrange     trace_bdrv_co_pwritev(child->bs, offset, bytes, flags);
1682f42cf447SDaniel P. Berrange 
168361007b31SStefan Hajnoczi     if (!bs->drv) {
168461007b31SStefan Hajnoczi         return -ENOMEDIUM;
168561007b31SStefan Hajnoczi     }
168661007b31SStefan Hajnoczi     if (bs->read_only) {
1687eaf5fe2dSPaolo Bonzini         return -EPERM;
168861007b31SStefan Hajnoczi     }
168904c01a5cSKevin Wolf     assert(!(bs->open_flags & BDRV_O_INACTIVE));
169061007b31SStefan Hajnoczi 
169161007b31SStefan Hajnoczi     ret = bdrv_check_byte_request(bs, offset, bytes);
169261007b31SStefan Hajnoczi     if (ret < 0) {
169361007b31SStefan Hajnoczi         return ret;
169461007b31SStefan Hajnoczi     }
169561007b31SStefan Hajnoczi 
169699723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
169761007b31SStefan Hajnoczi     /*
169861007b31SStefan Hajnoczi      * Align write if necessary by performing a read-modify-write cycle.
169961007b31SStefan Hajnoczi      * Pad qiov with the read parts and be sure to have a tracked request not
170061007b31SStefan Hajnoczi      * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle.
170161007b31SStefan Hajnoczi      */
1702ebde595cSFam Zheng     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
170361007b31SStefan Hajnoczi 
17049eeb6dd1SFam Zheng     if (!qiov) {
170585c97ca7SKevin Wolf         ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req);
17069eeb6dd1SFam Zheng         goto out;
17079eeb6dd1SFam Zheng     }
17089eeb6dd1SFam Zheng 
170961007b31SStefan Hajnoczi     if (offset & (align - 1)) {
171061007b31SStefan Hajnoczi         QEMUIOVector head_qiov;
171161007b31SStefan Hajnoczi         struct iovec head_iov;
171261007b31SStefan Hajnoczi 
171361007b31SStefan Hajnoczi         mark_request_serialising(&req, align);
171461007b31SStefan Hajnoczi         wait_serialising_requests(&req);
171561007b31SStefan Hajnoczi 
171661007b31SStefan Hajnoczi         head_buf = qemu_blockalign(bs, align);
171761007b31SStefan Hajnoczi         head_iov = (struct iovec) {
171861007b31SStefan Hajnoczi             .iov_base   = head_buf,
171961007b31SStefan Hajnoczi             .iov_len    = align,
172061007b31SStefan Hajnoczi         };
172161007b31SStefan Hajnoczi         qemu_iovec_init_external(&head_qiov, &head_iov, 1);
172261007b31SStefan Hajnoczi 
17239a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
172485c97ca7SKevin Wolf         ret = bdrv_aligned_preadv(child, &req, offset & ~(align - 1), align,
172561007b31SStefan Hajnoczi                                   align, &head_qiov, 0);
172661007b31SStefan Hajnoczi         if (ret < 0) {
172761007b31SStefan Hajnoczi             goto fail;
172861007b31SStefan Hajnoczi         }
17299a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
173061007b31SStefan Hajnoczi 
173161007b31SStefan Hajnoczi         qemu_iovec_init(&local_qiov, qiov->niov + 2);
173261007b31SStefan Hajnoczi         qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
173361007b31SStefan Hajnoczi         qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
173461007b31SStefan Hajnoczi         use_local_qiov = true;
173561007b31SStefan Hajnoczi 
173661007b31SStefan Hajnoczi         bytes += offset & (align - 1);
173761007b31SStefan Hajnoczi         offset = offset & ~(align - 1);
1738117bc3faSPeter Lieven 
1739117bc3faSPeter Lieven         /* We have read the tail already if the request is smaller
1740117bc3faSPeter Lieven          * than one aligned block.
1741117bc3faSPeter Lieven          */
1742117bc3faSPeter Lieven         if (bytes < align) {
1743117bc3faSPeter Lieven             qemu_iovec_add(&local_qiov, head_buf + bytes, align - bytes);
1744117bc3faSPeter Lieven             bytes = align;
1745117bc3faSPeter Lieven         }
174661007b31SStefan Hajnoczi     }
174761007b31SStefan Hajnoczi 
174861007b31SStefan Hajnoczi     if ((offset + bytes) & (align - 1)) {
174961007b31SStefan Hajnoczi         QEMUIOVector tail_qiov;
175061007b31SStefan Hajnoczi         struct iovec tail_iov;
175161007b31SStefan Hajnoczi         size_t tail_bytes;
175261007b31SStefan Hajnoczi         bool waited;
175361007b31SStefan Hajnoczi 
175461007b31SStefan Hajnoczi         mark_request_serialising(&req, align);
175561007b31SStefan Hajnoczi         waited = wait_serialising_requests(&req);
175661007b31SStefan Hajnoczi         assert(!waited || !use_local_qiov);
175761007b31SStefan Hajnoczi 
175861007b31SStefan Hajnoczi         tail_buf = qemu_blockalign(bs, align);
175961007b31SStefan Hajnoczi         tail_iov = (struct iovec) {
176061007b31SStefan Hajnoczi             .iov_base   = tail_buf,
176161007b31SStefan Hajnoczi             .iov_len    = align,
176261007b31SStefan Hajnoczi         };
176361007b31SStefan Hajnoczi         qemu_iovec_init_external(&tail_qiov, &tail_iov, 1);
176461007b31SStefan Hajnoczi 
17659a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
176685c97ca7SKevin Wolf         ret = bdrv_aligned_preadv(child, &req, (offset + bytes) & ~(align - 1),
176785c97ca7SKevin Wolf                                   align, align, &tail_qiov, 0);
176861007b31SStefan Hajnoczi         if (ret < 0) {
176961007b31SStefan Hajnoczi             goto fail;
177061007b31SStefan Hajnoczi         }
17719a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
177261007b31SStefan Hajnoczi 
177361007b31SStefan Hajnoczi         if (!use_local_qiov) {
177461007b31SStefan Hajnoczi             qemu_iovec_init(&local_qiov, qiov->niov + 1);
177561007b31SStefan Hajnoczi             qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
177661007b31SStefan Hajnoczi             use_local_qiov = true;
177761007b31SStefan Hajnoczi         }
177861007b31SStefan Hajnoczi 
177961007b31SStefan Hajnoczi         tail_bytes = (offset + bytes) & (align - 1);
178061007b31SStefan Hajnoczi         qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes);
178161007b31SStefan Hajnoczi 
178261007b31SStefan Hajnoczi         bytes = ROUND_UP(bytes, align);
178361007b31SStefan Hajnoczi     }
178461007b31SStefan Hajnoczi 
178585c97ca7SKevin Wolf     ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align,
178661007b31SStefan Hajnoczi                                use_local_qiov ? &local_qiov : qiov,
178761007b31SStefan Hajnoczi                                flags);
178861007b31SStefan Hajnoczi 
178961007b31SStefan Hajnoczi fail:
179061007b31SStefan Hajnoczi 
179161007b31SStefan Hajnoczi     if (use_local_qiov) {
179261007b31SStefan Hajnoczi         qemu_iovec_destroy(&local_qiov);
179361007b31SStefan Hajnoczi     }
179461007b31SStefan Hajnoczi     qemu_vfree(head_buf);
179561007b31SStefan Hajnoczi     qemu_vfree(tail_buf);
17969eeb6dd1SFam Zheng out:
17979eeb6dd1SFam Zheng     tracked_request_end(&req);
179899723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
179961007b31SStefan Hajnoczi     return ret;
180061007b31SStefan Hajnoczi }
180161007b31SStefan Hajnoczi 
1802adad6496SKevin Wolf static int coroutine_fn bdrv_co_do_writev(BdrvChild *child,
180361007b31SStefan Hajnoczi     int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
180461007b31SStefan Hajnoczi     BdrvRequestFlags flags)
180561007b31SStefan Hajnoczi {
180661007b31SStefan Hajnoczi     if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
180761007b31SStefan Hajnoczi         return -EINVAL;
180861007b31SStefan Hajnoczi     }
180961007b31SStefan Hajnoczi 
1810a03ef88fSKevin Wolf     return bdrv_co_pwritev(child, sector_num << BDRV_SECTOR_BITS,
181161007b31SStefan Hajnoczi                            nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
181261007b31SStefan Hajnoczi }
181361007b31SStefan Hajnoczi 
181425ec177dSKevin Wolf int coroutine_fn bdrv_co_writev(BdrvChild *child, int64_t sector_num,
181561007b31SStefan Hajnoczi     int nb_sectors, QEMUIOVector *qiov)
181661007b31SStefan Hajnoczi {
1817adad6496SKevin Wolf     return bdrv_co_do_writev(child, sector_num, nb_sectors, qiov, 0);
181861007b31SStefan Hajnoczi }
181961007b31SStefan Hajnoczi 
1820a03ef88fSKevin Wolf int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset,
1821f5a5ca79SManos Pitsidianakis                                        int bytes, BdrvRequestFlags flags)
182261007b31SStefan Hajnoczi {
1823f5a5ca79SManos Pitsidianakis     trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags);
182461007b31SStefan Hajnoczi 
1825a03ef88fSKevin Wolf     if (!(child->bs->open_flags & BDRV_O_UNMAP)) {
182661007b31SStefan Hajnoczi         flags &= ~BDRV_REQ_MAY_UNMAP;
182761007b31SStefan Hajnoczi     }
182861007b31SStefan Hajnoczi 
1829f5a5ca79SManos Pitsidianakis     return bdrv_co_pwritev(child, offset, bytes, NULL,
183061007b31SStefan Hajnoczi                            BDRV_REQ_ZERO_WRITE | flags);
183161007b31SStefan Hajnoczi }
183261007b31SStefan Hajnoczi 
18334085f5c7SJohn Snow /*
18344085f5c7SJohn Snow  * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not.
18354085f5c7SJohn Snow  */
18364085f5c7SJohn Snow int bdrv_flush_all(void)
18374085f5c7SJohn Snow {
18384085f5c7SJohn Snow     BdrvNextIterator it;
18394085f5c7SJohn Snow     BlockDriverState *bs = NULL;
18404085f5c7SJohn Snow     int result = 0;
18414085f5c7SJohn Snow 
18424085f5c7SJohn Snow     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
18434085f5c7SJohn Snow         AioContext *aio_context = bdrv_get_aio_context(bs);
18444085f5c7SJohn Snow         int ret;
18454085f5c7SJohn Snow 
18464085f5c7SJohn Snow         aio_context_acquire(aio_context);
18474085f5c7SJohn Snow         ret = bdrv_flush(bs);
18484085f5c7SJohn Snow         if (ret < 0 && !result) {
18494085f5c7SJohn Snow             result = ret;
18504085f5c7SJohn Snow         }
18514085f5c7SJohn Snow         aio_context_release(aio_context);
18524085f5c7SJohn Snow     }
18534085f5c7SJohn Snow 
18544085f5c7SJohn Snow     return result;
18554085f5c7SJohn Snow }
18564085f5c7SJohn Snow 
18574085f5c7SJohn Snow 
18584bcd936eSEric Blake typedef struct BdrvCoBlockStatusData {
185961007b31SStefan Hajnoczi     BlockDriverState *bs;
186061007b31SStefan Hajnoczi     BlockDriverState *base;
1861c9ce8c4dSEric Blake     bool want_zero;
18624bcd936eSEric Blake     int64_t offset;
18634bcd936eSEric Blake     int64_t bytes;
18644bcd936eSEric Blake     int64_t *pnum;
18654bcd936eSEric Blake     int64_t *map;
1866c9ce8c4dSEric Blake     BlockDriverState **file;
18674bcd936eSEric Blake     int ret;
186861007b31SStefan Hajnoczi     bool done;
18694bcd936eSEric Blake } BdrvCoBlockStatusData;
187061007b31SStefan Hajnoczi 
18713e4d0e72SEric Blake int coroutine_fn bdrv_co_block_status_from_file(BlockDriverState *bs,
18723e4d0e72SEric Blake                                                 bool want_zero,
18733e4d0e72SEric Blake                                                 int64_t offset,
18743e4d0e72SEric Blake                                                 int64_t bytes,
18753e4d0e72SEric Blake                                                 int64_t *pnum,
18763e4d0e72SEric Blake                                                 int64_t *map,
1877f7cc69b3SManos Pitsidianakis                                                 BlockDriverState **file)
1878f7cc69b3SManos Pitsidianakis {
1879f7cc69b3SManos Pitsidianakis     assert(bs->file && bs->file->bs);
18803e4d0e72SEric Blake     *pnum = bytes;
18813e4d0e72SEric Blake     *map = offset;
1882f7cc69b3SManos Pitsidianakis     *file = bs->file->bs;
18833e4d0e72SEric Blake     return BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID;
1884f7cc69b3SManos Pitsidianakis }
1885f7cc69b3SManos Pitsidianakis 
18863e4d0e72SEric Blake int coroutine_fn bdrv_co_block_status_from_backing(BlockDriverState *bs,
18873e4d0e72SEric Blake                                                    bool want_zero,
18883e4d0e72SEric Blake                                                    int64_t offset,
18893e4d0e72SEric Blake                                                    int64_t bytes,
18903e4d0e72SEric Blake                                                    int64_t *pnum,
18913e4d0e72SEric Blake                                                    int64_t *map,
1892f7cc69b3SManos Pitsidianakis                                                    BlockDriverState **file)
1893f7cc69b3SManos Pitsidianakis {
1894f7cc69b3SManos Pitsidianakis     assert(bs->backing && bs->backing->bs);
18953e4d0e72SEric Blake     *pnum = bytes;
18963e4d0e72SEric Blake     *map = offset;
1897f7cc69b3SManos Pitsidianakis     *file = bs->backing->bs;
18983e4d0e72SEric Blake     return BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID;
1899f7cc69b3SManos Pitsidianakis }
1900f7cc69b3SManos Pitsidianakis 
190161007b31SStefan Hajnoczi /*
190261007b31SStefan Hajnoczi  * Returns the allocation status of the specified sectors.
190361007b31SStefan Hajnoczi  * Drivers not implementing the functionality are assumed to not support
190461007b31SStefan Hajnoczi  * backing files, hence all their sectors are reported as allocated.
190561007b31SStefan Hajnoczi  *
190686a3d5c6SEric Blake  * If 'want_zero' is true, the caller is querying for mapping
190786a3d5c6SEric Blake  * purposes, with a focus on valid BDRV_BLOCK_OFFSET_VALID, _DATA, and
190886a3d5c6SEric Blake  * _ZERO where possible; otherwise, the result favors larger 'pnum',
190986a3d5c6SEric Blake  * with a focus on accurate BDRV_BLOCK_ALLOCATED.
1910c9ce8c4dSEric Blake  *
19112e8bc787SEric Blake  * If 'offset' is beyond the end of the disk image the return value is
1912fb0d8654SEric Blake  * BDRV_BLOCK_EOF and 'pnum' is set to 0.
191361007b31SStefan Hajnoczi  *
19142e8bc787SEric Blake  * 'bytes' is the max value 'pnum' should be set to.  If bytes goes
1915fb0d8654SEric Blake  * beyond the end of the disk image it will be clamped; if 'pnum' is set to
1916fb0d8654SEric Blake  * the end of the image, then the returned value will include BDRV_BLOCK_EOF.
191767a0fd2aSFam Zheng  *
19182e8bc787SEric Blake  * 'pnum' is set to the number of bytes (including and immediately
19192e8bc787SEric Blake  * following the specified offset) that are easily known to be in the
19202e8bc787SEric Blake  * same allocated/unallocated state.  Note that a second call starting
19212e8bc787SEric Blake  * at the original offset plus returned pnum may have the same status.
19222e8bc787SEric Blake  * The returned value is non-zero on success except at end-of-file.
19232e8bc787SEric Blake  *
19242e8bc787SEric Blake  * Returns negative errno on failure.  Otherwise, if the
19252e8bc787SEric Blake  * BDRV_BLOCK_OFFSET_VALID bit is set, 'map' and 'file' (if non-NULL) are
19262e8bc787SEric Blake  * set to the host mapping and BDS corresponding to the guest offset.
192761007b31SStefan Hajnoczi  */
19282e8bc787SEric Blake static int coroutine_fn bdrv_co_block_status(BlockDriverState *bs,
1929c9ce8c4dSEric Blake                                              bool want_zero,
19302e8bc787SEric Blake                                              int64_t offset, int64_t bytes,
19312e8bc787SEric Blake                                              int64_t *pnum, int64_t *map,
193267a0fd2aSFam Zheng                                              BlockDriverState **file)
193361007b31SStefan Hajnoczi {
19342e8bc787SEric Blake     int64_t total_size;
19352e8bc787SEric Blake     int64_t n; /* bytes */
1936efa6e2edSEric Blake     int ret;
19372e8bc787SEric Blake     int64_t local_map = 0;
1938298a1665SEric Blake     BlockDriverState *local_file = NULL;
1939efa6e2edSEric Blake     int64_t aligned_offset, aligned_bytes;
1940efa6e2edSEric Blake     uint32_t align;
194161007b31SStefan Hajnoczi 
1942298a1665SEric Blake     assert(pnum);
1943298a1665SEric Blake     *pnum = 0;
19442e8bc787SEric Blake     total_size = bdrv_getlength(bs);
19452e8bc787SEric Blake     if (total_size < 0) {
19462e8bc787SEric Blake         ret = total_size;
1947298a1665SEric Blake         goto early_out;
194861007b31SStefan Hajnoczi     }
194961007b31SStefan Hajnoczi 
19502e8bc787SEric Blake     if (offset >= total_size) {
1951298a1665SEric Blake         ret = BDRV_BLOCK_EOF;
1952298a1665SEric Blake         goto early_out;
195361007b31SStefan Hajnoczi     }
19542e8bc787SEric Blake     if (!bytes) {
1955298a1665SEric Blake         ret = 0;
1956298a1665SEric Blake         goto early_out;
19579cdcfd9fSEric Blake     }
195861007b31SStefan Hajnoczi 
19592e8bc787SEric Blake     n = total_size - offset;
19602e8bc787SEric Blake     if (n < bytes) {
19612e8bc787SEric Blake         bytes = n;
196261007b31SStefan Hajnoczi     }
196361007b31SStefan Hajnoczi 
1964d470ad42SMax Reitz     /* Must be non-NULL or bdrv_getlength() would have failed */
1965d470ad42SMax Reitz     assert(bs->drv);
1966*636cb512SEric Blake     if (!bs->drv->bdrv_co_block_status) {
19672e8bc787SEric Blake         *pnum = bytes;
196861007b31SStefan Hajnoczi         ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
19692e8bc787SEric Blake         if (offset + bytes == total_size) {
1970fb0d8654SEric Blake             ret |= BDRV_BLOCK_EOF;
1971fb0d8654SEric Blake         }
197261007b31SStefan Hajnoczi         if (bs->drv->protocol_name) {
19732e8bc787SEric Blake             ret |= BDRV_BLOCK_OFFSET_VALID;
19742e8bc787SEric Blake             local_map = offset;
1975298a1665SEric Blake             local_file = bs;
197661007b31SStefan Hajnoczi         }
1977298a1665SEric Blake         goto early_out;
197861007b31SStefan Hajnoczi     }
197961007b31SStefan Hajnoczi 
198099723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
1981efa6e2edSEric Blake 
1982efa6e2edSEric Blake     /* Round out to request_alignment boundaries */
198386a3d5c6SEric Blake     align = bs->bl.request_alignment;
1984efa6e2edSEric Blake     aligned_offset = QEMU_ALIGN_DOWN(offset, align);
1985efa6e2edSEric Blake     aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset;
1986efa6e2edSEric Blake 
198786a3d5c6SEric Blake     ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset,
198886a3d5c6SEric Blake                                         aligned_bytes, pnum, &local_map,
198986a3d5c6SEric Blake                                         &local_file);
199086a3d5c6SEric Blake     if (ret < 0) {
199186a3d5c6SEric Blake         *pnum = 0;
199286a3d5c6SEric Blake         goto out;
199386a3d5c6SEric Blake     }
1994efa6e2edSEric Blake 
1995efa6e2edSEric Blake     /*
1996*636cb512SEric Blake      * The driver's result must be a non-zero multiple of request_alignment.
1997efa6e2edSEric Blake      * Clamp pnum and adjust map to original request.
1998efa6e2edSEric Blake      */
1999*636cb512SEric Blake     assert(*pnum && QEMU_IS_ALIGNED(*pnum, align) &&
2000*636cb512SEric Blake            align > offset - aligned_offset);
2001efa6e2edSEric Blake     *pnum -= offset - aligned_offset;
2002efa6e2edSEric Blake     if (*pnum > bytes) {
2003efa6e2edSEric Blake         *pnum = bytes;
2004efa6e2edSEric Blake     }
2005efa6e2edSEric Blake     if (ret & BDRV_BLOCK_OFFSET_VALID) {
2006efa6e2edSEric Blake         local_map += offset - aligned_offset;
2007efa6e2edSEric Blake     }
200861007b31SStefan Hajnoczi 
200961007b31SStefan Hajnoczi     if (ret & BDRV_BLOCK_RAW) {
2010298a1665SEric Blake         assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file);
20112e8bc787SEric Blake         ret = bdrv_co_block_status(local_file, want_zero, local_map,
20122e8bc787SEric Blake                                    *pnum, pnum, &local_map, &local_file);
201399723548SPaolo Bonzini         goto out;
201461007b31SStefan Hajnoczi     }
201561007b31SStefan Hajnoczi 
201661007b31SStefan Hajnoczi     if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
201761007b31SStefan Hajnoczi         ret |= BDRV_BLOCK_ALLOCATED;
2018c9ce8c4dSEric Blake     } else if (want_zero) {
201961007b31SStefan Hajnoczi         if (bdrv_unallocated_blocks_are_zero(bs)) {
202061007b31SStefan Hajnoczi             ret |= BDRV_BLOCK_ZERO;
2021760e0063SKevin Wolf         } else if (bs->backing) {
2022760e0063SKevin Wolf             BlockDriverState *bs2 = bs->backing->bs;
20232e8bc787SEric Blake             int64_t size2 = bdrv_getlength(bs2);
2024c9ce8c4dSEric Blake 
20252e8bc787SEric Blake             if (size2 >= 0 && offset >= size2) {
202661007b31SStefan Hajnoczi                 ret |= BDRV_BLOCK_ZERO;
202761007b31SStefan Hajnoczi             }
202861007b31SStefan Hajnoczi         }
202961007b31SStefan Hajnoczi     }
203061007b31SStefan Hajnoczi 
2031c9ce8c4dSEric Blake     if (want_zero && local_file && local_file != bs &&
203261007b31SStefan Hajnoczi         (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
203361007b31SStefan Hajnoczi         (ret & BDRV_BLOCK_OFFSET_VALID)) {
20342e8bc787SEric Blake         int64_t file_pnum;
20352e8bc787SEric Blake         int ret2;
203661007b31SStefan Hajnoczi 
20372e8bc787SEric Blake         ret2 = bdrv_co_block_status(local_file, want_zero, local_map,
20382e8bc787SEric Blake                                     *pnum, &file_pnum, NULL, NULL);
203961007b31SStefan Hajnoczi         if (ret2 >= 0) {
204061007b31SStefan Hajnoczi             /* Ignore errors.  This is just providing extra information, it
204161007b31SStefan Hajnoczi              * is useful but not necessary.
204261007b31SStefan Hajnoczi              */
2043c61e684eSEric Blake             if (ret2 & BDRV_BLOCK_EOF &&
2044c61e684eSEric Blake                 (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) {
2045c61e684eSEric Blake                 /*
2046c61e684eSEric Blake                  * It is valid for the format block driver to read
2047c61e684eSEric Blake                  * beyond the end of the underlying file's current
2048c61e684eSEric Blake                  * size; such areas read as zero.
2049c61e684eSEric Blake                  */
205061007b31SStefan Hajnoczi                 ret |= BDRV_BLOCK_ZERO;
205161007b31SStefan Hajnoczi             } else {
205261007b31SStefan Hajnoczi                 /* Limit request to the range reported by the protocol driver */
205361007b31SStefan Hajnoczi                 *pnum = file_pnum;
205461007b31SStefan Hajnoczi                 ret |= (ret2 & BDRV_BLOCK_ZERO);
205561007b31SStefan Hajnoczi             }
205661007b31SStefan Hajnoczi         }
205761007b31SStefan Hajnoczi     }
205861007b31SStefan Hajnoczi 
205999723548SPaolo Bonzini out:
206099723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
20612e8bc787SEric Blake     if (ret >= 0 && offset + *pnum == total_size) {
2062fb0d8654SEric Blake         ret |= BDRV_BLOCK_EOF;
2063fb0d8654SEric Blake     }
2064298a1665SEric Blake early_out:
2065298a1665SEric Blake     if (file) {
2066298a1665SEric Blake         *file = local_file;
2067298a1665SEric Blake     }
20682e8bc787SEric Blake     if (map) {
20692e8bc787SEric Blake         *map = local_map;
20702e8bc787SEric Blake     }
207161007b31SStefan Hajnoczi     return ret;
207261007b31SStefan Hajnoczi }
207361007b31SStefan Hajnoczi 
20745b648c67SEric Blake static int coroutine_fn bdrv_co_block_status_above(BlockDriverState *bs,
2075ba3f0e25SFam Zheng                                                    BlockDriverState *base,
2076c9ce8c4dSEric Blake                                                    bool want_zero,
20775b648c67SEric Blake                                                    int64_t offset,
20785b648c67SEric Blake                                                    int64_t bytes,
20795b648c67SEric Blake                                                    int64_t *pnum,
20805b648c67SEric Blake                                                    int64_t *map,
208167a0fd2aSFam Zheng                                                    BlockDriverState **file)
2082ba3f0e25SFam Zheng {
2083ba3f0e25SFam Zheng     BlockDriverState *p;
20845b648c67SEric Blake     int ret = 0;
2085c61e684eSEric Blake     bool first = true;
2086ba3f0e25SFam Zheng 
2087ba3f0e25SFam Zheng     assert(bs != base);
2088760e0063SKevin Wolf     for (p = bs; p != base; p = backing_bs(p)) {
20895b648c67SEric Blake         ret = bdrv_co_block_status(p, want_zero, offset, bytes, pnum, map,
20905b648c67SEric Blake                                    file);
2091c61e684eSEric Blake         if (ret < 0) {
2092c61e684eSEric Blake             break;
2093c61e684eSEric Blake         }
2094c61e684eSEric Blake         if (ret & BDRV_BLOCK_ZERO && ret & BDRV_BLOCK_EOF && !first) {
2095c61e684eSEric Blake             /*
2096c61e684eSEric Blake              * Reading beyond the end of the file continues to read
2097c61e684eSEric Blake              * zeroes, but we can only widen the result to the
2098c61e684eSEric Blake              * unallocated length we learned from an earlier
2099c61e684eSEric Blake              * iteration.
2100c61e684eSEric Blake              */
21015b648c67SEric Blake             *pnum = bytes;
2102c61e684eSEric Blake         }
2103c61e684eSEric Blake         if (ret & (BDRV_BLOCK_ZERO | BDRV_BLOCK_DATA)) {
2104ba3f0e25SFam Zheng             break;
2105ba3f0e25SFam Zheng         }
21065b648c67SEric Blake         /* [offset, pnum] unallocated on this layer, which could be only
21075b648c67SEric Blake          * the first part of [offset, bytes].  */
21085b648c67SEric Blake         bytes = MIN(bytes, *pnum);
2109c61e684eSEric Blake         first = false;
2110ba3f0e25SFam Zheng     }
2111ba3f0e25SFam Zheng     return ret;
2112ba3f0e25SFam Zheng }
2113ba3f0e25SFam Zheng 
211431826642SEric Blake /* Coroutine wrapper for bdrv_block_status_above() */
21155b648c67SEric Blake static void coroutine_fn bdrv_block_status_above_co_entry(void *opaque)
211661007b31SStefan Hajnoczi {
21174bcd936eSEric Blake     BdrvCoBlockStatusData *data = opaque;
211861007b31SStefan Hajnoczi 
21195b648c67SEric Blake     data->ret = bdrv_co_block_status_above(data->bs, data->base,
2120c9ce8c4dSEric Blake                                            data->want_zero,
21215b648c67SEric Blake                                            data->offset, data->bytes,
21225b648c67SEric Blake                                            data->pnum, data->map, data->file);
212361007b31SStefan Hajnoczi     data->done = true;
212461007b31SStefan Hajnoczi }
212561007b31SStefan Hajnoczi 
212661007b31SStefan Hajnoczi /*
21275b648c67SEric Blake  * Synchronous wrapper around bdrv_co_block_status_above().
212861007b31SStefan Hajnoczi  *
21295b648c67SEric Blake  * See bdrv_co_block_status_above() for details.
213061007b31SStefan Hajnoczi  */
21317ddb99b9SEric Blake static int bdrv_common_block_status_above(BlockDriverState *bs,
2132ba3f0e25SFam Zheng                                           BlockDriverState *base,
21337ddb99b9SEric Blake                                           bool want_zero, int64_t offset,
21347ddb99b9SEric Blake                                           int64_t bytes, int64_t *pnum,
21357ddb99b9SEric Blake                                           int64_t *map,
213667a0fd2aSFam Zheng                                           BlockDriverState **file)
213761007b31SStefan Hajnoczi {
213861007b31SStefan Hajnoczi     Coroutine *co;
21394bcd936eSEric Blake     BdrvCoBlockStatusData data = {
214061007b31SStefan Hajnoczi         .bs = bs,
2141ba3f0e25SFam Zheng         .base = base,
2142c9ce8c4dSEric Blake         .want_zero = want_zero,
21437ddb99b9SEric Blake         .offset = offset,
21447ddb99b9SEric Blake         .bytes = bytes,
21457ddb99b9SEric Blake         .pnum = pnum,
21467ddb99b9SEric Blake         .map = map,
2147c9ce8c4dSEric Blake         .file = file,
214861007b31SStefan Hajnoczi         .done = false,
214961007b31SStefan Hajnoczi     };
215061007b31SStefan Hajnoczi 
215161007b31SStefan Hajnoczi     if (qemu_in_coroutine()) {
215261007b31SStefan Hajnoczi         /* Fast-path if already in coroutine context */
21535b648c67SEric Blake         bdrv_block_status_above_co_entry(&data);
215461007b31SStefan Hajnoczi     } else {
21555b648c67SEric Blake         co = qemu_coroutine_create(bdrv_block_status_above_co_entry, &data);
2156e92f0e19SFam Zheng         bdrv_coroutine_enter(bs, co);
215788b062c2SPaolo Bonzini         BDRV_POLL_WHILE(bs, !data.done);
215861007b31SStefan Hajnoczi     }
215961007b31SStefan Hajnoczi     return data.ret;
216061007b31SStefan Hajnoczi }
216161007b31SStefan Hajnoczi 
216231826642SEric Blake int bdrv_block_status_above(BlockDriverState *bs, BlockDriverState *base,
216331826642SEric Blake                             int64_t offset, int64_t bytes, int64_t *pnum,
216431826642SEric Blake                             int64_t *map, BlockDriverState **file)
2165c9ce8c4dSEric Blake {
216631826642SEric Blake     return bdrv_common_block_status_above(bs, base, true, offset, bytes,
216731826642SEric Blake                                           pnum, map, file);
2168c9ce8c4dSEric Blake }
2169c9ce8c4dSEric Blake 
2170237d78f8SEric Blake int bdrv_block_status(BlockDriverState *bs, int64_t offset, int64_t bytes,
2171237d78f8SEric Blake                       int64_t *pnum, int64_t *map, BlockDriverState **file)
2172ba3f0e25SFam Zheng {
217331826642SEric Blake     return bdrv_block_status_above(bs, backing_bs(bs),
217431826642SEric Blake                                    offset, bytes, pnum, map, file);
2175ba3f0e25SFam Zheng }
2176ba3f0e25SFam Zheng 
2177d6a644bbSEric Blake int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t offset,
2178d6a644bbSEric Blake                                    int64_t bytes, int64_t *pnum)
217961007b31SStefan Hajnoczi {
21807ddb99b9SEric Blake     int ret;
21817ddb99b9SEric Blake     int64_t dummy;
2182d6a644bbSEric Blake 
21837ddb99b9SEric Blake     ret = bdrv_common_block_status_above(bs, backing_bs(bs), false, offset,
21847ddb99b9SEric Blake                                          bytes, pnum ? pnum : &dummy, NULL,
2185298a1665SEric Blake                                          NULL);
218661007b31SStefan Hajnoczi     if (ret < 0) {
218761007b31SStefan Hajnoczi         return ret;
218861007b31SStefan Hajnoczi     }
218961007b31SStefan Hajnoczi     return !!(ret & BDRV_BLOCK_ALLOCATED);
219061007b31SStefan Hajnoczi }
219161007b31SStefan Hajnoczi 
219261007b31SStefan Hajnoczi /*
219361007b31SStefan Hajnoczi  * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
219461007b31SStefan Hajnoczi  *
219551b0a488SEric Blake  * Return true if (a prefix of) the given range is allocated in any image
219651b0a488SEric Blake  * between BASE and TOP (inclusive).  BASE can be NULL to check if the given
219751b0a488SEric Blake  * offset is allocated in any image of the chain.  Return false otherwise,
2198d6a644bbSEric Blake  * or negative errno on failure.
219961007b31SStefan Hajnoczi  *
220051b0a488SEric Blake  * 'pnum' is set to the number of bytes (including and immediately
220151b0a488SEric Blake  * following the specified offset) that are known to be in the same
220251b0a488SEric Blake  * allocated/unallocated state.  Note that a subsequent call starting
220351b0a488SEric Blake  * at 'offset + *pnum' may return the same allocation status (in other
220451b0a488SEric Blake  * words, the result is not necessarily the maximum possible range);
220551b0a488SEric Blake  * but 'pnum' will only be 0 when end of file is reached.
220661007b31SStefan Hajnoczi  *
220761007b31SStefan Hajnoczi  */
220861007b31SStefan Hajnoczi int bdrv_is_allocated_above(BlockDriverState *top,
220961007b31SStefan Hajnoczi                             BlockDriverState *base,
221051b0a488SEric Blake                             int64_t offset, int64_t bytes, int64_t *pnum)
221161007b31SStefan Hajnoczi {
221261007b31SStefan Hajnoczi     BlockDriverState *intermediate;
221351b0a488SEric Blake     int ret;
221451b0a488SEric Blake     int64_t n = bytes;
221561007b31SStefan Hajnoczi 
221661007b31SStefan Hajnoczi     intermediate = top;
221761007b31SStefan Hajnoczi     while (intermediate && intermediate != base) {
2218d6a644bbSEric Blake         int64_t pnum_inter;
2219c00716beSEric Blake         int64_t size_inter;
2220d6a644bbSEric Blake 
222151b0a488SEric Blake         ret = bdrv_is_allocated(intermediate, offset, bytes, &pnum_inter);
222261007b31SStefan Hajnoczi         if (ret < 0) {
222361007b31SStefan Hajnoczi             return ret;
2224d6a644bbSEric Blake         }
2225d6a644bbSEric Blake         if (ret) {
222651b0a488SEric Blake             *pnum = pnum_inter;
222761007b31SStefan Hajnoczi             return 1;
222861007b31SStefan Hajnoczi         }
222961007b31SStefan Hajnoczi 
223051b0a488SEric Blake         size_inter = bdrv_getlength(intermediate);
2231c00716beSEric Blake         if (size_inter < 0) {
2232c00716beSEric Blake             return size_inter;
2233c00716beSEric Blake         }
223451b0a488SEric Blake         if (n > pnum_inter &&
223551b0a488SEric Blake             (intermediate == top || offset + pnum_inter < size_inter)) {
223651b0a488SEric Blake             n = pnum_inter;
223761007b31SStefan Hajnoczi         }
223861007b31SStefan Hajnoczi 
2239760e0063SKevin Wolf         intermediate = backing_bs(intermediate);
224061007b31SStefan Hajnoczi     }
224161007b31SStefan Hajnoczi 
224261007b31SStefan Hajnoczi     *pnum = n;
224361007b31SStefan Hajnoczi     return 0;
224461007b31SStefan Hajnoczi }
224561007b31SStefan Hajnoczi 
22461a8ae822SKevin Wolf typedef struct BdrvVmstateCo {
22471a8ae822SKevin Wolf     BlockDriverState    *bs;
22481a8ae822SKevin Wolf     QEMUIOVector        *qiov;
22491a8ae822SKevin Wolf     int64_t             pos;
22501a8ae822SKevin Wolf     bool                is_read;
22511a8ae822SKevin Wolf     int                 ret;
22521a8ae822SKevin Wolf } BdrvVmstateCo;
22531a8ae822SKevin Wolf 
22541a8ae822SKevin Wolf static int coroutine_fn
22551a8ae822SKevin Wolf bdrv_co_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos,
22561a8ae822SKevin Wolf                    bool is_read)
22571a8ae822SKevin Wolf {
22581a8ae822SKevin Wolf     BlockDriver *drv = bs->drv;
2259dc88a467SStefan Hajnoczi     int ret = -ENOTSUP;
2260dc88a467SStefan Hajnoczi 
2261dc88a467SStefan Hajnoczi     bdrv_inc_in_flight(bs);
22621a8ae822SKevin Wolf 
22631a8ae822SKevin Wolf     if (!drv) {
2264dc88a467SStefan Hajnoczi         ret = -ENOMEDIUM;
22651a8ae822SKevin Wolf     } else if (drv->bdrv_load_vmstate) {
2266dc88a467SStefan Hajnoczi         if (is_read) {
2267dc88a467SStefan Hajnoczi             ret = drv->bdrv_load_vmstate(bs, qiov, pos);
2268dc88a467SStefan Hajnoczi         } else {
2269dc88a467SStefan Hajnoczi             ret = drv->bdrv_save_vmstate(bs, qiov, pos);
2270dc88a467SStefan Hajnoczi         }
22711a8ae822SKevin Wolf     } else if (bs->file) {
2272dc88a467SStefan Hajnoczi         ret = bdrv_co_rw_vmstate(bs->file->bs, qiov, pos, is_read);
22731a8ae822SKevin Wolf     }
22741a8ae822SKevin Wolf 
2275dc88a467SStefan Hajnoczi     bdrv_dec_in_flight(bs);
2276dc88a467SStefan Hajnoczi     return ret;
22771a8ae822SKevin Wolf }
22781a8ae822SKevin Wolf 
22791a8ae822SKevin Wolf static void coroutine_fn bdrv_co_rw_vmstate_entry(void *opaque)
22801a8ae822SKevin Wolf {
22811a8ae822SKevin Wolf     BdrvVmstateCo *co = opaque;
22821a8ae822SKevin Wolf     co->ret = bdrv_co_rw_vmstate(co->bs, co->qiov, co->pos, co->is_read);
22831a8ae822SKevin Wolf }
22841a8ae822SKevin Wolf 
22851a8ae822SKevin Wolf static inline int
22861a8ae822SKevin Wolf bdrv_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos,
22871a8ae822SKevin Wolf                 bool is_read)
22881a8ae822SKevin Wolf {
22891a8ae822SKevin Wolf     if (qemu_in_coroutine()) {
22901a8ae822SKevin Wolf         return bdrv_co_rw_vmstate(bs, qiov, pos, is_read);
22911a8ae822SKevin Wolf     } else {
22921a8ae822SKevin Wolf         BdrvVmstateCo data = {
22931a8ae822SKevin Wolf             .bs         = bs,
22941a8ae822SKevin Wolf             .qiov       = qiov,
22951a8ae822SKevin Wolf             .pos        = pos,
22961a8ae822SKevin Wolf             .is_read    = is_read,
22971a8ae822SKevin Wolf             .ret        = -EINPROGRESS,
22981a8ae822SKevin Wolf         };
22990b8b8753SPaolo Bonzini         Coroutine *co = qemu_coroutine_create(bdrv_co_rw_vmstate_entry, &data);
23001a8ae822SKevin Wolf 
2301e92f0e19SFam Zheng         bdrv_coroutine_enter(bs, co);
2302ea17c9d2SStefan Hajnoczi         BDRV_POLL_WHILE(bs, data.ret == -EINPROGRESS);
23031a8ae822SKevin Wolf         return data.ret;
23041a8ae822SKevin Wolf     }
23051a8ae822SKevin Wolf }
23061a8ae822SKevin Wolf 
230761007b31SStefan Hajnoczi int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
230861007b31SStefan Hajnoczi                       int64_t pos, int size)
230961007b31SStefan Hajnoczi {
231061007b31SStefan Hajnoczi     QEMUIOVector qiov;
231161007b31SStefan Hajnoczi     struct iovec iov = {
231261007b31SStefan Hajnoczi         .iov_base   = (void *) buf,
231361007b31SStefan Hajnoczi         .iov_len    = size,
231461007b31SStefan Hajnoczi     };
2315b433d942SKevin Wolf     int ret;
231661007b31SStefan Hajnoczi 
231761007b31SStefan Hajnoczi     qemu_iovec_init_external(&qiov, &iov, 1);
2318b433d942SKevin Wolf 
2319b433d942SKevin Wolf     ret = bdrv_writev_vmstate(bs, &qiov, pos);
2320b433d942SKevin Wolf     if (ret < 0) {
2321b433d942SKevin Wolf         return ret;
2322b433d942SKevin Wolf     }
2323b433d942SKevin Wolf 
2324b433d942SKevin Wolf     return size;
232561007b31SStefan Hajnoczi }
232661007b31SStefan Hajnoczi 
232761007b31SStefan Hajnoczi int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
232861007b31SStefan Hajnoczi {
23291a8ae822SKevin Wolf     return bdrv_rw_vmstate(bs, qiov, pos, false);
233061007b31SStefan Hajnoczi }
233161007b31SStefan Hajnoczi 
233261007b31SStefan Hajnoczi int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
233361007b31SStefan Hajnoczi                       int64_t pos, int size)
233461007b31SStefan Hajnoczi {
23355ddda0b8SKevin Wolf     QEMUIOVector qiov;
23365ddda0b8SKevin Wolf     struct iovec iov = {
23375ddda0b8SKevin Wolf         .iov_base   = buf,
23385ddda0b8SKevin Wolf         .iov_len    = size,
23395ddda0b8SKevin Wolf     };
2340b433d942SKevin Wolf     int ret;
23415ddda0b8SKevin Wolf 
23425ddda0b8SKevin Wolf     qemu_iovec_init_external(&qiov, &iov, 1);
2343b433d942SKevin Wolf     ret = bdrv_readv_vmstate(bs, &qiov, pos);
2344b433d942SKevin Wolf     if (ret < 0) {
2345b433d942SKevin Wolf         return ret;
2346b433d942SKevin Wolf     }
2347b433d942SKevin Wolf 
2348b433d942SKevin Wolf     return size;
23495ddda0b8SKevin Wolf }
23505ddda0b8SKevin Wolf 
23515ddda0b8SKevin Wolf int bdrv_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
23525ddda0b8SKevin Wolf {
23531a8ae822SKevin Wolf     return bdrv_rw_vmstate(bs, qiov, pos, true);
235461007b31SStefan Hajnoczi }
235561007b31SStefan Hajnoczi 
235661007b31SStefan Hajnoczi /**************************************************************/
235761007b31SStefan Hajnoczi /* async I/Os */
235861007b31SStefan Hajnoczi 
235961007b31SStefan Hajnoczi void bdrv_aio_cancel(BlockAIOCB *acb)
236061007b31SStefan Hajnoczi {
236161007b31SStefan Hajnoczi     qemu_aio_ref(acb);
236261007b31SStefan Hajnoczi     bdrv_aio_cancel_async(acb);
236361007b31SStefan Hajnoczi     while (acb->refcnt > 1) {
236461007b31SStefan Hajnoczi         if (acb->aiocb_info->get_aio_context) {
236561007b31SStefan Hajnoczi             aio_poll(acb->aiocb_info->get_aio_context(acb), true);
236661007b31SStefan Hajnoczi         } else if (acb->bs) {
23672f47da5fSPaolo Bonzini             /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so
23682f47da5fSPaolo Bonzini              * assert that we're not using an I/O thread.  Thread-safe
23692f47da5fSPaolo Bonzini              * code should use bdrv_aio_cancel_async exclusively.
23702f47da5fSPaolo Bonzini              */
23712f47da5fSPaolo Bonzini             assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context());
237261007b31SStefan Hajnoczi             aio_poll(bdrv_get_aio_context(acb->bs), true);
237361007b31SStefan Hajnoczi         } else {
237461007b31SStefan Hajnoczi             abort();
237561007b31SStefan Hajnoczi         }
237661007b31SStefan Hajnoczi     }
237761007b31SStefan Hajnoczi     qemu_aio_unref(acb);
237861007b31SStefan Hajnoczi }
237961007b31SStefan Hajnoczi 
238061007b31SStefan Hajnoczi /* Async version of aio cancel. The caller is not blocked if the acb implements
238161007b31SStefan Hajnoczi  * cancel_async, otherwise we do nothing and let the request normally complete.
238261007b31SStefan Hajnoczi  * In either case the completion callback must be called. */
238361007b31SStefan Hajnoczi void bdrv_aio_cancel_async(BlockAIOCB *acb)
238461007b31SStefan Hajnoczi {
238561007b31SStefan Hajnoczi     if (acb->aiocb_info->cancel_async) {
238661007b31SStefan Hajnoczi         acb->aiocb_info->cancel_async(acb);
238761007b31SStefan Hajnoczi     }
238861007b31SStefan Hajnoczi }
238961007b31SStefan Hajnoczi 
239061007b31SStefan Hajnoczi /**************************************************************/
239161007b31SStefan Hajnoczi /* Coroutine block device emulation */
239261007b31SStefan Hajnoczi 
2393e293b7a3SKevin Wolf typedef struct FlushCo {
2394e293b7a3SKevin Wolf     BlockDriverState *bs;
2395e293b7a3SKevin Wolf     int ret;
2396e293b7a3SKevin Wolf } FlushCo;
2397e293b7a3SKevin Wolf 
2398e293b7a3SKevin Wolf 
239961007b31SStefan Hajnoczi static void coroutine_fn bdrv_flush_co_entry(void *opaque)
240061007b31SStefan Hajnoczi {
2401e293b7a3SKevin Wolf     FlushCo *rwco = opaque;
240261007b31SStefan Hajnoczi 
240361007b31SStefan Hajnoczi     rwco->ret = bdrv_co_flush(rwco->bs);
240461007b31SStefan Hajnoczi }
240561007b31SStefan Hajnoczi 
240661007b31SStefan Hajnoczi int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
240761007b31SStefan Hajnoczi {
240849ca6259SFam Zheng     int current_gen;
240949ca6259SFam Zheng     int ret = 0;
241061007b31SStefan Hajnoczi 
241199723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
2412c32b82afSPavel Dovgalyuk 
2413e914404eSFam Zheng     if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
241449ca6259SFam Zheng         bdrv_is_sg(bs)) {
241549ca6259SFam Zheng         goto early_exit;
241649ca6259SFam Zheng     }
241749ca6259SFam Zheng 
24183783fa3dSPaolo Bonzini     qemu_co_mutex_lock(&bs->reqs_lock);
241947fec599SPaolo Bonzini     current_gen = atomic_read(&bs->write_gen);
24203ff2f67aSEvgeny Yakovlev 
24213ff2f67aSEvgeny Yakovlev     /* Wait until any previous flushes are completed */
242299723548SPaolo Bonzini     while (bs->active_flush_req) {
24233783fa3dSPaolo Bonzini         qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock);
24243ff2f67aSEvgeny Yakovlev     }
24253ff2f67aSEvgeny Yakovlev 
24263783fa3dSPaolo Bonzini     /* Flushes reach this point in nondecreasing current_gen order.  */
242799723548SPaolo Bonzini     bs->active_flush_req = true;
24283783fa3dSPaolo Bonzini     qemu_co_mutex_unlock(&bs->reqs_lock);
24293ff2f67aSEvgeny Yakovlev 
2430c32b82afSPavel Dovgalyuk     /* Write back all layers by calling one driver function */
2431c32b82afSPavel Dovgalyuk     if (bs->drv->bdrv_co_flush) {
2432c32b82afSPavel Dovgalyuk         ret = bs->drv->bdrv_co_flush(bs);
2433c32b82afSPavel Dovgalyuk         goto out;
2434c32b82afSPavel Dovgalyuk     }
2435c32b82afSPavel Dovgalyuk 
243661007b31SStefan Hajnoczi     /* Write back cached data to the OS even with cache=unsafe */
243761007b31SStefan Hajnoczi     BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS);
243861007b31SStefan Hajnoczi     if (bs->drv->bdrv_co_flush_to_os) {
243961007b31SStefan Hajnoczi         ret = bs->drv->bdrv_co_flush_to_os(bs);
244061007b31SStefan Hajnoczi         if (ret < 0) {
2441cdb5e315SFam Zheng             goto out;
244261007b31SStefan Hajnoczi         }
244361007b31SStefan Hajnoczi     }
244461007b31SStefan Hajnoczi 
244561007b31SStefan Hajnoczi     /* But don't actually force it to the disk with cache=unsafe */
244661007b31SStefan Hajnoczi     if (bs->open_flags & BDRV_O_NO_FLUSH) {
244761007b31SStefan Hajnoczi         goto flush_parent;
244861007b31SStefan Hajnoczi     }
244961007b31SStefan Hajnoczi 
24503ff2f67aSEvgeny Yakovlev     /* Check if we really need to flush anything */
24513ff2f67aSEvgeny Yakovlev     if (bs->flushed_gen == current_gen) {
24523ff2f67aSEvgeny Yakovlev         goto flush_parent;
24533ff2f67aSEvgeny Yakovlev     }
24543ff2f67aSEvgeny Yakovlev 
245561007b31SStefan Hajnoczi     BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK);
2456d470ad42SMax Reitz     if (!bs->drv) {
2457d470ad42SMax Reitz         /* bs->drv->bdrv_co_flush() might have ejected the BDS
2458d470ad42SMax Reitz          * (even in case of apparent success) */
2459d470ad42SMax Reitz         ret = -ENOMEDIUM;
2460d470ad42SMax Reitz         goto out;
2461d470ad42SMax Reitz     }
246261007b31SStefan Hajnoczi     if (bs->drv->bdrv_co_flush_to_disk) {
246361007b31SStefan Hajnoczi         ret = bs->drv->bdrv_co_flush_to_disk(bs);
246461007b31SStefan Hajnoczi     } else if (bs->drv->bdrv_aio_flush) {
246561007b31SStefan Hajnoczi         BlockAIOCB *acb;
246661007b31SStefan Hajnoczi         CoroutineIOCompletion co = {
246761007b31SStefan Hajnoczi             .coroutine = qemu_coroutine_self(),
246861007b31SStefan Hajnoczi         };
246961007b31SStefan Hajnoczi 
247061007b31SStefan Hajnoczi         acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
247161007b31SStefan Hajnoczi         if (acb == NULL) {
247261007b31SStefan Hajnoczi             ret = -EIO;
247361007b31SStefan Hajnoczi         } else {
247461007b31SStefan Hajnoczi             qemu_coroutine_yield();
247561007b31SStefan Hajnoczi             ret = co.ret;
247661007b31SStefan Hajnoczi         }
247761007b31SStefan Hajnoczi     } else {
247861007b31SStefan Hajnoczi         /*
247961007b31SStefan Hajnoczi          * Some block drivers always operate in either writethrough or unsafe
248061007b31SStefan Hajnoczi          * mode and don't support bdrv_flush therefore. Usually qemu doesn't
248161007b31SStefan Hajnoczi          * know how the server works (because the behaviour is hardcoded or
248261007b31SStefan Hajnoczi          * depends on server-side configuration), so we can't ensure that
248361007b31SStefan Hajnoczi          * everything is safe on disk. Returning an error doesn't work because
248461007b31SStefan Hajnoczi          * that would break guests even if the server operates in writethrough
248561007b31SStefan Hajnoczi          * mode.
248661007b31SStefan Hajnoczi          *
248761007b31SStefan Hajnoczi          * Let's hope the user knows what he's doing.
248861007b31SStefan Hajnoczi          */
248961007b31SStefan Hajnoczi         ret = 0;
249061007b31SStefan Hajnoczi     }
24913ff2f67aSEvgeny Yakovlev 
249261007b31SStefan Hajnoczi     if (ret < 0) {
2493cdb5e315SFam Zheng         goto out;
249461007b31SStefan Hajnoczi     }
249561007b31SStefan Hajnoczi 
249661007b31SStefan Hajnoczi     /* Now flush the underlying protocol.  It will also have BDRV_O_NO_FLUSH
249761007b31SStefan Hajnoczi      * in the case of cache=unsafe, so there are no useless flushes.
249861007b31SStefan Hajnoczi      */
249961007b31SStefan Hajnoczi flush_parent:
2500cdb5e315SFam Zheng     ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0;
2501cdb5e315SFam Zheng out:
25023ff2f67aSEvgeny Yakovlev     /* Notify any pending flushes that we have completed */
2503e6af1e08SKevin Wolf     if (ret == 0) {
25043ff2f67aSEvgeny Yakovlev         bs->flushed_gen = current_gen;
2505e6af1e08SKevin Wolf     }
25063783fa3dSPaolo Bonzini 
25073783fa3dSPaolo Bonzini     qemu_co_mutex_lock(&bs->reqs_lock);
250899723548SPaolo Bonzini     bs->active_flush_req = false;
2509156af3acSDenis V. Lunev     /* Return value is ignored - it's ok if wait queue is empty */
2510156af3acSDenis V. Lunev     qemu_co_queue_next(&bs->flush_queue);
25113783fa3dSPaolo Bonzini     qemu_co_mutex_unlock(&bs->reqs_lock);
25123ff2f67aSEvgeny Yakovlev 
251349ca6259SFam Zheng early_exit:
251499723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
2515cdb5e315SFam Zheng     return ret;
251661007b31SStefan Hajnoczi }
251761007b31SStefan Hajnoczi 
251861007b31SStefan Hajnoczi int bdrv_flush(BlockDriverState *bs)
251961007b31SStefan Hajnoczi {
252061007b31SStefan Hajnoczi     Coroutine *co;
2521e293b7a3SKevin Wolf     FlushCo flush_co = {
252261007b31SStefan Hajnoczi         .bs = bs,
252361007b31SStefan Hajnoczi         .ret = NOT_DONE,
252461007b31SStefan Hajnoczi     };
252561007b31SStefan Hajnoczi 
252661007b31SStefan Hajnoczi     if (qemu_in_coroutine()) {
252761007b31SStefan Hajnoczi         /* Fast-path if already in coroutine context */
2528e293b7a3SKevin Wolf         bdrv_flush_co_entry(&flush_co);
252961007b31SStefan Hajnoczi     } else {
25300b8b8753SPaolo Bonzini         co = qemu_coroutine_create(bdrv_flush_co_entry, &flush_co);
2531e92f0e19SFam Zheng         bdrv_coroutine_enter(bs, co);
253288b062c2SPaolo Bonzini         BDRV_POLL_WHILE(bs, flush_co.ret == NOT_DONE);
253361007b31SStefan Hajnoczi     }
253461007b31SStefan Hajnoczi 
2535e293b7a3SKevin Wolf     return flush_co.ret;
253661007b31SStefan Hajnoczi }
253761007b31SStefan Hajnoczi 
253861007b31SStefan Hajnoczi typedef struct DiscardCo {
253961007b31SStefan Hajnoczi     BlockDriverState *bs;
25400c51a893SEric Blake     int64_t offset;
2541f5a5ca79SManos Pitsidianakis     int bytes;
254261007b31SStefan Hajnoczi     int ret;
254361007b31SStefan Hajnoczi } DiscardCo;
25440c51a893SEric Blake static void coroutine_fn bdrv_pdiscard_co_entry(void *opaque)
254561007b31SStefan Hajnoczi {
254661007b31SStefan Hajnoczi     DiscardCo *rwco = opaque;
254761007b31SStefan Hajnoczi 
2548f5a5ca79SManos Pitsidianakis     rwco->ret = bdrv_co_pdiscard(rwco->bs, rwco->offset, rwco->bytes);
254961007b31SStefan Hajnoczi }
255061007b31SStefan Hajnoczi 
25519f1963b3SEric Blake int coroutine_fn bdrv_co_pdiscard(BlockDriverState *bs, int64_t offset,
2552f5a5ca79SManos Pitsidianakis                                   int bytes)
255361007b31SStefan Hajnoczi {
2554b1066c87SFam Zheng     BdrvTrackedRequest req;
25559f1963b3SEric Blake     int max_pdiscard, ret;
25563482b9bcSEric Blake     int head, tail, align;
255761007b31SStefan Hajnoczi 
255861007b31SStefan Hajnoczi     if (!bs->drv) {
255961007b31SStefan Hajnoczi         return -ENOMEDIUM;
256061007b31SStefan Hajnoczi     }
256161007b31SStefan Hajnoczi 
2562d6883bc9SVladimir Sementsov-Ogievskiy     if (bdrv_has_readonly_bitmaps(bs)) {
2563d6883bc9SVladimir Sementsov-Ogievskiy         return -EPERM;
2564d6883bc9SVladimir Sementsov-Ogievskiy     }
2565d6883bc9SVladimir Sementsov-Ogievskiy 
2566f5a5ca79SManos Pitsidianakis     ret = bdrv_check_byte_request(bs, offset, bytes);
256761007b31SStefan Hajnoczi     if (ret < 0) {
256861007b31SStefan Hajnoczi         return ret;
256961007b31SStefan Hajnoczi     } else if (bs->read_only) {
2570eaf5fe2dSPaolo Bonzini         return -EPERM;
257161007b31SStefan Hajnoczi     }
257204c01a5cSKevin Wolf     assert(!(bs->open_flags & BDRV_O_INACTIVE));
257361007b31SStefan Hajnoczi 
257461007b31SStefan Hajnoczi     /* Do nothing if disabled.  */
257561007b31SStefan Hajnoczi     if (!(bs->open_flags & BDRV_O_UNMAP)) {
257661007b31SStefan Hajnoczi         return 0;
257761007b31SStefan Hajnoczi     }
257861007b31SStefan Hajnoczi 
257902aefe43SEric Blake     if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) {
258061007b31SStefan Hajnoczi         return 0;
258161007b31SStefan Hajnoczi     }
258261007b31SStefan Hajnoczi 
25833482b9bcSEric Blake     /* Discard is advisory, but some devices track and coalesce
25843482b9bcSEric Blake      * unaligned requests, so we must pass everything down rather than
25853482b9bcSEric Blake      * round here.  Still, most devices will just silently ignore
25863482b9bcSEric Blake      * unaligned requests (by returning -ENOTSUP), so we must fragment
25873482b9bcSEric Blake      * the request accordingly.  */
258802aefe43SEric Blake     align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment);
2589b8d0a980SEric Blake     assert(align % bs->bl.request_alignment == 0);
2590b8d0a980SEric Blake     head = offset % align;
2591f5a5ca79SManos Pitsidianakis     tail = (offset + bytes) % align;
25929f1963b3SEric Blake 
259399723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
2594f5a5ca79SManos Pitsidianakis     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD);
259550824995SFam Zheng 
2596ec050f77SDenis V. Lunev     ret = notifier_with_return_list_notify(&bs->before_write_notifiers, &req);
2597ec050f77SDenis V. Lunev     if (ret < 0) {
2598ec050f77SDenis V. Lunev         goto out;
2599ec050f77SDenis V. Lunev     }
2600ec050f77SDenis V. Lunev 
26019f1963b3SEric Blake     max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT_MAX),
26029f1963b3SEric Blake                                    align);
26033482b9bcSEric Blake     assert(max_pdiscard >= bs->bl.request_alignment);
26049f1963b3SEric Blake 
2605f5a5ca79SManos Pitsidianakis     while (bytes > 0) {
2606f5a5ca79SManos Pitsidianakis         int num = bytes;
26073482b9bcSEric Blake 
26083482b9bcSEric Blake         if (head) {
26093482b9bcSEric Blake             /* Make small requests to get to alignment boundaries. */
2610f5a5ca79SManos Pitsidianakis             num = MIN(bytes, align - head);
26113482b9bcSEric Blake             if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) {
26123482b9bcSEric Blake                 num %= bs->bl.request_alignment;
26133482b9bcSEric Blake             }
26143482b9bcSEric Blake             head = (head + num) % align;
26153482b9bcSEric Blake             assert(num < max_pdiscard);
26163482b9bcSEric Blake         } else if (tail) {
26173482b9bcSEric Blake             if (num > align) {
26183482b9bcSEric Blake                 /* Shorten the request to the last aligned cluster.  */
26193482b9bcSEric Blake                 num -= tail;
26203482b9bcSEric Blake             } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) &&
26213482b9bcSEric Blake                        tail > bs->bl.request_alignment) {
26223482b9bcSEric Blake                 tail %= bs->bl.request_alignment;
26233482b9bcSEric Blake                 num -= tail;
26243482b9bcSEric Blake             }
26253482b9bcSEric Blake         }
26263482b9bcSEric Blake         /* limit request size */
26273482b9bcSEric Blake         if (num > max_pdiscard) {
26283482b9bcSEric Blake             num = max_pdiscard;
26293482b9bcSEric Blake         }
263061007b31SStefan Hajnoczi 
2631d470ad42SMax Reitz         if (!bs->drv) {
2632d470ad42SMax Reitz             ret = -ENOMEDIUM;
2633d470ad42SMax Reitz             goto out;
2634d470ad42SMax Reitz         }
263547a5486dSEric Blake         if (bs->drv->bdrv_co_pdiscard) {
263647a5486dSEric Blake             ret = bs->drv->bdrv_co_pdiscard(bs, offset, num);
263761007b31SStefan Hajnoczi         } else {
263861007b31SStefan Hajnoczi             BlockAIOCB *acb;
263961007b31SStefan Hajnoczi             CoroutineIOCompletion co = {
264061007b31SStefan Hajnoczi                 .coroutine = qemu_coroutine_self(),
264161007b31SStefan Hajnoczi             };
264261007b31SStefan Hajnoczi 
26434da444a0SEric Blake             acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num,
264461007b31SStefan Hajnoczi                                              bdrv_co_io_em_complete, &co);
264561007b31SStefan Hajnoczi             if (acb == NULL) {
2646b1066c87SFam Zheng                 ret = -EIO;
2647b1066c87SFam Zheng                 goto out;
264861007b31SStefan Hajnoczi             } else {
264961007b31SStefan Hajnoczi                 qemu_coroutine_yield();
265061007b31SStefan Hajnoczi                 ret = co.ret;
265161007b31SStefan Hajnoczi             }
265261007b31SStefan Hajnoczi         }
265361007b31SStefan Hajnoczi         if (ret && ret != -ENOTSUP) {
2654b1066c87SFam Zheng             goto out;
265561007b31SStefan Hajnoczi         }
265661007b31SStefan Hajnoczi 
26579f1963b3SEric Blake         offset += num;
2658f5a5ca79SManos Pitsidianakis         bytes -= num;
265961007b31SStefan Hajnoczi     }
2660b1066c87SFam Zheng     ret = 0;
2661b1066c87SFam Zheng out:
266247fec599SPaolo Bonzini     atomic_inc(&bs->write_gen);
26630fdf1a4fSEric Blake     bdrv_set_dirty(bs, req.offset, req.bytes);
2664b1066c87SFam Zheng     tracked_request_end(&req);
266599723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
2666b1066c87SFam Zheng     return ret;
266761007b31SStefan Hajnoczi }
266861007b31SStefan Hajnoczi 
2669f5a5ca79SManos Pitsidianakis int bdrv_pdiscard(BlockDriverState *bs, int64_t offset, int bytes)
267061007b31SStefan Hajnoczi {
267161007b31SStefan Hajnoczi     Coroutine *co;
267261007b31SStefan Hajnoczi     DiscardCo rwco = {
267361007b31SStefan Hajnoczi         .bs = bs,
26740c51a893SEric Blake         .offset = offset,
2675f5a5ca79SManos Pitsidianakis         .bytes = bytes,
267661007b31SStefan Hajnoczi         .ret = NOT_DONE,
267761007b31SStefan Hajnoczi     };
267861007b31SStefan Hajnoczi 
267961007b31SStefan Hajnoczi     if (qemu_in_coroutine()) {
268061007b31SStefan Hajnoczi         /* Fast-path if already in coroutine context */
26810c51a893SEric Blake         bdrv_pdiscard_co_entry(&rwco);
268261007b31SStefan Hajnoczi     } else {
26830c51a893SEric Blake         co = qemu_coroutine_create(bdrv_pdiscard_co_entry, &rwco);
2684e92f0e19SFam Zheng         bdrv_coroutine_enter(bs, co);
268588b062c2SPaolo Bonzini         BDRV_POLL_WHILE(bs, rwco.ret == NOT_DONE);
268661007b31SStefan Hajnoczi     }
268761007b31SStefan Hajnoczi 
268861007b31SStefan Hajnoczi     return rwco.ret;
268961007b31SStefan Hajnoczi }
269061007b31SStefan Hajnoczi 
269148af776aSKevin Wolf int bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf)
269261007b31SStefan Hajnoczi {
269361007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
26945c5ae76aSFam Zheng     CoroutineIOCompletion co = {
26955c5ae76aSFam Zheng         .coroutine = qemu_coroutine_self(),
26965c5ae76aSFam Zheng     };
26975c5ae76aSFam Zheng     BlockAIOCB *acb;
269861007b31SStefan Hajnoczi 
269999723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
270016a389dcSKevin Wolf     if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) {
27015c5ae76aSFam Zheng         co.ret = -ENOTSUP;
27025c5ae76aSFam Zheng         goto out;
27035c5ae76aSFam Zheng     }
27045c5ae76aSFam Zheng 
270516a389dcSKevin Wolf     if (drv->bdrv_co_ioctl) {
270616a389dcSKevin Wolf         co.ret = drv->bdrv_co_ioctl(bs, req, buf);
270716a389dcSKevin Wolf     } else {
27085c5ae76aSFam Zheng         acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
27095c5ae76aSFam Zheng         if (!acb) {
2710c8a9fd80SFam Zheng             co.ret = -ENOTSUP;
2711c8a9fd80SFam Zheng             goto out;
27125c5ae76aSFam Zheng         }
27135c5ae76aSFam Zheng         qemu_coroutine_yield();
271416a389dcSKevin Wolf     }
27155c5ae76aSFam Zheng out:
271699723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
27175c5ae76aSFam Zheng     return co.ret;
27185c5ae76aSFam Zheng }
27195c5ae76aSFam Zheng 
272061007b31SStefan Hajnoczi void *qemu_blockalign(BlockDriverState *bs, size_t size)
272161007b31SStefan Hajnoczi {
272261007b31SStefan Hajnoczi     return qemu_memalign(bdrv_opt_mem_align(bs), size);
272361007b31SStefan Hajnoczi }
272461007b31SStefan Hajnoczi 
272561007b31SStefan Hajnoczi void *qemu_blockalign0(BlockDriverState *bs, size_t size)
272661007b31SStefan Hajnoczi {
272761007b31SStefan Hajnoczi     return memset(qemu_blockalign(bs, size), 0, size);
272861007b31SStefan Hajnoczi }
272961007b31SStefan Hajnoczi 
273061007b31SStefan Hajnoczi void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
273161007b31SStefan Hajnoczi {
273261007b31SStefan Hajnoczi     size_t align = bdrv_opt_mem_align(bs);
273361007b31SStefan Hajnoczi 
273461007b31SStefan Hajnoczi     /* Ensure that NULL is never returned on success */
273561007b31SStefan Hajnoczi     assert(align > 0);
273661007b31SStefan Hajnoczi     if (size == 0) {
273761007b31SStefan Hajnoczi         size = align;
273861007b31SStefan Hajnoczi     }
273961007b31SStefan Hajnoczi 
274061007b31SStefan Hajnoczi     return qemu_try_memalign(align, size);
274161007b31SStefan Hajnoczi }
274261007b31SStefan Hajnoczi 
274361007b31SStefan Hajnoczi void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
274461007b31SStefan Hajnoczi {
274561007b31SStefan Hajnoczi     void *mem = qemu_try_blockalign(bs, size);
274661007b31SStefan Hajnoczi 
274761007b31SStefan Hajnoczi     if (mem) {
274861007b31SStefan Hajnoczi         memset(mem, 0, size);
274961007b31SStefan Hajnoczi     }
275061007b31SStefan Hajnoczi 
275161007b31SStefan Hajnoczi     return mem;
275261007b31SStefan Hajnoczi }
275361007b31SStefan Hajnoczi 
275461007b31SStefan Hajnoczi /*
275561007b31SStefan Hajnoczi  * Check if all memory in this vector is sector aligned.
275661007b31SStefan Hajnoczi  */
275761007b31SStefan Hajnoczi bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
275861007b31SStefan Hajnoczi {
275961007b31SStefan Hajnoczi     int i;
27604196d2f0SDenis V. Lunev     size_t alignment = bdrv_min_mem_align(bs);
276161007b31SStefan Hajnoczi 
276261007b31SStefan Hajnoczi     for (i = 0; i < qiov->niov; i++) {
276361007b31SStefan Hajnoczi         if ((uintptr_t) qiov->iov[i].iov_base % alignment) {
276461007b31SStefan Hajnoczi             return false;
276561007b31SStefan Hajnoczi         }
276661007b31SStefan Hajnoczi         if (qiov->iov[i].iov_len % alignment) {
276761007b31SStefan Hajnoczi             return false;
276861007b31SStefan Hajnoczi         }
276961007b31SStefan Hajnoczi     }
277061007b31SStefan Hajnoczi 
277161007b31SStefan Hajnoczi     return true;
277261007b31SStefan Hajnoczi }
277361007b31SStefan Hajnoczi 
277461007b31SStefan Hajnoczi void bdrv_add_before_write_notifier(BlockDriverState *bs,
277561007b31SStefan Hajnoczi                                     NotifierWithReturn *notifier)
277661007b31SStefan Hajnoczi {
277761007b31SStefan Hajnoczi     notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
277861007b31SStefan Hajnoczi }
277961007b31SStefan Hajnoczi 
278061007b31SStefan Hajnoczi void bdrv_io_plug(BlockDriverState *bs)
278161007b31SStefan Hajnoczi {
27826b98bd64SPaolo Bonzini     BdrvChild *child;
27836b98bd64SPaolo Bonzini 
27846b98bd64SPaolo Bonzini     QLIST_FOREACH(child, &bs->children, next) {
27856b98bd64SPaolo Bonzini         bdrv_io_plug(child->bs);
27866b98bd64SPaolo Bonzini     }
27876b98bd64SPaolo Bonzini 
2788850d54a2SPaolo Bonzini     if (atomic_fetch_inc(&bs->io_plugged) == 0) {
278961007b31SStefan Hajnoczi         BlockDriver *drv = bs->drv;
279061007b31SStefan Hajnoczi         if (drv && drv->bdrv_io_plug) {
279161007b31SStefan Hajnoczi             drv->bdrv_io_plug(bs);
27926b98bd64SPaolo Bonzini         }
279361007b31SStefan Hajnoczi     }
279461007b31SStefan Hajnoczi }
279561007b31SStefan Hajnoczi 
279661007b31SStefan Hajnoczi void bdrv_io_unplug(BlockDriverState *bs)
279761007b31SStefan Hajnoczi {
27986b98bd64SPaolo Bonzini     BdrvChild *child;
27996b98bd64SPaolo Bonzini 
28006b98bd64SPaolo Bonzini     assert(bs->io_plugged);
2801850d54a2SPaolo Bonzini     if (atomic_fetch_dec(&bs->io_plugged) == 1) {
280261007b31SStefan Hajnoczi         BlockDriver *drv = bs->drv;
280361007b31SStefan Hajnoczi         if (drv && drv->bdrv_io_unplug) {
280461007b31SStefan Hajnoczi             drv->bdrv_io_unplug(bs);
280561007b31SStefan Hajnoczi         }
280661007b31SStefan Hajnoczi     }
280761007b31SStefan Hajnoczi 
28086b98bd64SPaolo Bonzini     QLIST_FOREACH(child, &bs->children, next) {
28096b98bd64SPaolo Bonzini         bdrv_io_unplug(child->bs);
28106b98bd64SPaolo Bonzini     }
28116b98bd64SPaolo Bonzini }
281223d0ba93SFam Zheng 
281323d0ba93SFam Zheng void bdrv_register_buf(BlockDriverState *bs, void *host, size_t size)
281423d0ba93SFam Zheng {
281523d0ba93SFam Zheng     BdrvChild *child;
281623d0ba93SFam Zheng 
281723d0ba93SFam Zheng     if (bs->drv && bs->drv->bdrv_register_buf) {
281823d0ba93SFam Zheng         bs->drv->bdrv_register_buf(bs, host, size);
281923d0ba93SFam Zheng     }
282023d0ba93SFam Zheng     QLIST_FOREACH(child, &bs->children, next) {
282123d0ba93SFam Zheng         bdrv_register_buf(child->bs, host, size);
282223d0ba93SFam Zheng     }
282323d0ba93SFam Zheng }
282423d0ba93SFam Zheng 
282523d0ba93SFam Zheng void bdrv_unregister_buf(BlockDriverState *bs, void *host)
282623d0ba93SFam Zheng {
282723d0ba93SFam Zheng     BdrvChild *child;
282823d0ba93SFam Zheng 
282923d0ba93SFam Zheng     if (bs->drv && bs->drv->bdrv_unregister_buf) {
283023d0ba93SFam Zheng         bs->drv->bdrv_unregister_buf(bs, host);
283123d0ba93SFam Zheng     }
283223d0ba93SFam Zheng     QLIST_FOREACH(child, &bs->children, next) {
283323d0ba93SFam Zheng         bdrv_unregister_buf(child->bs, host);
283423d0ba93SFam Zheng     }
283523d0ba93SFam Zheng }
2836