xref: /qemu/block/io.c (revision 960d5fb3e8ee09bc5f1a5c84f66dce42a6cef920)
161007b31SStefan Hajnoczi /*
261007b31SStefan Hajnoczi  * Block layer I/O functions
361007b31SStefan Hajnoczi  *
461007b31SStefan Hajnoczi  * Copyright (c) 2003 Fabrice Bellard
561007b31SStefan Hajnoczi  *
661007b31SStefan Hajnoczi  * Permission is hereby granted, free of charge, to any person obtaining a copy
761007b31SStefan Hajnoczi  * of this software and associated documentation files (the "Software"), to deal
861007b31SStefan Hajnoczi  * in the Software without restriction, including without limitation the rights
961007b31SStefan Hajnoczi  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
1061007b31SStefan Hajnoczi  * copies of the Software, and to permit persons to whom the Software is
1161007b31SStefan Hajnoczi  * furnished to do so, subject to the following conditions:
1261007b31SStefan Hajnoczi  *
1361007b31SStefan Hajnoczi  * The above copyright notice and this permission notice shall be included in
1461007b31SStefan Hajnoczi  * all copies or substantial portions of the Software.
1561007b31SStefan Hajnoczi  *
1661007b31SStefan Hajnoczi  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
1761007b31SStefan Hajnoczi  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
1861007b31SStefan Hajnoczi  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
1961007b31SStefan Hajnoczi  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
2061007b31SStefan Hajnoczi  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
2161007b31SStefan Hajnoczi  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
2261007b31SStefan Hajnoczi  * THE SOFTWARE.
2361007b31SStefan Hajnoczi  */
2461007b31SStefan Hajnoczi 
2580c71a24SPeter Maydell #include "qemu/osdep.h"
2661007b31SStefan Hajnoczi #include "trace.h"
277f0e9da6SMax Reitz #include "sysemu/block-backend.h"
287719f3c9SStefan Hajnoczi #include "block/aio-wait.h"
2961007b31SStefan Hajnoczi #include "block/blockjob.h"
30f321dcb5SPaolo Bonzini #include "block/blockjob_int.h"
3161007b31SStefan Hajnoczi #include "block/block_int.h"
3221c2283eSVladimir Sementsov-Ogievskiy #include "block/coroutines.h"
33f348b6d1SVeronia Bahaa #include "qemu/cutils.h"
34da34e65cSMarkus Armbruster #include "qapi/error.h"
35d49b6836SMarkus Armbruster #include "qemu/error-report.h"
36db725815SMarkus Armbruster #include "qemu/main-loop.h"
37c8aa7895SPavel Dovgalyuk #include "sysemu/replay.h"
3861007b31SStefan Hajnoczi 
39cb2e2878SEric Blake /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */
40cb2e2878SEric Blake #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS)
41cb2e2878SEric Blake 
427f8f03efSFam Zheng static void bdrv_parent_cb_resize(BlockDriverState *bs);
43d05aa8bbSEric Blake static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
44f5a5ca79SManos Pitsidianakis     int64_t offset, int bytes, BdrvRequestFlags flags);
4561007b31SStefan Hajnoczi 
46f4c8a43bSMax Reitz static void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore,
476cd5c9d7SKevin Wolf                                       bool ignore_bds_parents)
4861007b31SStefan Hajnoczi {
4902d21300SKevin Wolf     BdrvChild *c, *next;
5027ccdd52SKevin Wolf 
5102d21300SKevin Wolf     QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
52bd86fb99SMax Reitz         if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) {
530152bf40SKevin Wolf             continue;
540152bf40SKevin Wolf         }
554be6a6d1SKevin Wolf         bdrv_parent_drained_begin_single(c, false);
56ce0f1412SPaolo Bonzini     }
57ce0f1412SPaolo Bonzini }
58ce0f1412SPaolo Bonzini 
59e037c09cSMax Reitz static void bdrv_parent_drained_end_single_no_poll(BdrvChild *c,
60e037c09cSMax Reitz                                                    int *drained_end_counter)
61804db8eaSMax Reitz {
62804db8eaSMax Reitz     assert(c->parent_quiesce_counter > 0);
63804db8eaSMax Reitz     c->parent_quiesce_counter--;
64bd86fb99SMax Reitz     if (c->klass->drained_end) {
65bd86fb99SMax Reitz         c->klass->drained_end(c, drained_end_counter);
66804db8eaSMax Reitz     }
67804db8eaSMax Reitz }
68804db8eaSMax Reitz 
69e037c09cSMax Reitz void bdrv_parent_drained_end_single(BdrvChild *c)
70e037c09cSMax Reitz {
71e037c09cSMax Reitz     int drained_end_counter = 0;
72e037c09cSMax Reitz     bdrv_parent_drained_end_single_no_poll(c, &drained_end_counter);
73d73415a3SStefan Hajnoczi     BDRV_POLL_WHILE(c->bs, qatomic_read(&drained_end_counter) > 0);
74e037c09cSMax Reitz }
75e037c09cSMax Reitz 
76f4c8a43bSMax Reitz static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
77e037c09cSMax Reitz                                     bool ignore_bds_parents,
78e037c09cSMax Reitz                                     int *drained_end_counter)
79ce0f1412SPaolo Bonzini {
8061ad631cSMax Reitz     BdrvChild *c;
8127ccdd52SKevin Wolf 
8261ad631cSMax Reitz     QLIST_FOREACH(c, &bs->parents, next_parent) {
83bd86fb99SMax Reitz         if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) {
840152bf40SKevin Wolf             continue;
850152bf40SKevin Wolf         }
86e037c09cSMax Reitz         bdrv_parent_drained_end_single_no_poll(c, drained_end_counter);
87c2066af0SKevin Wolf     }
8861007b31SStefan Hajnoczi }
8961007b31SStefan Hajnoczi 
904be6a6d1SKevin Wolf static bool bdrv_parent_drained_poll_single(BdrvChild *c)
914be6a6d1SKevin Wolf {
92bd86fb99SMax Reitz     if (c->klass->drained_poll) {
93bd86fb99SMax Reitz         return c->klass->drained_poll(c);
944be6a6d1SKevin Wolf     }
954be6a6d1SKevin Wolf     return false;
964be6a6d1SKevin Wolf }
974be6a6d1SKevin Wolf 
986cd5c9d7SKevin Wolf static bool bdrv_parent_drained_poll(BlockDriverState *bs, BdrvChild *ignore,
996cd5c9d7SKevin Wolf                                      bool ignore_bds_parents)
10089bd0305SKevin Wolf {
10189bd0305SKevin Wolf     BdrvChild *c, *next;
10289bd0305SKevin Wolf     bool busy = false;
10389bd0305SKevin Wolf 
10489bd0305SKevin Wolf     QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
105bd86fb99SMax Reitz         if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) {
10689bd0305SKevin Wolf             continue;
10789bd0305SKevin Wolf         }
1084be6a6d1SKevin Wolf         busy |= bdrv_parent_drained_poll_single(c);
10989bd0305SKevin Wolf     }
11089bd0305SKevin Wolf 
11189bd0305SKevin Wolf     return busy;
11289bd0305SKevin Wolf }
11389bd0305SKevin Wolf 
1144be6a6d1SKevin Wolf void bdrv_parent_drained_begin_single(BdrvChild *c, bool poll)
1154be6a6d1SKevin Wolf {
116804db8eaSMax Reitz     c->parent_quiesce_counter++;
117bd86fb99SMax Reitz     if (c->klass->drained_begin) {
118bd86fb99SMax Reitz         c->klass->drained_begin(c);
1194be6a6d1SKevin Wolf     }
1204be6a6d1SKevin Wolf     if (poll) {
1214be6a6d1SKevin Wolf         BDRV_POLL_WHILE(c->bs, bdrv_parent_drained_poll_single(c));
1224be6a6d1SKevin Wolf     }
1234be6a6d1SKevin Wolf }
1244be6a6d1SKevin Wolf 
125d9e0dfa2SEric Blake static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src)
126d9e0dfa2SEric Blake {
127d9e0dfa2SEric Blake     dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer);
128d9e0dfa2SEric Blake     dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer);
129d9e0dfa2SEric Blake     dst->opt_mem_alignment = MAX(dst->opt_mem_alignment,
130d9e0dfa2SEric Blake                                  src->opt_mem_alignment);
131d9e0dfa2SEric Blake     dst->min_mem_alignment = MAX(dst->min_mem_alignment,
132d9e0dfa2SEric Blake                                  src->min_mem_alignment);
133d9e0dfa2SEric Blake     dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov);
134d9e0dfa2SEric Blake }
135d9e0dfa2SEric Blake 
13661007b31SStefan Hajnoczi void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
13761007b31SStefan Hajnoczi {
13833985614SVladimir Sementsov-Ogievskiy     ERRP_GUARD();
13961007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
14066b129acSMax Reitz     BdrvChild *c;
14166b129acSMax Reitz     bool have_limits;
14261007b31SStefan Hajnoczi 
14361007b31SStefan Hajnoczi     memset(&bs->bl, 0, sizeof(bs->bl));
14461007b31SStefan Hajnoczi 
14561007b31SStefan Hajnoczi     if (!drv) {
14661007b31SStefan Hajnoczi         return;
14761007b31SStefan Hajnoczi     }
14861007b31SStefan Hajnoczi 
14979ba8c98SEric Blake     /* Default alignment based on whether driver has byte interface */
150e31f6864SEric Blake     bs->bl.request_alignment = (drv->bdrv_co_preadv ||
151ac850bf0SVladimir Sementsov-Ogievskiy                                 drv->bdrv_aio_preadv ||
152ac850bf0SVladimir Sementsov-Ogievskiy                                 drv->bdrv_co_preadv_part) ? 1 : 512;
15379ba8c98SEric Blake 
15461007b31SStefan Hajnoczi     /* Take some limits from the children as a default */
15566b129acSMax Reitz     have_limits = false;
15666b129acSMax Reitz     QLIST_FOREACH(c, &bs->children, next) {
15766b129acSMax Reitz         if (c->role & (BDRV_CHILD_DATA | BDRV_CHILD_FILTERED | BDRV_CHILD_COW))
15866b129acSMax Reitz         {
15933985614SVladimir Sementsov-Ogievskiy             bdrv_refresh_limits(c->bs, errp);
16033985614SVladimir Sementsov-Ogievskiy             if (*errp) {
16161007b31SStefan Hajnoczi                 return;
16261007b31SStefan Hajnoczi             }
16366b129acSMax Reitz             bdrv_merge_limits(&bs->bl, &c->bs->bl);
16466b129acSMax Reitz             have_limits = true;
16566b129acSMax Reitz         }
16666b129acSMax Reitz     }
16766b129acSMax Reitz 
16866b129acSMax Reitz     if (!have_limits) {
1694196d2f0SDenis V. Lunev         bs->bl.min_mem_alignment = 512;
170038adc2fSWei Yang         bs->bl.opt_mem_alignment = qemu_real_host_page_size;
171bd44feb7SStefan Hajnoczi 
172bd44feb7SStefan Hajnoczi         /* Safe default since most protocols use readv()/writev()/etc */
173bd44feb7SStefan Hajnoczi         bs->bl.max_iov = IOV_MAX;
17461007b31SStefan Hajnoczi     }
17561007b31SStefan Hajnoczi 
17661007b31SStefan Hajnoczi     /* Then let the driver override it */
17761007b31SStefan Hajnoczi     if (drv->bdrv_refresh_limits) {
17861007b31SStefan Hajnoczi         drv->bdrv_refresh_limits(bs, errp);
1798b117001SVladimir Sementsov-Ogievskiy         if (*errp) {
1808b117001SVladimir Sementsov-Ogievskiy             return;
1818b117001SVladimir Sementsov-Ogievskiy         }
1828b117001SVladimir Sementsov-Ogievskiy     }
1838b117001SVladimir Sementsov-Ogievskiy 
1848b117001SVladimir Sementsov-Ogievskiy     if (bs->bl.request_alignment > BDRV_MAX_ALIGNMENT) {
1858b117001SVladimir Sementsov-Ogievskiy         error_setg(errp, "Driver requires too large request alignment");
18661007b31SStefan Hajnoczi     }
18761007b31SStefan Hajnoczi }
18861007b31SStefan Hajnoczi 
18961007b31SStefan Hajnoczi /**
19061007b31SStefan Hajnoczi  * The copy-on-read flag is actually a reference count so multiple users may
19161007b31SStefan Hajnoczi  * use the feature without worrying about clobbering its previous state.
19261007b31SStefan Hajnoczi  * Copy-on-read stays enabled until all users have called to disable it.
19361007b31SStefan Hajnoczi  */
19461007b31SStefan Hajnoczi void bdrv_enable_copy_on_read(BlockDriverState *bs)
19561007b31SStefan Hajnoczi {
196d73415a3SStefan Hajnoczi     qatomic_inc(&bs->copy_on_read);
19761007b31SStefan Hajnoczi }
19861007b31SStefan Hajnoczi 
19961007b31SStefan Hajnoczi void bdrv_disable_copy_on_read(BlockDriverState *bs)
20061007b31SStefan Hajnoczi {
201d73415a3SStefan Hajnoczi     int old = qatomic_fetch_dec(&bs->copy_on_read);
202d3faa13eSPaolo Bonzini     assert(old >= 1);
20361007b31SStefan Hajnoczi }
20461007b31SStefan Hajnoczi 
20561124f03SPaolo Bonzini typedef struct {
20661124f03SPaolo Bonzini     Coroutine *co;
20761124f03SPaolo Bonzini     BlockDriverState *bs;
20861124f03SPaolo Bonzini     bool done;
209481cad48SManos Pitsidianakis     bool begin;
210b0165585SKevin Wolf     bool recursive;
211fe4f0614SKevin Wolf     bool poll;
2120152bf40SKevin Wolf     BdrvChild *parent;
2136cd5c9d7SKevin Wolf     bool ignore_bds_parents;
2148e1da77eSMax Reitz     int *drained_end_counter;
21561124f03SPaolo Bonzini } BdrvCoDrainData;
21661124f03SPaolo Bonzini 
21761124f03SPaolo Bonzini static void coroutine_fn bdrv_drain_invoke_entry(void *opaque)
21861124f03SPaolo Bonzini {
21961124f03SPaolo Bonzini     BdrvCoDrainData *data = opaque;
22061124f03SPaolo Bonzini     BlockDriverState *bs = data->bs;
22161124f03SPaolo Bonzini 
222481cad48SManos Pitsidianakis     if (data->begin) {
223f8ea8dacSManos Pitsidianakis         bs->drv->bdrv_co_drain_begin(bs);
224481cad48SManos Pitsidianakis     } else {
225481cad48SManos Pitsidianakis         bs->drv->bdrv_co_drain_end(bs);
226481cad48SManos Pitsidianakis     }
22761124f03SPaolo Bonzini 
22865181d63SMax Reitz     /* Set data->done and decrement drained_end_counter before bdrv_wakeup() */
229d73415a3SStefan Hajnoczi     qatomic_mb_set(&data->done, true);
230e037c09cSMax Reitz     if (!data->begin) {
231d73415a3SStefan Hajnoczi         qatomic_dec(data->drained_end_counter);
2328e1da77eSMax Reitz     }
23365181d63SMax Reitz     bdrv_dec_in_flight(bs);
2348e1da77eSMax Reitz 
2350109e7e6SKevin Wolf     g_free(data);
2360109e7e6SKevin Wolf }
23761124f03SPaolo Bonzini 
238db0289b9SKevin Wolf /* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */
2398e1da77eSMax Reitz static void bdrv_drain_invoke(BlockDriverState *bs, bool begin,
2408e1da77eSMax Reitz                               int *drained_end_counter)
24161124f03SPaolo Bonzini {
2420109e7e6SKevin Wolf     BdrvCoDrainData *data;
24361124f03SPaolo Bonzini 
244f8ea8dacSManos Pitsidianakis     if (!bs->drv || (begin && !bs->drv->bdrv_co_drain_begin) ||
245481cad48SManos Pitsidianakis             (!begin && !bs->drv->bdrv_co_drain_end)) {
24661124f03SPaolo Bonzini         return;
24761124f03SPaolo Bonzini     }
24861124f03SPaolo Bonzini 
2490109e7e6SKevin Wolf     data = g_new(BdrvCoDrainData, 1);
2500109e7e6SKevin Wolf     *data = (BdrvCoDrainData) {
2510109e7e6SKevin Wolf         .bs = bs,
2520109e7e6SKevin Wolf         .done = false,
2538e1da77eSMax Reitz         .begin = begin,
2548e1da77eSMax Reitz         .drained_end_counter = drained_end_counter,
2550109e7e6SKevin Wolf     };
2560109e7e6SKevin Wolf 
257e037c09cSMax Reitz     if (!begin) {
258d73415a3SStefan Hajnoczi         qatomic_inc(drained_end_counter);
2598e1da77eSMax Reitz     }
2608e1da77eSMax Reitz 
2610109e7e6SKevin Wolf     /* Make sure the driver callback completes during the polling phase for
2620109e7e6SKevin Wolf      * drain_begin. */
2630109e7e6SKevin Wolf     bdrv_inc_in_flight(bs);
2640109e7e6SKevin Wolf     data->co = qemu_coroutine_create(bdrv_drain_invoke_entry, data);
2650109e7e6SKevin Wolf     aio_co_schedule(bdrv_get_aio_context(bs), data->co);
26661124f03SPaolo Bonzini }
26761124f03SPaolo Bonzini 
2681cc8e54aSKevin Wolf /* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */
269fe4f0614SKevin Wolf bool bdrv_drain_poll(BlockDriverState *bs, bool recursive,
2706cd5c9d7SKevin Wolf                      BdrvChild *ignore_parent, bool ignore_bds_parents)
27189bd0305SKevin Wolf {
272fe4f0614SKevin Wolf     BdrvChild *child, *next;
273fe4f0614SKevin Wolf 
2746cd5c9d7SKevin Wolf     if (bdrv_parent_drained_poll(bs, ignore_parent, ignore_bds_parents)) {
27589bd0305SKevin Wolf         return true;
27689bd0305SKevin Wolf     }
27789bd0305SKevin Wolf 
278d73415a3SStefan Hajnoczi     if (qatomic_read(&bs->in_flight)) {
279fe4f0614SKevin Wolf         return true;
28089bd0305SKevin Wolf     }
28189bd0305SKevin Wolf 
282fe4f0614SKevin Wolf     if (recursive) {
2836cd5c9d7SKevin Wolf         assert(!ignore_bds_parents);
284fe4f0614SKevin Wolf         QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
2856cd5c9d7SKevin Wolf             if (bdrv_drain_poll(child->bs, recursive, child, false)) {
286fe4f0614SKevin Wolf                 return true;
287fe4f0614SKevin Wolf             }
288fe4f0614SKevin Wolf         }
289fe4f0614SKevin Wolf     }
290fe4f0614SKevin Wolf 
291fe4f0614SKevin Wolf     return false;
292fe4f0614SKevin Wolf }
293fe4f0614SKevin Wolf 
294fe4f0614SKevin Wolf static bool bdrv_drain_poll_top_level(BlockDriverState *bs, bool recursive,
29589bd0305SKevin Wolf                                       BdrvChild *ignore_parent)
2961cc8e54aSKevin Wolf {
2976cd5c9d7SKevin Wolf     return bdrv_drain_poll(bs, recursive, ignore_parent, false);
2981cc8e54aSKevin Wolf }
2991cc8e54aSKevin Wolf 
300b0165585SKevin Wolf static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
3016cd5c9d7SKevin Wolf                                   BdrvChild *parent, bool ignore_bds_parents,
3026cd5c9d7SKevin Wolf                                   bool poll);
303b0165585SKevin Wolf static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
3048e1da77eSMax Reitz                                 BdrvChild *parent, bool ignore_bds_parents,
3058e1da77eSMax Reitz                                 int *drained_end_counter);
3060152bf40SKevin Wolf 
307a77fd4bbSFam Zheng static void bdrv_co_drain_bh_cb(void *opaque)
308a77fd4bbSFam Zheng {
309a77fd4bbSFam Zheng     BdrvCoDrainData *data = opaque;
310a77fd4bbSFam Zheng     Coroutine *co = data->co;
31199723548SPaolo Bonzini     BlockDriverState *bs = data->bs;
312a77fd4bbSFam Zheng 
313c8ca33d0SKevin Wolf     if (bs) {
314aa1361d5SKevin Wolf         AioContext *ctx = bdrv_get_aio_context(bs);
315aa1361d5SKevin Wolf         aio_context_acquire(ctx);
31699723548SPaolo Bonzini         bdrv_dec_in_flight(bs);
317481cad48SManos Pitsidianakis         if (data->begin) {
318e037c09cSMax Reitz             assert(!data->drained_end_counter);
3196cd5c9d7SKevin Wolf             bdrv_do_drained_begin(bs, data->recursive, data->parent,
3206cd5c9d7SKevin Wolf                                   data->ignore_bds_parents, data->poll);
321481cad48SManos Pitsidianakis         } else {
322e037c09cSMax Reitz             assert(!data->poll);
3236cd5c9d7SKevin Wolf             bdrv_do_drained_end(bs, data->recursive, data->parent,
3248e1da77eSMax Reitz                                 data->ignore_bds_parents,
3258e1da77eSMax Reitz                                 data->drained_end_counter);
326481cad48SManos Pitsidianakis         }
327aa1361d5SKevin Wolf         aio_context_release(ctx);
328c8ca33d0SKevin Wolf     } else {
329c8ca33d0SKevin Wolf         assert(data->begin);
330c8ca33d0SKevin Wolf         bdrv_drain_all_begin();
331c8ca33d0SKevin Wolf     }
332481cad48SManos Pitsidianakis 
333a77fd4bbSFam Zheng     data->done = true;
3341919631eSPaolo Bonzini     aio_co_wake(co);
335a77fd4bbSFam Zheng }
336a77fd4bbSFam Zheng 
337481cad48SManos Pitsidianakis static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
338b0165585SKevin Wolf                                                 bool begin, bool recursive,
3396cd5c9d7SKevin Wolf                                                 BdrvChild *parent,
3406cd5c9d7SKevin Wolf                                                 bool ignore_bds_parents,
3418e1da77eSMax Reitz                                                 bool poll,
3428e1da77eSMax Reitz                                                 int *drained_end_counter)
343a77fd4bbSFam Zheng {
344a77fd4bbSFam Zheng     BdrvCoDrainData data;
345*960d5fb3SKevin Wolf     Coroutine *self = qemu_coroutine_self();
346*960d5fb3SKevin Wolf     AioContext *ctx = bdrv_get_aio_context(bs);
347*960d5fb3SKevin Wolf     AioContext *co_ctx = qemu_coroutine_get_aio_context(self);
348a77fd4bbSFam Zheng 
349a77fd4bbSFam Zheng     /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
350c40a2545SStefan Hajnoczi      * other coroutines run if they were queued by aio_co_enter(). */
351a77fd4bbSFam Zheng 
352a77fd4bbSFam Zheng     assert(qemu_in_coroutine());
353a77fd4bbSFam Zheng     data = (BdrvCoDrainData) {
354*960d5fb3SKevin Wolf         .co = self,
355a77fd4bbSFam Zheng         .bs = bs,
356a77fd4bbSFam Zheng         .done = false,
357481cad48SManos Pitsidianakis         .begin = begin,
358b0165585SKevin Wolf         .recursive = recursive,
3590152bf40SKevin Wolf         .parent = parent,
3606cd5c9d7SKevin Wolf         .ignore_bds_parents = ignore_bds_parents,
361fe4f0614SKevin Wolf         .poll = poll,
3628e1da77eSMax Reitz         .drained_end_counter = drained_end_counter,
363a77fd4bbSFam Zheng     };
3648e1da77eSMax Reitz 
365c8ca33d0SKevin Wolf     if (bs) {
36699723548SPaolo Bonzini         bdrv_inc_in_flight(bs);
367c8ca33d0SKevin Wolf     }
368*960d5fb3SKevin Wolf 
369*960d5fb3SKevin Wolf     /*
370*960d5fb3SKevin Wolf      * Temporarily drop the lock across yield or we would get deadlocks.
371*960d5fb3SKevin Wolf      * bdrv_co_drain_bh_cb() reaquires the lock as needed.
372*960d5fb3SKevin Wolf      *
373*960d5fb3SKevin Wolf      * When we yield below, the lock for the current context will be
374*960d5fb3SKevin Wolf      * released, so if this is actually the lock that protects bs, don't drop
375*960d5fb3SKevin Wolf      * it a second time.
376*960d5fb3SKevin Wolf      */
377*960d5fb3SKevin Wolf     if (ctx != co_ctx) {
378*960d5fb3SKevin Wolf         aio_context_release(ctx);
379*960d5fb3SKevin Wolf     }
380*960d5fb3SKevin Wolf     replay_bh_schedule_oneshot_event(ctx, bdrv_co_drain_bh_cb, &data);
381a77fd4bbSFam Zheng 
382a77fd4bbSFam Zheng     qemu_coroutine_yield();
383a77fd4bbSFam Zheng     /* If we are resumed from some other event (such as an aio completion or a
384a77fd4bbSFam Zheng      * timer callback), it is a bug in the caller that should be fixed. */
385a77fd4bbSFam Zheng     assert(data.done);
386*960d5fb3SKevin Wolf 
387*960d5fb3SKevin Wolf     /* Reaquire the AioContext of bs if we dropped it */
388*960d5fb3SKevin Wolf     if (ctx != co_ctx) {
389*960d5fb3SKevin Wolf         aio_context_acquire(ctx);
390*960d5fb3SKevin Wolf     }
391a77fd4bbSFam Zheng }
392a77fd4bbSFam Zheng 
393dcf94a23SKevin Wolf void bdrv_do_drained_begin_quiesce(BlockDriverState *bs,
3946cd5c9d7SKevin Wolf                                    BdrvChild *parent, bool ignore_bds_parents)
395dcf94a23SKevin Wolf {
396dcf94a23SKevin Wolf     assert(!qemu_in_coroutine());
397dcf94a23SKevin Wolf 
398dcf94a23SKevin Wolf     /* Stop things in parent-to-child order */
399d73415a3SStefan Hajnoczi     if (qatomic_fetch_inc(&bs->quiesce_counter) == 0) {
400dcf94a23SKevin Wolf         aio_disable_external(bdrv_get_aio_context(bs));
401dcf94a23SKevin Wolf     }
402dcf94a23SKevin Wolf 
4036cd5c9d7SKevin Wolf     bdrv_parent_drained_begin(bs, parent, ignore_bds_parents);
4048e1da77eSMax Reitz     bdrv_drain_invoke(bs, true, NULL);
405dcf94a23SKevin Wolf }
406dcf94a23SKevin Wolf 
407dcf94a23SKevin Wolf static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
4086cd5c9d7SKevin Wolf                                   BdrvChild *parent, bool ignore_bds_parents,
4096cd5c9d7SKevin Wolf                                   bool poll)
4106820643fSKevin Wolf {
411b0165585SKevin Wolf     BdrvChild *child, *next;
412b0165585SKevin Wolf 
413d42cf288SPaolo Bonzini     if (qemu_in_coroutine()) {
4146cd5c9d7SKevin Wolf         bdrv_co_yield_to_drain(bs, true, recursive, parent, ignore_bds_parents,
4158e1da77eSMax Reitz                                poll, NULL);
416d42cf288SPaolo Bonzini         return;
417d42cf288SPaolo Bonzini     }
418d42cf288SPaolo Bonzini 
4196cd5c9d7SKevin Wolf     bdrv_do_drained_begin_quiesce(bs, parent, ignore_bds_parents);
420d30b8e64SKevin Wolf 
421b0165585SKevin Wolf     if (recursive) {
4226cd5c9d7SKevin Wolf         assert(!ignore_bds_parents);
423d736f119SKevin Wolf         bs->recursive_quiesce_counter++;
424b0165585SKevin Wolf         QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
4256cd5c9d7SKevin Wolf             bdrv_do_drained_begin(child->bs, true, child, ignore_bds_parents,
4266cd5c9d7SKevin Wolf                                   false);
427b0165585SKevin Wolf         }
428b0165585SKevin Wolf     }
429fe4f0614SKevin Wolf 
430fe4f0614SKevin Wolf     /*
431fe4f0614SKevin Wolf      * Wait for drained requests to finish.
432fe4f0614SKevin Wolf      *
433fe4f0614SKevin Wolf      * Calling BDRV_POLL_WHILE() only once for the top-level node is okay: The
434fe4f0614SKevin Wolf      * call is needed so things in this AioContext can make progress even
435fe4f0614SKevin Wolf      * though we don't return to the main AioContext loop - this automatically
436fe4f0614SKevin Wolf      * includes other nodes in the same AioContext and therefore all child
437fe4f0614SKevin Wolf      * nodes.
438fe4f0614SKevin Wolf      */
439fe4f0614SKevin Wolf     if (poll) {
4406cd5c9d7SKevin Wolf         assert(!ignore_bds_parents);
441fe4f0614SKevin Wolf         BDRV_POLL_WHILE(bs, bdrv_drain_poll_top_level(bs, recursive, parent));
442fe4f0614SKevin Wolf     }
4436820643fSKevin Wolf }
4446820643fSKevin Wolf 
4450152bf40SKevin Wolf void bdrv_drained_begin(BlockDriverState *bs)
4460152bf40SKevin Wolf {
4476cd5c9d7SKevin Wolf     bdrv_do_drained_begin(bs, false, NULL, false, true);
4480152bf40SKevin Wolf }
4490152bf40SKevin Wolf 
450b0165585SKevin Wolf void bdrv_subtree_drained_begin(BlockDriverState *bs)
4516820643fSKevin Wolf {
4526cd5c9d7SKevin Wolf     bdrv_do_drained_begin(bs, true, NULL, false, true);
453b0165585SKevin Wolf }
454b0165585SKevin Wolf 
455e037c09cSMax Reitz /**
456e037c09cSMax Reitz  * This function does not poll, nor must any of its recursively called
457e037c09cSMax Reitz  * functions.  The *drained_end_counter pointee will be incremented
458e037c09cSMax Reitz  * once for every background operation scheduled, and decremented once
459e037c09cSMax Reitz  * the operation settles.  Therefore, the pointer must remain valid
460e037c09cSMax Reitz  * until the pointee reaches 0.  That implies that whoever sets up the
461e037c09cSMax Reitz  * pointee has to poll until it is 0.
462e037c09cSMax Reitz  *
463e037c09cSMax Reitz  * We use atomic operations to access *drained_end_counter, because
464e037c09cSMax Reitz  * (1) when called from bdrv_set_aio_context_ignore(), the subgraph of
465e037c09cSMax Reitz  *     @bs may contain nodes in different AioContexts,
466e037c09cSMax Reitz  * (2) bdrv_drain_all_end() uses the same counter for all nodes,
467e037c09cSMax Reitz  *     regardless of which AioContext they are in.
468e037c09cSMax Reitz  */
4696cd5c9d7SKevin Wolf static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
4708e1da77eSMax Reitz                                 BdrvChild *parent, bool ignore_bds_parents,
4718e1da77eSMax Reitz                                 int *drained_end_counter)
472b0165585SKevin Wolf {
47361ad631cSMax Reitz     BdrvChild *child;
4740f115168SKevin Wolf     int old_quiesce_counter;
4750f115168SKevin Wolf 
476e037c09cSMax Reitz     assert(drained_end_counter != NULL);
477e037c09cSMax Reitz 
478481cad48SManos Pitsidianakis     if (qemu_in_coroutine()) {
4796cd5c9d7SKevin Wolf         bdrv_co_yield_to_drain(bs, false, recursive, parent, ignore_bds_parents,
4808e1da77eSMax Reitz                                false, drained_end_counter);
481481cad48SManos Pitsidianakis         return;
482481cad48SManos Pitsidianakis     }
4836820643fSKevin Wolf     assert(bs->quiesce_counter > 0);
4846820643fSKevin Wolf 
48560369b86SKevin Wolf     /* Re-enable things in child-to-parent order */
4868e1da77eSMax Reitz     bdrv_drain_invoke(bs, false, drained_end_counter);
487e037c09cSMax Reitz     bdrv_parent_drained_end(bs, parent, ignore_bds_parents,
488e037c09cSMax Reitz                             drained_end_counter);
4895cb2737eSMax Reitz 
490d73415a3SStefan Hajnoczi     old_quiesce_counter = qatomic_fetch_dec(&bs->quiesce_counter);
4910f115168SKevin Wolf     if (old_quiesce_counter == 1) {
4926820643fSKevin Wolf         aio_enable_external(bdrv_get_aio_context(bs));
4936820643fSKevin Wolf     }
494b0165585SKevin Wolf 
495b0165585SKevin Wolf     if (recursive) {
4966cd5c9d7SKevin Wolf         assert(!ignore_bds_parents);
497d736f119SKevin Wolf         bs->recursive_quiesce_counter--;
49861ad631cSMax Reitz         QLIST_FOREACH(child, &bs->children, next) {
4998e1da77eSMax Reitz             bdrv_do_drained_end(child->bs, true, child, ignore_bds_parents,
5008e1da77eSMax Reitz                                 drained_end_counter);
501b0165585SKevin Wolf         }
502b0165585SKevin Wolf     }
5030f115168SKevin Wolf }
5046820643fSKevin Wolf 
5050152bf40SKevin Wolf void bdrv_drained_end(BlockDriverState *bs)
5060152bf40SKevin Wolf {
507e037c09cSMax Reitz     int drained_end_counter = 0;
508e037c09cSMax Reitz     bdrv_do_drained_end(bs, false, NULL, false, &drained_end_counter);
509d73415a3SStefan Hajnoczi     BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0);
510e037c09cSMax Reitz }
511e037c09cSMax Reitz 
512e037c09cSMax Reitz void bdrv_drained_end_no_poll(BlockDriverState *bs, int *drained_end_counter)
513e037c09cSMax Reitz {
514e037c09cSMax Reitz     bdrv_do_drained_end(bs, false, NULL, false, drained_end_counter);
515b0165585SKevin Wolf }
516b0165585SKevin Wolf 
517b0165585SKevin Wolf void bdrv_subtree_drained_end(BlockDriverState *bs)
518b0165585SKevin Wolf {
519e037c09cSMax Reitz     int drained_end_counter = 0;
520e037c09cSMax Reitz     bdrv_do_drained_end(bs, true, NULL, false, &drained_end_counter);
521d73415a3SStefan Hajnoczi     BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0);
5220152bf40SKevin Wolf }
5230152bf40SKevin Wolf 
524d736f119SKevin Wolf void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent)
525d736f119SKevin Wolf {
526d736f119SKevin Wolf     int i;
527d736f119SKevin Wolf 
528d736f119SKevin Wolf     for (i = 0; i < new_parent->recursive_quiesce_counter; i++) {
5296cd5c9d7SKevin Wolf         bdrv_do_drained_begin(child->bs, true, child, false, true);
530d736f119SKevin Wolf     }
531d736f119SKevin Wolf }
532d736f119SKevin Wolf 
533d736f119SKevin Wolf void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent)
534d736f119SKevin Wolf {
535e037c09cSMax Reitz     int drained_end_counter = 0;
536d736f119SKevin Wolf     int i;
537d736f119SKevin Wolf 
538d736f119SKevin Wolf     for (i = 0; i < old_parent->recursive_quiesce_counter; i++) {
539e037c09cSMax Reitz         bdrv_do_drained_end(child->bs, true, child, false,
540e037c09cSMax Reitz                             &drained_end_counter);
541d736f119SKevin Wolf     }
542e037c09cSMax Reitz 
543d73415a3SStefan Hajnoczi     BDRV_POLL_WHILE(child->bs, qatomic_read(&drained_end_counter) > 0);
544d736f119SKevin Wolf }
545d736f119SKevin Wolf 
54661007b31SStefan Hajnoczi /*
54767da1dc5SFam Zheng  * Wait for pending requests to complete on a single BlockDriverState subtree,
54867da1dc5SFam Zheng  * and suspend block driver's internal I/O until next request arrives.
54961007b31SStefan Hajnoczi  *
55061007b31SStefan Hajnoczi  * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState
55161007b31SStefan Hajnoczi  * AioContext.
55261007b31SStefan Hajnoczi  */
553b6e84c97SPaolo Bonzini void coroutine_fn bdrv_co_drain(BlockDriverState *bs)
554b6e84c97SPaolo Bonzini {
5556820643fSKevin Wolf     assert(qemu_in_coroutine());
5566820643fSKevin Wolf     bdrv_drained_begin(bs);
5576820643fSKevin Wolf     bdrv_drained_end(bs);
558b6e84c97SPaolo Bonzini }
559b6e84c97SPaolo Bonzini 
56061007b31SStefan Hajnoczi void bdrv_drain(BlockDriverState *bs)
56161007b31SStefan Hajnoczi {
5626820643fSKevin Wolf     bdrv_drained_begin(bs);
5636820643fSKevin Wolf     bdrv_drained_end(bs);
56461007b31SStefan Hajnoczi }
56561007b31SStefan Hajnoczi 
566c13ad59fSKevin Wolf static void bdrv_drain_assert_idle(BlockDriverState *bs)
567c13ad59fSKevin Wolf {
568c13ad59fSKevin Wolf     BdrvChild *child, *next;
569c13ad59fSKevin Wolf 
570d73415a3SStefan Hajnoczi     assert(qatomic_read(&bs->in_flight) == 0);
571c13ad59fSKevin Wolf     QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
572c13ad59fSKevin Wolf         bdrv_drain_assert_idle(child->bs);
573c13ad59fSKevin Wolf     }
574c13ad59fSKevin Wolf }
575c13ad59fSKevin Wolf 
5760f12264eSKevin Wolf unsigned int bdrv_drain_all_count = 0;
5770f12264eSKevin Wolf 
5780f12264eSKevin Wolf static bool bdrv_drain_all_poll(void)
5790f12264eSKevin Wolf {
5800f12264eSKevin Wolf     BlockDriverState *bs = NULL;
5810f12264eSKevin Wolf     bool result = false;
5820f12264eSKevin Wolf 
5830f12264eSKevin Wolf     /* bdrv_drain_poll() can't make changes to the graph and we are holding the
5840f12264eSKevin Wolf      * main AioContext lock, so iterating bdrv_next_all_states() is safe. */
5850f12264eSKevin Wolf     while ((bs = bdrv_next_all_states(bs))) {
5860f12264eSKevin Wolf         AioContext *aio_context = bdrv_get_aio_context(bs);
5870f12264eSKevin Wolf         aio_context_acquire(aio_context);
5880f12264eSKevin Wolf         result |= bdrv_drain_poll(bs, false, NULL, true);
5890f12264eSKevin Wolf         aio_context_release(aio_context);
5900f12264eSKevin Wolf     }
5910f12264eSKevin Wolf 
5920f12264eSKevin Wolf     return result;
5930f12264eSKevin Wolf }
5940f12264eSKevin Wolf 
59561007b31SStefan Hajnoczi /*
59661007b31SStefan Hajnoczi  * Wait for pending requests to complete across all BlockDriverStates
59761007b31SStefan Hajnoczi  *
59861007b31SStefan Hajnoczi  * This function does not flush data to disk, use bdrv_flush_all() for that
59961007b31SStefan Hajnoczi  * after calling this function.
600c0778f66SAlberto Garcia  *
601c0778f66SAlberto Garcia  * This pauses all block jobs and disables external clients. It must
602c0778f66SAlberto Garcia  * be paired with bdrv_drain_all_end().
603c0778f66SAlberto Garcia  *
604c0778f66SAlberto Garcia  * NOTE: no new block jobs or BlockDriverStates can be created between
605c0778f66SAlberto Garcia  * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls.
60661007b31SStefan Hajnoczi  */
607c0778f66SAlberto Garcia void bdrv_drain_all_begin(void)
60861007b31SStefan Hajnoczi {
6090f12264eSKevin Wolf     BlockDriverState *bs = NULL;
61061007b31SStefan Hajnoczi 
611c8ca33d0SKevin Wolf     if (qemu_in_coroutine()) {
6128e1da77eSMax Reitz         bdrv_co_yield_to_drain(NULL, true, false, NULL, true, true, NULL);
613c8ca33d0SKevin Wolf         return;
614c8ca33d0SKevin Wolf     }
615c8ca33d0SKevin Wolf 
616c8aa7895SPavel Dovgalyuk     /*
617c8aa7895SPavel Dovgalyuk      * bdrv queue is managed by record/replay,
618c8aa7895SPavel Dovgalyuk      * waiting for finishing the I/O requests may
619c8aa7895SPavel Dovgalyuk      * be infinite
620c8aa7895SPavel Dovgalyuk      */
621c8aa7895SPavel Dovgalyuk     if (replay_events_enabled()) {
622c8aa7895SPavel Dovgalyuk         return;
623c8aa7895SPavel Dovgalyuk     }
624c8aa7895SPavel Dovgalyuk 
6250f12264eSKevin Wolf     /* AIO_WAIT_WHILE() with a NULL context can only be called from the main
6260f12264eSKevin Wolf      * loop AioContext, so make sure we're in the main context. */
6279a7e86c8SKevin Wolf     assert(qemu_get_current_aio_context() == qemu_get_aio_context());
6280f12264eSKevin Wolf     assert(bdrv_drain_all_count < INT_MAX);
6290f12264eSKevin Wolf     bdrv_drain_all_count++;
6309a7e86c8SKevin Wolf 
6310f12264eSKevin Wolf     /* Quiesce all nodes, without polling in-flight requests yet. The graph
6320f12264eSKevin Wolf      * cannot change during this loop. */
6330f12264eSKevin Wolf     while ((bs = bdrv_next_all_states(bs))) {
63461007b31SStefan Hajnoczi         AioContext *aio_context = bdrv_get_aio_context(bs);
63561007b31SStefan Hajnoczi 
63661007b31SStefan Hajnoczi         aio_context_acquire(aio_context);
6370f12264eSKevin Wolf         bdrv_do_drained_begin(bs, false, NULL, true, false);
63861007b31SStefan Hajnoczi         aio_context_release(aio_context);
63961007b31SStefan Hajnoczi     }
64061007b31SStefan Hajnoczi 
6410f12264eSKevin Wolf     /* Now poll the in-flight requests */
642cfe29d82SKevin Wolf     AIO_WAIT_WHILE(NULL, bdrv_drain_all_poll());
6430f12264eSKevin Wolf 
6440f12264eSKevin Wolf     while ((bs = bdrv_next_all_states(bs))) {
645c13ad59fSKevin Wolf         bdrv_drain_assert_idle(bs);
646f406c03cSAlexander Yarygin     }
647f406c03cSAlexander Yarygin }
648c0778f66SAlberto Garcia 
6491a6d3bd2SGreg Kurz void bdrv_drain_all_end_quiesce(BlockDriverState *bs)
6501a6d3bd2SGreg Kurz {
6511a6d3bd2SGreg Kurz     int drained_end_counter = 0;
6521a6d3bd2SGreg Kurz 
6531a6d3bd2SGreg Kurz     g_assert(bs->quiesce_counter > 0);
6541a6d3bd2SGreg Kurz     g_assert(!bs->refcnt);
6551a6d3bd2SGreg Kurz 
6561a6d3bd2SGreg Kurz     while (bs->quiesce_counter) {
6571a6d3bd2SGreg Kurz         bdrv_do_drained_end(bs, false, NULL, true, &drained_end_counter);
6581a6d3bd2SGreg Kurz     }
6591a6d3bd2SGreg Kurz     BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0);
6601a6d3bd2SGreg Kurz }
6611a6d3bd2SGreg Kurz 
662c0778f66SAlberto Garcia void bdrv_drain_all_end(void)
663c0778f66SAlberto Garcia {
6640f12264eSKevin Wolf     BlockDriverState *bs = NULL;
665e037c09cSMax Reitz     int drained_end_counter = 0;
666c0778f66SAlberto Garcia 
667c8aa7895SPavel Dovgalyuk     /*
668c8aa7895SPavel Dovgalyuk      * bdrv queue is managed by record/replay,
669c8aa7895SPavel Dovgalyuk      * waiting for finishing the I/O requests may
670c8aa7895SPavel Dovgalyuk      * be endless
671c8aa7895SPavel Dovgalyuk      */
672c8aa7895SPavel Dovgalyuk     if (replay_events_enabled()) {
673c8aa7895SPavel Dovgalyuk         return;
674c8aa7895SPavel Dovgalyuk     }
675c8aa7895SPavel Dovgalyuk 
6760f12264eSKevin Wolf     while ((bs = bdrv_next_all_states(bs))) {
67761007b31SStefan Hajnoczi         AioContext *aio_context = bdrv_get_aio_context(bs);
67861007b31SStefan Hajnoczi 
67961007b31SStefan Hajnoczi         aio_context_acquire(aio_context);
680e037c09cSMax Reitz         bdrv_do_drained_end(bs, false, NULL, true, &drained_end_counter);
68161007b31SStefan Hajnoczi         aio_context_release(aio_context);
68261007b31SStefan Hajnoczi     }
6830f12264eSKevin Wolf 
684e037c09cSMax Reitz     assert(qemu_get_current_aio_context() == qemu_get_aio_context());
685d73415a3SStefan Hajnoczi     AIO_WAIT_WHILE(NULL, qatomic_read(&drained_end_counter) > 0);
686e037c09cSMax Reitz 
6870f12264eSKevin Wolf     assert(bdrv_drain_all_count > 0);
6880f12264eSKevin Wolf     bdrv_drain_all_count--;
68961007b31SStefan Hajnoczi }
69061007b31SStefan Hajnoczi 
691c0778f66SAlberto Garcia void bdrv_drain_all(void)
692c0778f66SAlberto Garcia {
693c0778f66SAlberto Garcia     bdrv_drain_all_begin();
694c0778f66SAlberto Garcia     bdrv_drain_all_end();
695c0778f66SAlberto Garcia }
696c0778f66SAlberto Garcia 
69761007b31SStefan Hajnoczi /**
69861007b31SStefan Hajnoczi  * Remove an active request from the tracked requests list
69961007b31SStefan Hajnoczi  *
70061007b31SStefan Hajnoczi  * This function should be called when a tracked request is completing.
70161007b31SStefan Hajnoczi  */
70261007b31SStefan Hajnoczi static void tracked_request_end(BdrvTrackedRequest *req)
70361007b31SStefan Hajnoczi {
70461007b31SStefan Hajnoczi     if (req->serialising) {
705d73415a3SStefan Hajnoczi         qatomic_dec(&req->bs->serialising_in_flight);
70661007b31SStefan Hajnoczi     }
70761007b31SStefan Hajnoczi 
7083783fa3dSPaolo Bonzini     qemu_co_mutex_lock(&req->bs->reqs_lock);
70961007b31SStefan Hajnoczi     QLIST_REMOVE(req, list);
71061007b31SStefan Hajnoczi     qemu_co_queue_restart_all(&req->wait_queue);
7113783fa3dSPaolo Bonzini     qemu_co_mutex_unlock(&req->bs->reqs_lock);
71261007b31SStefan Hajnoczi }
71361007b31SStefan Hajnoczi 
71461007b31SStefan Hajnoczi /**
71561007b31SStefan Hajnoczi  * Add an active request to the tracked requests list
71661007b31SStefan Hajnoczi  */
71761007b31SStefan Hajnoczi static void tracked_request_begin(BdrvTrackedRequest *req,
71861007b31SStefan Hajnoczi                                   BlockDriverState *bs,
71961007b31SStefan Hajnoczi                                   int64_t offset,
72022931a15SFam Zheng                                   uint64_t bytes,
721ebde595cSFam Zheng                                   enum BdrvTrackedRequestType type)
72261007b31SStefan Hajnoczi {
72322931a15SFam Zheng     assert(bytes <= INT64_MAX && offset <= INT64_MAX - bytes);
72422931a15SFam Zheng 
72561007b31SStefan Hajnoczi     *req = (BdrvTrackedRequest){
72661007b31SStefan Hajnoczi         .bs = bs,
72761007b31SStefan Hajnoczi         .offset         = offset,
72861007b31SStefan Hajnoczi         .bytes          = bytes,
729ebde595cSFam Zheng         .type           = type,
73061007b31SStefan Hajnoczi         .co             = qemu_coroutine_self(),
73161007b31SStefan Hajnoczi         .serialising    = false,
73261007b31SStefan Hajnoczi         .overlap_offset = offset,
73361007b31SStefan Hajnoczi         .overlap_bytes  = bytes,
73461007b31SStefan Hajnoczi     };
73561007b31SStefan Hajnoczi 
73661007b31SStefan Hajnoczi     qemu_co_queue_init(&req->wait_queue);
73761007b31SStefan Hajnoczi 
7383783fa3dSPaolo Bonzini     qemu_co_mutex_lock(&bs->reqs_lock);
73961007b31SStefan Hajnoczi     QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
7403783fa3dSPaolo Bonzini     qemu_co_mutex_unlock(&bs->reqs_lock);
74161007b31SStefan Hajnoczi }
74261007b31SStefan Hajnoczi 
7433ba0e1a0SPaolo Bonzini static bool tracked_request_overlaps(BdrvTrackedRequest *req,
7443ba0e1a0SPaolo Bonzini                                      int64_t offset, uint64_t bytes)
7453ba0e1a0SPaolo Bonzini {
7463ba0e1a0SPaolo Bonzini     /*        aaaa   bbbb */
7473ba0e1a0SPaolo Bonzini     if (offset >= req->overlap_offset + req->overlap_bytes) {
7483ba0e1a0SPaolo Bonzini         return false;
7493ba0e1a0SPaolo Bonzini     }
7503ba0e1a0SPaolo Bonzini     /* bbbb   aaaa        */
7513ba0e1a0SPaolo Bonzini     if (req->overlap_offset >= offset + bytes) {
7523ba0e1a0SPaolo Bonzini         return false;
7533ba0e1a0SPaolo Bonzini     }
7543ba0e1a0SPaolo Bonzini     return true;
7553ba0e1a0SPaolo Bonzini }
7563ba0e1a0SPaolo Bonzini 
7573ba0e1a0SPaolo Bonzini static bool coroutine_fn
7583ba0e1a0SPaolo Bonzini bdrv_wait_serialising_requests_locked(BlockDriverState *bs,
7593ba0e1a0SPaolo Bonzini                                       BdrvTrackedRequest *self)
7603ba0e1a0SPaolo Bonzini {
7613ba0e1a0SPaolo Bonzini     BdrvTrackedRequest *req;
7623ba0e1a0SPaolo Bonzini     bool retry;
7633ba0e1a0SPaolo Bonzini     bool waited = false;
7643ba0e1a0SPaolo Bonzini 
7653ba0e1a0SPaolo Bonzini     do {
7663ba0e1a0SPaolo Bonzini         retry = false;
7673ba0e1a0SPaolo Bonzini         QLIST_FOREACH(req, &bs->tracked_requests, list) {
7683ba0e1a0SPaolo Bonzini             if (req == self || (!req->serialising && !self->serialising)) {
7693ba0e1a0SPaolo Bonzini                 continue;
7703ba0e1a0SPaolo Bonzini             }
7713ba0e1a0SPaolo Bonzini             if (tracked_request_overlaps(req, self->overlap_offset,
7723ba0e1a0SPaolo Bonzini                                          self->overlap_bytes))
7733ba0e1a0SPaolo Bonzini             {
7743ba0e1a0SPaolo Bonzini                 /* Hitting this means there was a reentrant request, for
7753ba0e1a0SPaolo Bonzini                  * example, a block driver issuing nested requests.  This must
7763ba0e1a0SPaolo Bonzini                  * never happen since it means deadlock.
7773ba0e1a0SPaolo Bonzini                  */
7783ba0e1a0SPaolo Bonzini                 assert(qemu_coroutine_self() != req->co);
7793ba0e1a0SPaolo Bonzini 
7803ba0e1a0SPaolo Bonzini                 /* If the request is already (indirectly) waiting for us, or
7813ba0e1a0SPaolo Bonzini                  * will wait for us as soon as it wakes up, then just go on
7823ba0e1a0SPaolo Bonzini                  * (instead of producing a deadlock in the former case). */
7833ba0e1a0SPaolo Bonzini                 if (!req->waiting_for) {
7843ba0e1a0SPaolo Bonzini                     self->waiting_for = req;
7853ba0e1a0SPaolo Bonzini                     qemu_co_queue_wait(&req->wait_queue, &bs->reqs_lock);
7863ba0e1a0SPaolo Bonzini                     self->waiting_for = NULL;
7873ba0e1a0SPaolo Bonzini                     retry = true;
7883ba0e1a0SPaolo Bonzini                     waited = true;
7893ba0e1a0SPaolo Bonzini                     break;
7903ba0e1a0SPaolo Bonzini                 }
7913ba0e1a0SPaolo Bonzini             }
7923ba0e1a0SPaolo Bonzini         }
7933ba0e1a0SPaolo Bonzini     } while (retry);
7943ba0e1a0SPaolo Bonzini     return waited;
7953ba0e1a0SPaolo Bonzini }
7963ba0e1a0SPaolo Bonzini 
79718fbd0deSPaolo Bonzini bool bdrv_mark_request_serialising(BdrvTrackedRequest *req, uint64_t align)
79861007b31SStefan Hajnoczi {
7993ba0e1a0SPaolo Bonzini     BlockDriverState *bs = req->bs;
80061007b31SStefan Hajnoczi     int64_t overlap_offset = req->offset & ~(align - 1);
80122931a15SFam Zheng     uint64_t overlap_bytes = ROUND_UP(req->offset + req->bytes, align)
80261007b31SStefan Hajnoczi                                - overlap_offset;
8033ba0e1a0SPaolo Bonzini     bool waited;
80461007b31SStefan Hajnoczi 
8053ba0e1a0SPaolo Bonzini     qemu_co_mutex_lock(&bs->reqs_lock);
80661007b31SStefan Hajnoczi     if (!req->serialising) {
807d73415a3SStefan Hajnoczi         qatomic_inc(&req->bs->serialising_in_flight);
80861007b31SStefan Hajnoczi         req->serialising = true;
80961007b31SStefan Hajnoczi     }
81061007b31SStefan Hajnoczi 
81161007b31SStefan Hajnoczi     req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
81261007b31SStefan Hajnoczi     req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
8133ba0e1a0SPaolo Bonzini     waited = bdrv_wait_serialising_requests_locked(bs, req);
8143ba0e1a0SPaolo Bonzini     qemu_co_mutex_unlock(&bs->reqs_lock);
8153ba0e1a0SPaolo Bonzini     return waited;
81609d2f948SVladimir Sementsov-Ogievskiy }
81709d2f948SVladimir Sementsov-Ogievskiy 
81861007b31SStefan Hajnoczi /**
819c28107e9SMax Reitz  * Return the tracked request on @bs for the current coroutine, or
820c28107e9SMax Reitz  * NULL if there is none.
821c28107e9SMax Reitz  */
822c28107e9SMax Reitz BdrvTrackedRequest *coroutine_fn bdrv_co_get_self_request(BlockDriverState *bs)
823c28107e9SMax Reitz {
824c28107e9SMax Reitz     BdrvTrackedRequest *req;
825c28107e9SMax Reitz     Coroutine *self = qemu_coroutine_self();
826c28107e9SMax Reitz 
827c28107e9SMax Reitz     QLIST_FOREACH(req, &bs->tracked_requests, list) {
828c28107e9SMax Reitz         if (req->co == self) {
829c28107e9SMax Reitz             return req;
830c28107e9SMax Reitz         }
831c28107e9SMax Reitz     }
832c28107e9SMax Reitz 
833c28107e9SMax Reitz     return NULL;
834c28107e9SMax Reitz }
835c28107e9SMax Reitz 
836c28107e9SMax Reitz /**
837244483e6SKevin Wolf  * Round a region to cluster boundaries
838244483e6SKevin Wolf  */
839244483e6SKevin Wolf void bdrv_round_to_clusters(BlockDriverState *bs,
8407cfd5275SEric Blake                             int64_t offset, int64_t bytes,
841244483e6SKevin Wolf                             int64_t *cluster_offset,
8427cfd5275SEric Blake                             int64_t *cluster_bytes)
843244483e6SKevin Wolf {
844244483e6SKevin Wolf     BlockDriverInfo bdi;
845244483e6SKevin Wolf 
846244483e6SKevin Wolf     if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
847244483e6SKevin Wolf         *cluster_offset = offset;
848244483e6SKevin Wolf         *cluster_bytes = bytes;
849244483e6SKevin Wolf     } else {
850244483e6SKevin Wolf         int64_t c = bdi.cluster_size;
851244483e6SKevin Wolf         *cluster_offset = QEMU_ALIGN_DOWN(offset, c);
852244483e6SKevin Wolf         *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c);
853244483e6SKevin Wolf     }
854244483e6SKevin Wolf }
855244483e6SKevin Wolf 
85661007b31SStefan Hajnoczi static int bdrv_get_cluster_size(BlockDriverState *bs)
85761007b31SStefan Hajnoczi {
85861007b31SStefan Hajnoczi     BlockDriverInfo bdi;
85961007b31SStefan Hajnoczi     int ret;
86061007b31SStefan Hajnoczi 
86161007b31SStefan Hajnoczi     ret = bdrv_get_info(bs, &bdi);
86261007b31SStefan Hajnoczi     if (ret < 0 || bdi.cluster_size == 0) {
863a5b8dd2cSEric Blake         return bs->bl.request_alignment;
86461007b31SStefan Hajnoczi     } else {
86561007b31SStefan Hajnoczi         return bdi.cluster_size;
86661007b31SStefan Hajnoczi     }
86761007b31SStefan Hajnoczi }
86861007b31SStefan Hajnoczi 
86999723548SPaolo Bonzini void bdrv_inc_in_flight(BlockDriverState *bs)
87099723548SPaolo Bonzini {
871d73415a3SStefan Hajnoczi     qatomic_inc(&bs->in_flight);
87299723548SPaolo Bonzini }
87399723548SPaolo Bonzini 
874c9d1a561SPaolo Bonzini void bdrv_wakeup(BlockDriverState *bs)
875c9d1a561SPaolo Bonzini {
876cfe29d82SKevin Wolf     aio_wait_kick();
877c9d1a561SPaolo Bonzini }
878c9d1a561SPaolo Bonzini 
87999723548SPaolo Bonzini void bdrv_dec_in_flight(BlockDriverState *bs)
88099723548SPaolo Bonzini {
881d73415a3SStefan Hajnoczi     qatomic_dec(&bs->in_flight);
882c9d1a561SPaolo Bonzini     bdrv_wakeup(bs);
88399723548SPaolo Bonzini }
88499723548SPaolo Bonzini 
88518fbd0deSPaolo Bonzini static bool coroutine_fn bdrv_wait_serialising_requests(BdrvTrackedRequest *self)
88661007b31SStefan Hajnoczi {
88761007b31SStefan Hajnoczi     BlockDriverState *bs = self->bs;
88861007b31SStefan Hajnoczi     bool waited = false;
88961007b31SStefan Hajnoczi 
890d73415a3SStefan Hajnoczi     if (!qatomic_read(&bs->serialising_in_flight)) {
89161007b31SStefan Hajnoczi         return false;
89261007b31SStefan Hajnoczi     }
89361007b31SStefan Hajnoczi 
8943783fa3dSPaolo Bonzini     qemu_co_mutex_lock(&bs->reqs_lock);
8953ba0e1a0SPaolo Bonzini     waited = bdrv_wait_serialising_requests_locked(bs, self);
8963783fa3dSPaolo Bonzini     qemu_co_mutex_unlock(&bs->reqs_lock);
89761007b31SStefan Hajnoczi 
89861007b31SStefan Hajnoczi     return waited;
89961007b31SStefan Hajnoczi }
90061007b31SStefan Hajnoczi 
9018b117001SVladimir Sementsov-Ogievskiy int bdrv_check_request(int64_t offset, int64_t bytes)
90261007b31SStefan Hajnoczi {
9038b117001SVladimir Sementsov-Ogievskiy     if (offset < 0 || bytes < 0) {
90461007b31SStefan Hajnoczi         return -EIO;
90561007b31SStefan Hajnoczi     }
90661007b31SStefan Hajnoczi 
9078b117001SVladimir Sementsov-Ogievskiy     if (bytes > BDRV_MAX_LENGTH) {
9088b117001SVladimir Sementsov-Ogievskiy         return -EIO;
9098b117001SVladimir Sementsov-Ogievskiy     }
9108b117001SVladimir Sementsov-Ogievskiy 
9118b117001SVladimir Sementsov-Ogievskiy     if (offset > BDRV_MAX_LENGTH - bytes) {
9128b117001SVladimir Sementsov-Ogievskiy         return -EIO;
9138b117001SVladimir Sementsov-Ogievskiy     }
9148b117001SVladimir Sementsov-Ogievskiy 
9158b117001SVladimir Sementsov-Ogievskiy     return 0;
9168b117001SVladimir Sementsov-Ogievskiy }
9178b117001SVladimir Sementsov-Ogievskiy 
9188b117001SVladimir Sementsov-Ogievskiy static int bdrv_check_request32(int64_t offset, int64_t bytes)
9198b117001SVladimir Sementsov-Ogievskiy {
9208b117001SVladimir Sementsov-Ogievskiy     int ret = bdrv_check_request(offset, bytes);
9218b117001SVladimir Sementsov-Ogievskiy     if (ret < 0) {
9228b117001SVladimir Sementsov-Ogievskiy         return ret;
9238b117001SVladimir Sementsov-Ogievskiy     }
9248b117001SVladimir Sementsov-Ogievskiy 
9258b117001SVladimir Sementsov-Ogievskiy     if (bytes > BDRV_REQUEST_MAX_BYTES) {
92661007b31SStefan Hajnoczi         return -EIO;
92761007b31SStefan Hajnoczi     }
92861007b31SStefan Hajnoczi 
92961007b31SStefan Hajnoczi     return 0;
93061007b31SStefan Hajnoczi }
93161007b31SStefan Hajnoczi 
932720ff280SKevin Wolf int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset,
933f5a5ca79SManos Pitsidianakis                        int bytes, BdrvRequestFlags flags)
93461007b31SStefan Hajnoczi {
935fae2681aSVladimir Sementsov-Ogievskiy     return bdrv_pwritev(child, offset, bytes, NULL,
936fae2681aSVladimir Sementsov-Ogievskiy                         BDRV_REQ_ZERO_WRITE | flags);
93761007b31SStefan Hajnoczi }
93861007b31SStefan Hajnoczi 
93961007b31SStefan Hajnoczi /*
94074021bc4SEric Blake  * Completely zero out a block device with the help of bdrv_pwrite_zeroes.
94161007b31SStefan Hajnoczi  * The operation is sped up by checking the block status and only writing
94261007b31SStefan Hajnoczi  * zeroes to the device if they currently do not return zeroes. Optional
94374021bc4SEric Blake  * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
944465fe887SEric Blake  * BDRV_REQ_FUA).
94561007b31SStefan Hajnoczi  *
946f4649069SEric Blake  * Returns < 0 on error, 0 on success. For error codes see bdrv_pwrite().
94761007b31SStefan Hajnoczi  */
948720ff280SKevin Wolf int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags)
94961007b31SStefan Hajnoczi {
950237d78f8SEric Blake     int ret;
951237d78f8SEric Blake     int64_t target_size, bytes, offset = 0;
952720ff280SKevin Wolf     BlockDriverState *bs = child->bs;
95361007b31SStefan Hajnoczi 
9547286d610SEric Blake     target_size = bdrv_getlength(bs);
9557286d610SEric Blake     if (target_size < 0) {
9567286d610SEric Blake         return target_size;
95761007b31SStefan Hajnoczi     }
95861007b31SStefan Hajnoczi 
95961007b31SStefan Hajnoczi     for (;;) {
9607286d610SEric Blake         bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES);
9617286d610SEric Blake         if (bytes <= 0) {
96261007b31SStefan Hajnoczi             return 0;
96361007b31SStefan Hajnoczi         }
964237d78f8SEric Blake         ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL);
96561007b31SStefan Hajnoczi         if (ret < 0) {
96661007b31SStefan Hajnoczi             return ret;
96761007b31SStefan Hajnoczi         }
96861007b31SStefan Hajnoczi         if (ret & BDRV_BLOCK_ZERO) {
969237d78f8SEric Blake             offset += bytes;
97061007b31SStefan Hajnoczi             continue;
97161007b31SStefan Hajnoczi         }
972237d78f8SEric Blake         ret = bdrv_pwrite_zeroes(child, offset, bytes, flags);
97361007b31SStefan Hajnoczi         if (ret < 0) {
97461007b31SStefan Hajnoczi             return ret;
97561007b31SStefan Hajnoczi         }
976237d78f8SEric Blake         offset += bytes;
97761007b31SStefan Hajnoczi     }
97861007b31SStefan Hajnoczi }
97961007b31SStefan Hajnoczi 
9802e11d756SAlberto Garcia /* See bdrv_pwrite() for the return codes */
981cf2ab8fcSKevin Wolf int bdrv_pread(BdrvChild *child, int64_t offset, void *buf, int bytes)
98261007b31SStefan Hajnoczi {
983fae2681aSVladimir Sementsov-Ogievskiy     int ret;
9840d93ed08SVladimir Sementsov-Ogievskiy     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes);
98561007b31SStefan Hajnoczi 
98661007b31SStefan Hajnoczi     if (bytes < 0) {
98761007b31SStefan Hajnoczi         return -EINVAL;
98861007b31SStefan Hajnoczi     }
98961007b31SStefan Hajnoczi 
990fae2681aSVladimir Sementsov-Ogievskiy     ret = bdrv_preadv(child, offset, bytes, &qiov,  0);
99161007b31SStefan Hajnoczi 
992fae2681aSVladimir Sementsov-Ogievskiy     return ret < 0 ? ret : bytes;
99361007b31SStefan Hajnoczi }
99461007b31SStefan Hajnoczi 
9952e11d756SAlberto Garcia /* Return no. of bytes on success or < 0 on error. Important errors are:
9962e11d756SAlberto Garcia   -EIO         generic I/O error (may happen for all errors)
9972e11d756SAlberto Garcia   -ENOMEDIUM   No media inserted.
9982e11d756SAlberto Garcia   -EINVAL      Invalid offset or number of bytes
9992e11d756SAlberto Garcia   -EACCES      Trying to write a read-only device
10002e11d756SAlberto Garcia */
1001d9ca2ea2SKevin Wolf int bdrv_pwrite(BdrvChild *child, int64_t offset, const void *buf, int bytes)
100261007b31SStefan Hajnoczi {
1003fae2681aSVladimir Sementsov-Ogievskiy     int ret;
10040d93ed08SVladimir Sementsov-Ogievskiy     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes);
100561007b31SStefan Hajnoczi 
100661007b31SStefan Hajnoczi     if (bytes < 0) {
100761007b31SStefan Hajnoczi         return -EINVAL;
100861007b31SStefan Hajnoczi     }
100961007b31SStefan Hajnoczi 
1010fae2681aSVladimir Sementsov-Ogievskiy     ret = bdrv_pwritev(child, offset, bytes, &qiov, 0);
1011fae2681aSVladimir Sementsov-Ogievskiy 
1012fae2681aSVladimir Sementsov-Ogievskiy     return ret < 0 ? ret : bytes;
101361007b31SStefan Hajnoczi }
101461007b31SStefan Hajnoczi 
101561007b31SStefan Hajnoczi /*
101661007b31SStefan Hajnoczi  * Writes to the file and ensures that no writes are reordered across this
101761007b31SStefan Hajnoczi  * request (acts as a barrier)
101861007b31SStefan Hajnoczi  *
101961007b31SStefan Hajnoczi  * Returns 0 on success, -errno in error cases.
102061007b31SStefan Hajnoczi  */
1021d9ca2ea2SKevin Wolf int bdrv_pwrite_sync(BdrvChild *child, int64_t offset,
102261007b31SStefan Hajnoczi                      const void *buf, int count)
102361007b31SStefan Hajnoczi {
102461007b31SStefan Hajnoczi     int ret;
102561007b31SStefan Hajnoczi 
1026d9ca2ea2SKevin Wolf     ret = bdrv_pwrite(child, offset, buf, count);
102761007b31SStefan Hajnoczi     if (ret < 0) {
102861007b31SStefan Hajnoczi         return ret;
102961007b31SStefan Hajnoczi     }
103061007b31SStefan Hajnoczi 
1031d9ca2ea2SKevin Wolf     ret = bdrv_flush(child->bs);
1032855a6a93SKevin Wolf     if (ret < 0) {
1033855a6a93SKevin Wolf         return ret;
103461007b31SStefan Hajnoczi     }
103561007b31SStefan Hajnoczi 
103661007b31SStefan Hajnoczi     return 0;
103761007b31SStefan Hajnoczi }
103861007b31SStefan Hajnoczi 
103908844473SKevin Wolf typedef struct CoroutineIOCompletion {
104008844473SKevin Wolf     Coroutine *coroutine;
104108844473SKevin Wolf     int ret;
104208844473SKevin Wolf } CoroutineIOCompletion;
104308844473SKevin Wolf 
104408844473SKevin Wolf static void bdrv_co_io_em_complete(void *opaque, int ret)
104508844473SKevin Wolf {
104608844473SKevin Wolf     CoroutineIOCompletion *co = opaque;
104708844473SKevin Wolf 
104808844473SKevin Wolf     co->ret = ret;
1049b9e413ddSPaolo Bonzini     aio_co_wake(co->coroutine);
105008844473SKevin Wolf }
105108844473SKevin Wolf 
1052166fe960SKevin Wolf static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
1053166fe960SKevin Wolf                                            uint64_t offset, uint64_t bytes,
1054ac850bf0SVladimir Sementsov-Ogievskiy                                            QEMUIOVector *qiov,
1055ac850bf0SVladimir Sementsov-Ogievskiy                                            size_t qiov_offset, int flags)
1056166fe960SKevin Wolf {
1057166fe960SKevin Wolf     BlockDriver *drv = bs->drv;
10583fb06697SKevin Wolf     int64_t sector_num;
10593fb06697SKevin Wolf     unsigned int nb_sectors;
1060ac850bf0SVladimir Sementsov-Ogievskiy     QEMUIOVector local_qiov;
1061ac850bf0SVladimir Sementsov-Ogievskiy     int ret;
10623fb06697SKevin Wolf 
1063fa166538SEric Blake     assert(!(flags & ~BDRV_REQ_MASK));
1064fe0480d6SKevin Wolf     assert(!(flags & BDRV_REQ_NO_FALLBACK));
1065fa166538SEric Blake 
1066d470ad42SMax Reitz     if (!drv) {
1067d470ad42SMax Reitz         return -ENOMEDIUM;
1068d470ad42SMax Reitz     }
1069d470ad42SMax Reitz 
1070ac850bf0SVladimir Sementsov-Ogievskiy     if (drv->bdrv_co_preadv_part) {
1071ac850bf0SVladimir Sementsov-Ogievskiy         return drv->bdrv_co_preadv_part(bs, offset, bytes, qiov, qiov_offset,
1072ac850bf0SVladimir Sementsov-Ogievskiy                                         flags);
1073ac850bf0SVladimir Sementsov-Ogievskiy     }
1074ac850bf0SVladimir Sementsov-Ogievskiy 
1075ac850bf0SVladimir Sementsov-Ogievskiy     if (qiov_offset > 0 || bytes != qiov->size) {
1076ac850bf0SVladimir Sementsov-Ogievskiy         qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1077ac850bf0SVladimir Sementsov-Ogievskiy         qiov = &local_qiov;
1078ac850bf0SVladimir Sementsov-Ogievskiy     }
1079ac850bf0SVladimir Sementsov-Ogievskiy 
10803fb06697SKevin Wolf     if (drv->bdrv_co_preadv) {
1081ac850bf0SVladimir Sementsov-Ogievskiy         ret = drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
1082ac850bf0SVladimir Sementsov-Ogievskiy         goto out;
10833fb06697SKevin Wolf     }
10843fb06697SKevin Wolf 
1085edfab6a0SEric Blake     if (drv->bdrv_aio_preadv) {
108608844473SKevin Wolf         BlockAIOCB *acb;
108708844473SKevin Wolf         CoroutineIOCompletion co = {
108808844473SKevin Wolf             .coroutine = qemu_coroutine_self(),
108908844473SKevin Wolf         };
109008844473SKevin Wolf 
1091e31f6864SEric Blake         acb = drv->bdrv_aio_preadv(bs, offset, bytes, qiov, flags,
109208844473SKevin Wolf                                    bdrv_co_io_em_complete, &co);
109308844473SKevin Wolf         if (acb == NULL) {
1094ac850bf0SVladimir Sementsov-Ogievskiy             ret = -EIO;
1095ac850bf0SVladimir Sementsov-Ogievskiy             goto out;
109608844473SKevin Wolf         } else {
109708844473SKevin Wolf             qemu_coroutine_yield();
1098ac850bf0SVladimir Sementsov-Ogievskiy             ret = co.ret;
1099ac850bf0SVladimir Sementsov-Ogievskiy             goto out;
110008844473SKevin Wolf         }
110108844473SKevin Wolf     }
1102edfab6a0SEric Blake 
1103edfab6a0SEric Blake     sector_num = offset >> BDRV_SECTOR_BITS;
1104edfab6a0SEric Blake     nb_sectors = bytes >> BDRV_SECTOR_BITS;
1105edfab6a0SEric Blake 
11061bbbf32dSNir Soffer     assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
11071bbbf32dSNir Soffer     assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
110841ae31e3SAlberto Garcia     assert(bytes <= BDRV_REQUEST_MAX_BYTES);
1109edfab6a0SEric Blake     assert(drv->bdrv_co_readv);
1110edfab6a0SEric Blake 
1111ac850bf0SVladimir Sementsov-Ogievskiy     ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
1112ac850bf0SVladimir Sementsov-Ogievskiy 
1113ac850bf0SVladimir Sementsov-Ogievskiy out:
1114ac850bf0SVladimir Sementsov-Ogievskiy     if (qiov == &local_qiov) {
1115ac850bf0SVladimir Sementsov-Ogievskiy         qemu_iovec_destroy(&local_qiov);
1116ac850bf0SVladimir Sementsov-Ogievskiy     }
1117ac850bf0SVladimir Sementsov-Ogievskiy 
1118ac850bf0SVladimir Sementsov-Ogievskiy     return ret;
1119166fe960SKevin Wolf }
1120166fe960SKevin Wolf 
112178a07294SKevin Wolf static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
112278a07294SKevin Wolf                                             uint64_t offset, uint64_t bytes,
1123ac850bf0SVladimir Sementsov-Ogievskiy                                             QEMUIOVector *qiov,
1124ac850bf0SVladimir Sementsov-Ogievskiy                                             size_t qiov_offset, int flags)
112578a07294SKevin Wolf {
112678a07294SKevin Wolf     BlockDriver *drv = bs->drv;
11273fb06697SKevin Wolf     int64_t sector_num;
11283fb06697SKevin Wolf     unsigned int nb_sectors;
1129ac850bf0SVladimir Sementsov-Ogievskiy     QEMUIOVector local_qiov;
113078a07294SKevin Wolf     int ret;
113178a07294SKevin Wolf 
1132fa166538SEric Blake     assert(!(flags & ~BDRV_REQ_MASK));
1133fe0480d6SKevin Wolf     assert(!(flags & BDRV_REQ_NO_FALLBACK));
1134fa166538SEric Blake 
1135d470ad42SMax Reitz     if (!drv) {
1136d470ad42SMax Reitz         return -ENOMEDIUM;
1137d470ad42SMax Reitz     }
1138d470ad42SMax Reitz 
1139ac850bf0SVladimir Sementsov-Ogievskiy     if (drv->bdrv_co_pwritev_part) {
1140ac850bf0SVladimir Sementsov-Ogievskiy         ret = drv->bdrv_co_pwritev_part(bs, offset, bytes, qiov, qiov_offset,
1141ac850bf0SVladimir Sementsov-Ogievskiy                                         flags & bs->supported_write_flags);
1142ac850bf0SVladimir Sementsov-Ogievskiy         flags &= ~bs->supported_write_flags;
1143ac850bf0SVladimir Sementsov-Ogievskiy         goto emulate_flags;
1144ac850bf0SVladimir Sementsov-Ogievskiy     }
1145ac850bf0SVladimir Sementsov-Ogievskiy 
1146ac850bf0SVladimir Sementsov-Ogievskiy     if (qiov_offset > 0 || bytes != qiov->size) {
1147ac850bf0SVladimir Sementsov-Ogievskiy         qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1148ac850bf0SVladimir Sementsov-Ogievskiy         qiov = &local_qiov;
1149ac850bf0SVladimir Sementsov-Ogievskiy     }
1150ac850bf0SVladimir Sementsov-Ogievskiy 
11513fb06697SKevin Wolf     if (drv->bdrv_co_pwritev) {
1152515c2f43SKevin Wolf         ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov,
1153515c2f43SKevin Wolf                                    flags & bs->supported_write_flags);
1154515c2f43SKevin Wolf         flags &= ~bs->supported_write_flags;
11553fb06697SKevin Wolf         goto emulate_flags;
11563fb06697SKevin Wolf     }
11573fb06697SKevin Wolf 
1158edfab6a0SEric Blake     if (drv->bdrv_aio_pwritev) {
115908844473SKevin Wolf         BlockAIOCB *acb;
116008844473SKevin Wolf         CoroutineIOCompletion co = {
116108844473SKevin Wolf             .coroutine = qemu_coroutine_self(),
116208844473SKevin Wolf         };
116308844473SKevin Wolf 
1164e31f6864SEric Blake         acb = drv->bdrv_aio_pwritev(bs, offset, bytes, qiov,
1165e31f6864SEric Blake                                     flags & bs->supported_write_flags,
116608844473SKevin Wolf                                     bdrv_co_io_em_complete, &co);
1167e31f6864SEric Blake         flags &= ~bs->supported_write_flags;
116808844473SKevin Wolf         if (acb == NULL) {
11693fb06697SKevin Wolf             ret = -EIO;
117008844473SKevin Wolf         } else {
117108844473SKevin Wolf             qemu_coroutine_yield();
11723fb06697SKevin Wolf             ret = co.ret;
117308844473SKevin Wolf         }
1174edfab6a0SEric Blake         goto emulate_flags;
1175edfab6a0SEric Blake     }
1176edfab6a0SEric Blake 
1177edfab6a0SEric Blake     sector_num = offset >> BDRV_SECTOR_BITS;
1178edfab6a0SEric Blake     nb_sectors = bytes >> BDRV_SECTOR_BITS;
1179edfab6a0SEric Blake 
11801bbbf32dSNir Soffer     assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
11811bbbf32dSNir Soffer     assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
118241ae31e3SAlberto Garcia     assert(bytes <= BDRV_REQUEST_MAX_BYTES);
1183edfab6a0SEric Blake 
1184e18a58b4SEric Blake     assert(drv->bdrv_co_writev);
1185e18a58b4SEric Blake     ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov,
1186edfab6a0SEric Blake                               flags & bs->supported_write_flags);
1187edfab6a0SEric Blake     flags &= ~bs->supported_write_flags;
118878a07294SKevin Wolf 
11893fb06697SKevin Wolf emulate_flags:
11904df863f3SEric Blake     if (ret == 0 && (flags & BDRV_REQ_FUA)) {
119178a07294SKevin Wolf         ret = bdrv_co_flush(bs);
119278a07294SKevin Wolf     }
119378a07294SKevin Wolf 
1194ac850bf0SVladimir Sementsov-Ogievskiy     if (qiov == &local_qiov) {
1195ac850bf0SVladimir Sementsov-Ogievskiy         qemu_iovec_destroy(&local_qiov);
1196ac850bf0SVladimir Sementsov-Ogievskiy     }
1197ac850bf0SVladimir Sementsov-Ogievskiy 
119878a07294SKevin Wolf     return ret;
119978a07294SKevin Wolf }
120078a07294SKevin Wolf 
120129a298afSPavel Butsykin static int coroutine_fn
120229a298afSPavel Butsykin bdrv_driver_pwritev_compressed(BlockDriverState *bs, uint64_t offset,
1203ac850bf0SVladimir Sementsov-Ogievskiy                                uint64_t bytes, QEMUIOVector *qiov,
1204ac850bf0SVladimir Sementsov-Ogievskiy                                size_t qiov_offset)
120529a298afSPavel Butsykin {
120629a298afSPavel Butsykin     BlockDriver *drv = bs->drv;
1207ac850bf0SVladimir Sementsov-Ogievskiy     QEMUIOVector local_qiov;
1208ac850bf0SVladimir Sementsov-Ogievskiy     int ret;
120929a298afSPavel Butsykin 
1210d470ad42SMax Reitz     if (!drv) {
1211d470ad42SMax Reitz         return -ENOMEDIUM;
1212d470ad42SMax Reitz     }
1213d470ad42SMax Reitz 
1214ac850bf0SVladimir Sementsov-Ogievskiy     if (!block_driver_can_compress(drv)) {
121529a298afSPavel Butsykin         return -ENOTSUP;
121629a298afSPavel Butsykin     }
121729a298afSPavel Butsykin 
1218ac850bf0SVladimir Sementsov-Ogievskiy     if (drv->bdrv_co_pwritev_compressed_part) {
1219ac850bf0SVladimir Sementsov-Ogievskiy         return drv->bdrv_co_pwritev_compressed_part(bs, offset, bytes,
1220ac850bf0SVladimir Sementsov-Ogievskiy                                                     qiov, qiov_offset);
1221ac850bf0SVladimir Sementsov-Ogievskiy     }
1222ac850bf0SVladimir Sementsov-Ogievskiy 
1223ac850bf0SVladimir Sementsov-Ogievskiy     if (qiov_offset == 0) {
122429a298afSPavel Butsykin         return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov);
122529a298afSPavel Butsykin     }
122629a298afSPavel Butsykin 
1227ac850bf0SVladimir Sementsov-Ogievskiy     qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1228ac850bf0SVladimir Sementsov-Ogievskiy     ret = drv->bdrv_co_pwritev_compressed(bs, offset, bytes, &local_qiov);
1229ac850bf0SVladimir Sementsov-Ogievskiy     qemu_iovec_destroy(&local_qiov);
1230ac850bf0SVladimir Sementsov-Ogievskiy 
1231ac850bf0SVladimir Sementsov-Ogievskiy     return ret;
1232ac850bf0SVladimir Sementsov-Ogievskiy }
1233ac850bf0SVladimir Sementsov-Ogievskiy 
123485c97ca7SKevin Wolf static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child,
12353299e5ecSVladimir Sementsov-Ogievskiy         int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
12361143ec5eSVladimir Sementsov-Ogievskiy         size_t qiov_offset, int flags)
123761007b31SStefan Hajnoczi {
123885c97ca7SKevin Wolf     BlockDriverState *bs = child->bs;
123985c97ca7SKevin Wolf 
124061007b31SStefan Hajnoczi     /* Perform I/O through a temporary buffer so that users who scribble over
124161007b31SStefan Hajnoczi      * their read buffer while the operation is in progress do not end up
124261007b31SStefan Hajnoczi      * modifying the image file.  This is critical for zero-copy guest I/O
124361007b31SStefan Hajnoczi      * where anything might happen inside guest memory.
124461007b31SStefan Hajnoczi      */
12452275cc90SVladimir Sementsov-Ogievskiy     void *bounce_buffer = NULL;
124661007b31SStefan Hajnoczi 
124761007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
1248244483e6SKevin Wolf     int64_t cluster_offset;
12497cfd5275SEric Blake     int64_t cluster_bytes;
125061007b31SStefan Hajnoczi     size_t skip_bytes;
125161007b31SStefan Hajnoczi     int ret;
1252cb2e2878SEric Blake     int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer,
1253cb2e2878SEric Blake                                     BDRV_REQUEST_MAX_BYTES);
1254cb2e2878SEric Blake     unsigned int progress = 0;
12558644476eSMax Reitz     bool skip_write;
125661007b31SStefan Hajnoczi 
1257d470ad42SMax Reitz     if (!drv) {
1258d470ad42SMax Reitz         return -ENOMEDIUM;
1259d470ad42SMax Reitz     }
1260d470ad42SMax Reitz 
12618644476eSMax Reitz     /*
12628644476eSMax Reitz      * Do not write anything when the BDS is inactive.  That is not
12638644476eSMax Reitz      * allowed, and it would not help.
12648644476eSMax Reitz      */
12658644476eSMax Reitz     skip_write = (bs->open_flags & BDRV_O_INACTIVE);
12668644476eSMax Reitz 
12671bf03e66SKevin Wolf     /* FIXME We cannot require callers to have write permissions when all they
12681bf03e66SKevin Wolf      * are doing is a read request. If we did things right, write permissions
12691bf03e66SKevin Wolf      * would be obtained anyway, but internally by the copy-on-read code. As
1270765d9df9SEric Blake      * long as it is implemented here rather than in a separate filter driver,
12711bf03e66SKevin Wolf      * the copy-on-read code doesn't have its own BdrvChild, however, for which
12721bf03e66SKevin Wolf      * it could request permissions. Therefore we have to bypass the permission
12731bf03e66SKevin Wolf      * system for the moment. */
12741bf03e66SKevin Wolf     // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
1275afa4b293SKevin Wolf 
127661007b31SStefan Hajnoczi     /* Cover entire cluster so no additional backing file I/O is required when
1277cb2e2878SEric Blake      * allocating cluster in the image file.  Note that this value may exceed
1278cb2e2878SEric Blake      * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which
1279cb2e2878SEric Blake      * is one reason we loop rather than doing it all at once.
128061007b31SStefan Hajnoczi      */
1281244483e6SKevin Wolf     bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes);
1282cb2e2878SEric Blake     skip_bytes = offset - cluster_offset;
128361007b31SStefan Hajnoczi 
1284244483e6SKevin Wolf     trace_bdrv_co_do_copy_on_readv(bs, offset, bytes,
1285244483e6SKevin Wolf                                    cluster_offset, cluster_bytes);
128661007b31SStefan Hajnoczi 
1287cb2e2878SEric Blake     while (cluster_bytes) {
1288cb2e2878SEric Blake         int64_t pnum;
128961007b31SStefan Hajnoczi 
12908644476eSMax Reitz         if (skip_write) {
12918644476eSMax Reitz             ret = 1; /* "already allocated", so nothing will be copied */
12928644476eSMax Reitz             pnum = MIN(cluster_bytes, max_transfer);
12938644476eSMax Reitz         } else {
1294cb2e2878SEric Blake             ret = bdrv_is_allocated(bs, cluster_offset,
1295cb2e2878SEric Blake                                     MIN(cluster_bytes, max_transfer), &pnum);
1296cb2e2878SEric Blake             if (ret < 0) {
12978644476eSMax Reitz                 /*
12988644476eSMax Reitz                  * Safe to treat errors in querying allocation as if
1299cb2e2878SEric Blake                  * unallocated; we'll probably fail again soon on the
1300cb2e2878SEric Blake                  * read, but at least that will set a decent errno.
1301cb2e2878SEric Blake                  */
1302cb2e2878SEric Blake                 pnum = MIN(cluster_bytes, max_transfer);
1303cb2e2878SEric Blake             }
1304cb2e2878SEric Blake 
1305b0ddcbbbSKevin Wolf             /* Stop at EOF if the image ends in the middle of the cluster */
1306b0ddcbbbSKevin Wolf             if (ret == 0 && pnum == 0) {
1307b0ddcbbbSKevin Wolf                 assert(progress >= bytes);
1308b0ddcbbbSKevin Wolf                 break;
1309b0ddcbbbSKevin Wolf             }
1310b0ddcbbbSKevin Wolf 
1311cb2e2878SEric Blake             assert(skip_bytes < pnum);
13128644476eSMax Reitz         }
1313cb2e2878SEric Blake 
1314cb2e2878SEric Blake         if (ret <= 0) {
13151143ec5eSVladimir Sementsov-Ogievskiy             QEMUIOVector local_qiov;
13161143ec5eSVladimir Sementsov-Ogievskiy 
1317cb2e2878SEric Blake             /* Must copy-on-read; use the bounce buffer */
13180d93ed08SVladimir Sementsov-Ogievskiy             pnum = MIN(pnum, MAX_BOUNCE_BUFFER);
13192275cc90SVladimir Sementsov-Ogievskiy             if (!bounce_buffer) {
13202275cc90SVladimir Sementsov-Ogievskiy                 int64_t max_we_need = MAX(pnum, cluster_bytes - pnum);
13212275cc90SVladimir Sementsov-Ogievskiy                 int64_t max_allowed = MIN(max_transfer, MAX_BOUNCE_BUFFER);
13222275cc90SVladimir Sementsov-Ogievskiy                 int64_t bounce_buffer_len = MIN(max_we_need, max_allowed);
13232275cc90SVladimir Sementsov-Ogievskiy 
13242275cc90SVladimir Sementsov-Ogievskiy                 bounce_buffer = qemu_try_blockalign(bs, bounce_buffer_len);
13252275cc90SVladimir Sementsov-Ogievskiy                 if (!bounce_buffer) {
13262275cc90SVladimir Sementsov-Ogievskiy                     ret = -ENOMEM;
13272275cc90SVladimir Sementsov-Ogievskiy                     goto err;
13282275cc90SVladimir Sementsov-Ogievskiy                 }
13292275cc90SVladimir Sementsov-Ogievskiy             }
13300d93ed08SVladimir Sementsov-Ogievskiy             qemu_iovec_init_buf(&local_qiov, bounce_buffer, pnum);
1331cb2e2878SEric Blake 
1332cb2e2878SEric Blake             ret = bdrv_driver_preadv(bs, cluster_offset, pnum,
1333ac850bf0SVladimir Sementsov-Ogievskiy                                      &local_qiov, 0, 0);
133461007b31SStefan Hajnoczi             if (ret < 0) {
133561007b31SStefan Hajnoczi                 goto err;
133661007b31SStefan Hajnoczi             }
133761007b31SStefan Hajnoczi 
1338d855ebcdSEric Blake             bdrv_debug_event(bs, BLKDBG_COR_WRITE);
1339c1499a5eSEric Blake             if (drv->bdrv_co_pwrite_zeroes &&
1340cb2e2878SEric Blake                 buffer_is_zero(bounce_buffer, pnum)) {
1341a604fa2bSEric Blake                 /* FIXME: Should we (perhaps conditionally) be setting
1342a604fa2bSEric Blake                  * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy
1343a604fa2bSEric Blake                  * that still correctly reads as zero? */
13447adcf59fSMax Reitz                 ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, pnum,
13457adcf59fSMax Reitz                                                BDRV_REQ_WRITE_UNCHANGED);
134661007b31SStefan Hajnoczi             } else {
1347cb2e2878SEric Blake                 /* This does not change the data on the disk, it is not
1348cb2e2878SEric Blake                  * necessary to flush even in cache=writethrough mode.
134961007b31SStefan Hajnoczi                  */
1350cb2e2878SEric Blake                 ret = bdrv_driver_pwritev(bs, cluster_offset, pnum,
1351ac850bf0SVladimir Sementsov-Ogievskiy                                           &local_qiov, 0,
13527adcf59fSMax Reitz                                           BDRV_REQ_WRITE_UNCHANGED);
135361007b31SStefan Hajnoczi             }
135461007b31SStefan Hajnoczi 
135561007b31SStefan Hajnoczi             if (ret < 0) {
1356cb2e2878SEric Blake                 /* It might be okay to ignore write errors for guest
1357cb2e2878SEric Blake                  * requests.  If this is a deliberate copy-on-read
1358cb2e2878SEric Blake                  * then we don't want to ignore the error.  Simply
1359cb2e2878SEric Blake                  * report it in all cases.
136061007b31SStefan Hajnoczi                  */
136161007b31SStefan Hajnoczi                 goto err;
136261007b31SStefan Hajnoczi             }
136361007b31SStefan Hajnoczi 
13643299e5ecSVladimir Sementsov-Ogievskiy             if (!(flags & BDRV_REQ_PREFETCH)) {
13651143ec5eSVladimir Sementsov-Ogievskiy                 qemu_iovec_from_buf(qiov, qiov_offset + progress,
13661143ec5eSVladimir Sementsov-Ogievskiy                                     bounce_buffer + skip_bytes,
13674ab78b19SVladimir Sementsov-Ogievskiy                                     MIN(pnum - skip_bytes, bytes - progress));
13683299e5ecSVladimir Sementsov-Ogievskiy             }
13693299e5ecSVladimir Sementsov-Ogievskiy         } else if (!(flags & BDRV_REQ_PREFETCH)) {
1370cb2e2878SEric Blake             /* Read directly into the destination */
13711143ec5eSVladimir Sementsov-Ogievskiy             ret = bdrv_driver_preadv(bs, offset + progress,
13721143ec5eSVladimir Sementsov-Ogievskiy                                      MIN(pnum - skip_bytes, bytes - progress),
13731143ec5eSVladimir Sementsov-Ogievskiy                                      qiov, qiov_offset + progress, 0);
1374cb2e2878SEric Blake             if (ret < 0) {
1375cb2e2878SEric Blake                 goto err;
1376cb2e2878SEric Blake             }
1377cb2e2878SEric Blake         }
1378cb2e2878SEric Blake 
1379cb2e2878SEric Blake         cluster_offset += pnum;
1380cb2e2878SEric Blake         cluster_bytes -= pnum;
1381cb2e2878SEric Blake         progress += pnum - skip_bytes;
1382cb2e2878SEric Blake         skip_bytes = 0;
1383cb2e2878SEric Blake     }
1384cb2e2878SEric Blake     ret = 0;
138561007b31SStefan Hajnoczi 
138661007b31SStefan Hajnoczi err:
138761007b31SStefan Hajnoczi     qemu_vfree(bounce_buffer);
138861007b31SStefan Hajnoczi     return ret;
138961007b31SStefan Hajnoczi }
139061007b31SStefan Hajnoczi 
139161007b31SStefan Hajnoczi /*
139261007b31SStefan Hajnoczi  * Forwards an already correctly aligned request to the BlockDriver. This
13931a62d0acSEric Blake  * handles copy on read, zeroing after EOF, and fragmentation of large
13941a62d0acSEric Blake  * reads; any other features must be implemented by the caller.
139561007b31SStefan Hajnoczi  */
139685c97ca7SKevin Wolf static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,
139761007b31SStefan Hajnoczi     BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
139865cd4424SVladimir Sementsov-Ogievskiy     int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags)
139961007b31SStefan Hajnoczi {
140085c97ca7SKevin Wolf     BlockDriverState *bs = child->bs;
1401c9d20029SKevin Wolf     int64_t total_bytes, max_bytes;
14021a62d0acSEric Blake     int ret = 0;
14031a62d0acSEric Blake     uint64_t bytes_remaining = bytes;
14041a62d0acSEric Blake     int max_transfer;
140561007b31SStefan Hajnoczi 
140649c07526SKevin Wolf     assert(is_power_of_2(align));
140749c07526SKevin Wolf     assert((offset & (align - 1)) == 0);
140849c07526SKevin Wolf     assert((bytes & (align - 1)) == 0);
1409abb06c5aSDaniel P. Berrange     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
14101a62d0acSEric Blake     max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
14111a62d0acSEric Blake                                    align);
1412a604fa2bSEric Blake 
1413a604fa2bSEric Blake     /* TODO: We would need a per-BDS .supported_read_flags and
1414a604fa2bSEric Blake      * potential fallback support, if we ever implement any read flags
1415a604fa2bSEric Blake      * to pass through to drivers.  For now, there aren't any
1416a604fa2bSEric Blake      * passthrough flags.  */
1417c53cb427SPaolo Bonzini     assert(!(flags & ~(BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH)));
141861007b31SStefan Hajnoczi 
141961007b31SStefan Hajnoczi     /* Handle Copy on Read and associated serialisation */
142061007b31SStefan Hajnoczi     if (flags & BDRV_REQ_COPY_ON_READ) {
142161007b31SStefan Hajnoczi         /* If we touch the same cluster it counts as an overlap.  This
142261007b31SStefan Hajnoczi          * guarantees that allocating writes will be serialized and not race
142361007b31SStefan Hajnoczi          * with each other for the same cluster.  For example, in copy-on-read
142461007b31SStefan Hajnoczi          * it ensures that the CoR read and write operations are atomic and
142561007b31SStefan Hajnoczi          * guest writes cannot interleave between them. */
1426304d9d7fSMax Reitz         bdrv_mark_request_serialising(req, bdrv_get_cluster_size(bs));
142718fbd0deSPaolo Bonzini     } else {
1428304d9d7fSMax Reitz         bdrv_wait_serialising_requests(req);
142918fbd0deSPaolo Bonzini     }
143061007b31SStefan Hajnoczi 
143161007b31SStefan Hajnoczi     if (flags & BDRV_REQ_COPY_ON_READ) {
1432d6a644bbSEric Blake         int64_t pnum;
143361007b31SStefan Hajnoczi 
143488e63df2SEric Blake         ret = bdrv_is_allocated(bs, offset, bytes, &pnum);
143561007b31SStefan Hajnoczi         if (ret < 0) {
143661007b31SStefan Hajnoczi             goto out;
143761007b31SStefan Hajnoczi         }
143861007b31SStefan Hajnoczi 
143988e63df2SEric Blake         if (!ret || pnum != bytes) {
144065cd4424SVladimir Sementsov-Ogievskiy             ret = bdrv_co_do_copy_on_readv(child, offset, bytes,
144165cd4424SVladimir Sementsov-Ogievskiy                                            qiov, qiov_offset, flags);
14423299e5ecSVladimir Sementsov-Ogievskiy             goto out;
14433299e5ecSVladimir Sementsov-Ogievskiy         } else if (flags & BDRV_REQ_PREFETCH) {
144461007b31SStefan Hajnoczi             goto out;
144561007b31SStefan Hajnoczi         }
144661007b31SStefan Hajnoczi     }
144761007b31SStefan Hajnoczi 
14481a62d0acSEric Blake     /* Forward the request to the BlockDriver, possibly fragmenting it */
144949c07526SKevin Wolf     total_bytes = bdrv_getlength(bs);
145049c07526SKevin Wolf     if (total_bytes < 0) {
145149c07526SKevin Wolf         ret = total_bytes;
145261007b31SStefan Hajnoczi         goto out;
145361007b31SStefan Hajnoczi     }
145461007b31SStefan Hajnoczi 
145549c07526SKevin Wolf     max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align);
14561a62d0acSEric Blake     if (bytes <= max_bytes && bytes <= max_transfer) {
145765cd4424SVladimir Sementsov-Ogievskiy         ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, 0);
14581a62d0acSEric Blake         goto out;
145961007b31SStefan Hajnoczi     }
146061007b31SStefan Hajnoczi 
14611a62d0acSEric Blake     while (bytes_remaining) {
14621a62d0acSEric Blake         int num;
14631a62d0acSEric Blake 
14641a62d0acSEric Blake         if (max_bytes) {
14651a62d0acSEric Blake             num = MIN(bytes_remaining, MIN(max_bytes, max_transfer));
14661a62d0acSEric Blake             assert(num);
14671a62d0acSEric Blake 
14681a62d0acSEric Blake             ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining,
1469134b7decSMax Reitz                                      num, qiov,
1470134b7decSMax Reitz                                      qiov_offset + bytes - bytes_remaining, 0);
14711a62d0acSEric Blake             max_bytes -= num;
14721a62d0acSEric Blake         } else {
14731a62d0acSEric Blake             num = bytes_remaining;
1474134b7decSMax Reitz             ret = qemu_iovec_memset(qiov, qiov_offset + bytes - bytes_remaining,
1475134b7decSMax Reitz                                     0, bytes_remaining);
14761a62d0acSEric Blake         }
14771a62d0acSEric Blake         if (ret < 0) {
14781a62d0acSEric Blake             goto out;
14791a62d0acSEric Blake         }
14801a62d0acSEric Blake         bytes_remaining -= num;
148161007b31SStefan Hajnoczi     }
148261007b31SStefan Hajnoczi 
148361007b31SStefan Hajnoczi out:
14841a62d0acSEric Blake     return ret < 0 ? ret : 0;
148561007b31SStefan Hajnoczi }
148661007b31SStefan Hajnoczi 
148761007b31SStefan Hajnoczi /*
14887a3f542fSVladimir Sementsov-Ogievskiy  * Request padding
14897a3f542fSVladimir Sementsov-Ogievskiy  *
14907a3f542fSVladimir Sementsov-Ogievskiy  *  |<---- align ----->|                     |<----- align ---->|
14917a3f542fSVladimir Sementsov-Ogievskiy  *  |<- head ->|<------------- bytes ------------->|<-- tail -->|
14927a3f542fSVladimir Sementsov-Ogievskiy  *  |          |       |                     |     |            |
14937a3f542fSVladimir Sementsov-Ogievskiy  * -*----------$-------*-------- ... --------*-----$------------*---
14947a3f542fSVladimir Sementsov-Ogievskiy  *  |          |       |                     |     |            |
14957a3f542fSVladimir Sementsov-Ogievskiy  *  |          offset  |                     |     end          |
14967a3f542fSVladimir Sementsov-Ogievskiy  *  ALIGN_DOWN(offset) ALIGN_UP(offset)      ALIGN_DOWN(end)   ALIGN_UP(end)
14977a3f542fSVladimir Sementsov-Ogievskiy  *  [buf   ... )                             [tail_buf          )
14987a3f542fSVladimir Sementsov-Ogievskiy  *
14997a3f542fSVladimir Sementsov-Ogievskiy  * @buf is an aligned allocation needed to store @head and @tail paddings. @head
15007a3f542fSVladimir Sementsov-Ogievskiy  * is placed at the beginning of @buf and @tail at the @end.
15017a3f542fSVladimir Sementsov-Ogievskiy  *
15027a3f542fSVladimir Sementsov-Ogievskiy  * @tail_buf is a pointer to sub-buffer, corresponding to align-sized chunk
15037a3f542fSVladimir Sementsov-Ogievskiy  * around tail, if tail exists.
15047a3f542fSVladimir Sementsov-Ogievskiy  *
15057a3f542fSVladimir Sementsov-Ogievskiy  * @merge_reads is true for small requests,
15067a3f542fSVladimir Sementsov-Ogievskiy  * if @buf_len == @head + bytes + @tail. In this case it is possible that both
15077a3f542fSVladimir Sementsov-Ogievskiy  * head and tail exist but @buf_len == align and @tail_buf == @buf.
150861007b31SStefan Hajnoczi  */
15097a3f542fSVladimir Sementsov-Ogievskiy typedef struct BdrvRequestPadding {
15107a3f542fSVladimir Sementsov-Ogievskiy     uint8_t *buf;
15117a3f542fSVladimir Sementsov-Ogievskiy     size_t buf_len;
15127a3f542fSVladimir Sementsov-Ogievskiy     uint8_t *tail_buf;
15137a3f542fSVladimir Sementsov-Ogievskiy     size_t head;
15147a3f542fSVladimir Sementsov-Ogievskiy     size_t tail;
15157a3f542fSVladimir Sementsov-Ogievskiy     bool merge_reads;
15167a3f542fSVladimir Sementsov-Ogievskiy     QEMUIOVector local_qiov;
15177a3f542fSVladimir Sementsov-Ogievskiy } BdrvRequestPadding;
15187a3f542fSVladimir Sementsov-Ogievskiy 
15197a3f542fSVladimir Sementsov-Ogievskiy static bool bdrv_init_padding(BlockDriverState *bs,
15207a3f542fSVladimir Sementsov-Ogievskiy                               int64_t offset, int64_t bytes,
15217a3f542fSVladimir Sementsov-Ogievskiy                               BdrvRequestPadding *pad)
15227a3f542fSVladimir Sementsov-Ogievskiy {
15237a3f542fSVladimir Sementsov-Ogievskiy     uint64_t align = bs->bl.request_alignment;
15247a3f542fSVladimir Sementsov-Ogievskiy     size_t sum;
15257a3f542fSVladimir Sementsov-Ogievskiy 
15267a3f542fSVladimir Sementsov-Ogievskiy     memset(pad, 0, sizeof(*pad));
15277a3f542fSVladimir Sementsov-Ogievskiy 
15287a3f542fSVladimir Sementsov-Ogievskiy     pad->head = offset & (align - 1);
15297a3f542fSVladimir Sementsov-Ogievskiy     pad->tail = ((offset + bytes) & (align - 1));
15307a3f542fSVladimir Sementsov-Ogievskiy     if (pad->tail) {
15317a3f542fSVladimir Sementsov-Ogievskiy         pad->tail = align - pad->tail;
15327a3f542fSVladimir Sementsov-Ogievskiy     }
15337a3f542fSVladimir Sementsov-Ogievskiy 
1534ac9d00bfSVladimir Sementsov-Ogievskiy     if (!pad->head && !pad->tail) {
15357a3f542fSVladimir Sementsov-Ogievskiy         return false;
15367a3f542fSVladimir Sementsov-Ogievskiy     }
15377a3f542fSVladimir Sementsov-Ogievskiy 
1538ac9d00bfSVladimir Sementsov-Ogievskiy     assert(bytes); /* Nothing good in aligning zero-length requests */
1539ac9d00bfSVladimir Sementsov-Ogievskiy 
15407a3f542fSVladimir Sementsov-Ogievskiy     sum = pad->head + bytes + pad->tail;
15417a3f542fSVladimir Sementsov-Ogievskiy     pad->buf_len = (sum > align && pad->head && pad->tail) ? 2 * align : align;
15427a3f542fSVladimir Sementsov-Ogievskiy     pad->buf = qemu_blockalign(bs, pad->buf_len);
15437a3f542fSVladimir Sementsov-Ogievskiy     pad->merge_reads = sum == pad->buf_len;
15447a3f542fSVladimir Sementsov-Ogievskiy     if (pad->tail) {
15457a3f542fSVladimir Sementsov-Ogievskiy         pad->tail_buf = pad->buf + pad->buf_len - align;
15467a3f542fSVladimir Sementsov-Ogievskiy     }
15477a3f542fSVladimir Sementsov-Ogievskiy 
15487a3f542fSVladimir Sementsov-Ogievskiy     return true;
15497a3f542fSVladimir Sementsov-Ogievskiy }
15507a3f542fSVladimir Sementsov-Ogievskiy 
15517a3f542fSVladimir Sementsov-Ogievskiy static int bdrv_padding_rmw_read(BdrvChild *child,
15527a3f542fSVladimir Sementsov-Ogievskiy                                  BdrvTrackedRequest *req,
15537a3f542fSVladimir Sementsov-Ogievskiy                                  BdrvRequestPadding *pad,
15547a3f542fSVladimir Sementsov-Ogievskiy                                  bool zero_middle)
15557a3f542fSVladimir Sementsov-Ogievskiy {
15567a3f542fSVladimir Sementsov-Ogievskiy     QEMUIOVector local_qiov;
15577a3f542fSVladimir Sementsov-Ogievskiy     BlockDriverState *bs = child->bs;
15587a3f542fSVladimir Sementsov-Ogievskiy     uint64_t align = bs->bl.request_alignment;
15597a3f542fSVladimir Sementsov-Ogievskiy     int ret;
15607a3f542fSVladimir Sementsov-Ogievskiy 
15617a3f542fSVladimir Sementsov-Ogievskiy     assert(req->serialising && pad->buf);
15627a3f542fSVladimir Sementsov-Ogievskiy 
15637a3f542fSVladimir Sementsov-Ogievskiy     if (pad->head || pad->merge_reads) {
15647a3f542fSVladimir Sementsov-Ogievskiy         uint64_t bytes = pad->merge_reads ? pad->buf_len : align;
15657a3f542fSVladimir Sementsov-Ogievskiy 
15667a3f542fSVladimir Sementsov-Ogievskiy         qemu_iovec_init_buf(&local_qiov, pad->buf, bytes);
15677a3f542fSVladimir Sementsov-Ogievskiy 
15687a3f542fSVladimir Sementsov-Ogievskiy         if (pad->head) {
15697a3f542fSVladimir Sementsov-Ogievskiy             bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
15707a3f542fSVladimir Sementsov-Ogievskiy         }
15717a3f542fSVladimir Sementsov-Ogievskiy         if (pad->merge_reads && pad->tail) {
15727a3f542fSVladimir Sementsov-Ogievskiy             bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
15737a3f542fSVladimir Sementsov-Ogievskiy         }
15747a3f542fSVladimir Sementsov-Ogievskiy         ret = bdrv_aligned_preadv(child, req, req->overlap_offset, bytes,
157565cd4424SVladimir Sementsov-Ogievskiy                                   align, &local_qiov, 0, 0);
15767a3f542fSVladimir Sementsov-Ogievskiy         if (ret < 0) {
15777a3f542fSVladimir Sementsov-Ogievskiy             return ret;
15787a3f542fSVladimir Sementsov-Ogievskiy         }
15797a3f542fSVladimir Sementsov-Ogievskiy         if (pad->head) {
15807a3f542fSVladimir Sementsov-Ogievskiy             bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
15817a3f542fSVladimir Sementsov-Ogievskiy         }
15827a3f542fSVladimir Sementsov-Ogievskiy         if (pad->merge_reads && pad->tail) {
15837a3f542fSVladimir Sementsov-Ogievskiy             bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
15847a3f542fSVladimir Sementsov-Ogievskiy         }
15857a3f542fSVladimir Sementsov-Ogievskiy 
15867a3f542fSVladimir Sementsov-Ogievskiy         if (pad->merge_reads) {
15877a3f542fSVladimir Sementsov-Ogievskiy             goto zero_mem;
15887a3f542fSVladimir Sementsov-Ogievskiy         }
15897a3f542fSVladimir Sementsov-Ogievskiy     }
15907a3f542fSVladimir Sementsov-Ogievskiy 
15917a3f542fSVladimir Sementsov-Ogievskiy     if (pad->tail) {
15927a3f542fSVladimir Sementsov-Ogievskiy         qemu_iovec_init_buf(&local_qiov, pad->tail_buf, align);
15937a3f542fSVladimir Sementsov-Ogievskiy 
15947a3f542fSVladimir Sementsov-Ogievskiy         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
15957a3f542fSVladimir Sementsov-Ogievskiy         ret = bdrv_aligned_preadv(
15967a3f542fSVladimir Sementsov-Ogievskiy                 child, req,
15977a3f542fSVladimir Sementsov-Ogievskiy                 req->overlap_offset + req->overlap_bytes - align,
159865cd4424SVladimir Sementsov-Ogievskiy                 align, align, &local_qiov, 0, 0);
15997a3f542fSVladimir Sementsov-Ogievskiy         if (ret < 0) {
16007a3f542fSVladimir Sementsov-Ogievskiy             return ret;
16017a3f542fSVladimir Sementsov-Ogievskiy         }
16027a3f542fSVladimir Sementsov-Ogievskiy         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
16037a3f542fSVladimir Sementsov-Ogievskiy     }
16047a3f542fSVladimir Sementsov-Ogievskiy 
16057a3f542fSVladimir Sementsov-Ogievskiy zero_mem:
16067a3f542fSVladimir Sementsov-Ogievskiy     if (zero_middle) {
16077a3f542fSVladimir Sementsov-Ogievskiy         memset(pad->buf + pad->head, 0, pad->buf_len - pad->head - pad->tail);
16087a3f542fSVladimir Sementsov-Ogievskiy     }
16097a3f542fSVladimir Sementsov-Ogievskiy 
16107a3f542fSVladimir Sementsov-Ogievskiy     return 0;
16117a3f542fSVladimir Sementsov-Ogievskiy }
16127a3f542fSVladimir Sementsov-Ogievskiy 
16137a3f542fSVladimir Sementsov-Ogievskiy static void bdrv_padding_destroy(BdrvRequestPadding *pad)
16147a3f542fSVladimir Sementsov-Ogievskiy {
16157a3f542fSVladimir Sementsov-Ogievskiy     if (pad->buf) {
16167a3f542fSVladimir Sementsov-Ogievskiy         qemu_vfree(pad->buf);
16177a3f542fSVladimir Sementsov-Ogievskiy         qemu_iovec_destroy(&pad->local_qiov);
16187a3f542fSVladimir Sementsov-Ogievskiy     }
16197a3f542fSVladimir Sementsov-Ogievskiy }
16207a3f542fSVladimir Sementsov-Ogievskiy 
16217a3f542fSVladimir Sementsov-Ogievskiy /*
16227a3f542fSVladimir Sementsov-Ogievskiy  * bdrv_pad_request
16237a3f542fSVladimir Sementsov-Ogievskiy  *
16247a3f542fSVladimir Sementsov-Ogievskiy  * Exchange request parameters with padded request if needed. Don't include RMW
16257a3f542fSVladimir Sementsov-Ogievskiy  * read of padding, bdrv_padding_rmw_read() should be called separately if
16267a3f542fSVladimir Sementsov-Ogievskiy  * needed.
16277a3f542fSVladimir Sementsov-Ogievskiy  *
16287a3f542fSVladimir Sementsov-Ogievskiy  * All parameters except @bs are in-out: they represent original request at
16297a3f542fSVladimir Sementsov-Ogievskiy  * function call and padded (if padding needed) at function finish.
16307a3f542fSVladimir Sementsov-Ogievskiy  *
16317a3f542fSVladimir Sementsov-Ogievskiy  * Function always succeeds.
16327a3f542fSVladimir Sementsov-Ogievskiy  */
16331acc3466SVladimir Sementsov-Ogievskiy static bool bdrv_pad_request(BlockDriverState *bs,
16341acc3466SVladimir Sementsov-Ogievskiy                              QEMUIOVector **qiov, size_t *qiov_offset,
16357a3f542fSVladimir Sementsov-Ogievskiy                              int64_t *offset, unsigned int *bytes,
16367a3f542fSVladimir Sementsov-Ogievskiy                              BdrvRequestPadding *pad)
16377a3f542fSVladimir Sementsov-Ogievskiy {
16387a3f542fSVladimir Sementsov-Ogievskiy     if (!bdrv_init_padding(bs, *offset, *bytes, pad)) {
16397a3f542fSVladimir Sementsov-Ogievskiy         return false;
16407a3f542fSVladimir Sementsov-Ogievskiy     }
16417a3f542fSVladimir Sementsov-Ogievskiy 
16427a3f542fSVladimir Sementsov-Ogievskiy     qemu_iovec_init_extended(&pad->local_qiov, pad->buf, pad->head,
16431acc3466SVladimir Sementsov-Ogievskiy                              *qiov, *qiov_offset, *bytes,
16447a3f542fSVladimir Sementsov-Ogievskiy                              pad->buf + pad->buf_len - pad->tail, pad->tail);
16457a3f542fSVladimir Sementsov-Ogievskiy     *bytes += pad->head + pad->tail;
16467a3f542fSVladimir Sementsov-Ogievskiy     *offset -= pad->head;
16477a3f542fSVladimir Sementsov-Ogievskiy     *qiov = &pad->local_qiov;
16481acc3466SVladimir Sementsov-Ogievskiy     *qiov_offset = 0;
16497a3f542fSVladimir Sementsov-Ogievskiy 
16507a3f542fSVladimir Sementsov-Ogievskiy     return true;
16517a3f542fSVladimir Sementsov-Ogievskiy }
16527a3f542fSVladimir Sementsov-Ogievskiy 
1653a03ef88fSKevin Wolf int coroutine_fn bdrv_co_preadv(BdrvChild *child,
165461007b31SStefan Hajnoczi     int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
165561007b31SStefan Hajnoczi     BdrvRequestFlags flags)
165661007b31SStefan Hajnoczi {
16571acc3466SVladimir Sementsov-Ogievskiy     return bdrv_co_preadv_part(child, offset, bytes, qiov, 0, flags);
16581acc3466SVladimir Sementsov-Ogievskiy }
16591acc3466SVladimir Sementsov-Ogievskiy 
16601acc3466SVladimir Sementsov-Ogievskiy int coroutine_fn bdrv_co_preadv_part(BdrvChild *child,
16611acc3466SVladimir Sementsov-Ogievskiy     int64_t offset, unsigned int bytes,
16621acc3466SVladimir Sementsov-Ogievskiy     QEMUIOVector *qiov, size_t qiov_offset,
16631acc3466SVladimir Sementsov-Ogievskiy     BdrvRequestFlags flags)
16641acc3466SVladimir Sementsov-Ogievskiy {
1665a03ef88fSKevin Wolf     BlockDriverState *bs = child->bs;
166661007b31SStefan Hajnoczi     BdrvTrackedRequest req;
16677a3f542fSVladimir Sementsov-Ogievskiy     BdrvRequestPadding pad;
166861007b31SStefan Hajnoczi     int ret;
166961007b31SStefan Hajnoczi 
16707a3f542fSVladimir Sementsov-Ogievskiy     trace_bdrv_co_preadv(bs, offset, bytes, flags);
167161007b31SStefan Hajnoczi 
1672f4dad307SVladimir Sementsov-Ogievskiy     if (!bdrv_is_inserted(bs)) {
1673f4dad307SVladimir Sementsov-Ogievskiy         return -ENOMEDIUM;
1674f4dad307SVladimir Sementsov-Ogievskiy     }
1675f4dad307SVladimir Sementsov-Ogievskiy 
16768b117001SVladimir Sementsov-Ogievskiy     ret = bdrv_check_request32(offset, bytes);
167761007b31SStefan Hajnoczi     if (ret < 0) {
167861007b31SStefan Hajnoczi         return ret;
167961007b31SStefan Hajnoczi     }
168061007b31SStefan Hajnoczi 
1681ac9d00bfSVladimir Sementsov-Ogievskiy     if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) {
1682ac9d00bfSVladimir Sementsov-Ogievskiy         /*
1683ac9d00bfSVladimir Sementsov-Ogievskiy          * Aligning zero request is nonsense. Even if driver has special meaning
1684ac9d00bfSVladimir Sementsov-Ogievskiy          * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass
1685ac9d00bfSVladimir Sementsov-Ogievskiy          * it to driver due to request_alignment.
1686ac9d00bfSVladimir Sementsov-Ogievskiy          *
1687ac9d00bfSVladimir Sementsov-Ogievskiy          * Still, no reason to return an error if someone do unaligned
1688ac9d00bfSVladimir Sementsov-Ogievskiy          * zero-length read occasionally.
1689ac9d00bfSVladimir Sementsov-Ogievskiy          */
1690ac9d00bfSVladimir Sementsov-Ogievskiy         return 0;
1691ac9d00bfSVladimir Sementsov-Ogievskiy     }
1692ac9d00bfSVladimir Sementsov-Ogievskiy 
169399723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
169499723548SPaolo Bonzini 
16959568b511SWen Congyang     /* Don't do copy-on-read if we read data before write operation */
1696d73415a3SStefan Hajnoczi     if (qatomic_read(&bs->copy_on_read)) {
169761007b31SStefan Hajnoczi         flags |= BDRV_REQ_COPY_ON_READ;
169861007b31SStefan Hajnoczi     }
169961007b31SStefan Hajnoczi 
17001acc3466SVladimir Sementsov-Ogievskiy     bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad);
170161007b31SStefan Hajnoczi 
1702ebde595cSFam Zheng     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
17037a3f542fSVladimir Sementsov-Ogievskiy     ret = bdrv_aligned_preadv(child, &req, offset, bytes,
17047a3f542fSVladimir Sementsov-Ogievskiy                               bs->bl.request_alignment,
17051acc3466SVladimir Sementsov-Ogievskiy                               qiov, qiov_offset, flags);
170661007b31SStefan Hajnoczi     tracked_request_end(&req);
170799723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
170861007b31SStefan Hajnoczi 
17097a3f542fSVladimir Sementsov-Ogievskiy     bdrv_padding_destroy(&pad);
171061007b31SStefan Hajnoczi 
171161007b31SStefan Hajnoczi     return ret;
171261007b31SStefan Hajnoczi }
171361007b31SStefan Hajnoczi 
1714d05aa8bbSEric Blake static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
1715f5a5ca79SManos Pitsidianakis     int64_t offset, int bytes, BdrvRequestFlags flags)
171661007b31SStefan Hajnoczi {
171761007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
171861007b31SStefan Hajnoczi     QEMUIOVector qiov;
17190d93ed08SVladimir Sementsov-Ogievskiy     void *buf = NULL;
172061007b31SStefan Hajnoczi     int ret = 0;
1721465fe887SEric Blake     bool need_flush = false;
1722443668caSDenis V. Lunev     int head = 0;
1723443668caSDenis V. Lunev     int tail = 0;
172461007b31SStefan Hajnoczi 
1725cf081fcaSEric Blake     int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, INT_MAX);
1726a5b8dd2cSEric Blake     int alignment = MAX(bs->bl.pwrite_zeroes_alignment,
1727a5b8dd2cSEric Blake                         bs->bl.request_alignment);
1728cb2e2878SEric Blake     int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER);
1729cf081fcaSEric Blake 
1730d470ad42SMax Reitz     if (!drv) {
1731d470ad42SMax Reitz         return -ENOMEDIUM;
1732d470ad42SMax Reitz     }
1733d470ad42SMax Reitz 
1734fe0480d6SKevin Wolf     if ((flags & ~bs->supported_zero_flags) & BDRV_REQ_NO_FALLBACK) {
1735fe0480d6SKevin Wolf         return -ENOTSUP;
1736fe0480d6SKevin Wolf     }
1737fe0480d6SKevin Wolf 
1738b8d0a980SEric Blake     assert(alignment % bs->bl.request_alignment == 0);
1739b8d0a980SEric Blake     head = offset % alignment;
1740f5a5ca79SManos Pitsidianakis     tail = (offset + bytes) % alignment;
1741b8d0a980SEric Blake     max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment);
1742b8d0a980SEric Blake     assert(max_write_zeroes >= bs->bl.request_alignment);
174361007b31SStefan Hajnoczi 
1744f5a5ca79SManos Pitsidianakis     while (bytes > 0 && !ret) {
1745f5a5ca79SManos Pitsidianakis         int num = bytes;
174661007b31SStefan Hajnoczi 
174761007b31SStefan Hajnoczi         /* Align request.  Block drivers can expect the "bulk" of the request
1748443668caSDenis V. Lunev          * to be aligned, and that unaligned requests do not cross cluster
1749443668caSDenis V. Lunev          * boundaries.
175061007b31SStefan Hajnoczi          */
1751443668caSDenis V. Lunev         if (head) {
1752b2f95feeSEric Blake             /* Make a small request up to the first aligned sector. For
1753b2f95feeSEric Blake              * convenience, limit this request to max_transfer even if
1754b2f95feeSEric Blake              * we don't need to fall back to writes.  */
1755f5a5ca79SManos Pitsidianakis             num = MIN(MIN(bytes, max_transfer), alignment - head);
1756b2f95feeSEric Blake             head = (head + num) % alignment;
1757b2f95feeSEric Blake             assert(num < max_write_zeroes);
1758d05aa8bbSEric Blake         } else if (tail && num > alignment) {
1759443668caSDenis V. Lunev             /* Shorten the request to the last aligned sector.  */
1760443668caSDenis V. Lunev             num -= tail;
176161007b31SStefan Hajnoczi         }
176261007b31SStefan Hajnoczi 
176361007b31SStefan Hajnoczi         /* limit request size */
176461007b31SStefan Hajnoczi         if (num > max_write_zeroes) {
176561007b31SStefan Hajnoczi             num = max_write_zeroes;
176661007b31SStefan Hajnoczi         }
176761007b31SStefan Hajnoczi 
176861007b31SStefan Hajnoczi         ret = -ENOTSUP;
176961007b31SStefan Hajnoczi         /* First try the efficient write zeroes operation */
1770d05aa8bbSEric Blake         if (drv->bdrv_co_pwrite_zeroes) {
1771d05aa8bbSEric Blake             ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num,
1772d05aa8bbSEric Blake                                              flags & bs->supported_zero_flags);
1773d05aa8bbSEric Blake             if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) &&
1774d05aa8bbSEric Blake                 !(bs->supported_zero_flags & BDRV_REQ_FUA)) {
1775d05aa8bbSEric Blake                 need_flush = true;
1776d05aa8bbSEric Blake             }
1777465fe887SEric Blake         } else {
1778465fe887SEric Blake             assert(!bs->supported_zero_flags);
177961007b31SStefan Hajnoczi         }
178061007b31SStefan Hajnoczi 
1781294682ccSAndrey Shinkevich         if (ret == -ENOTSUP && !(flags & BDRV_REQ_NO_FALLBACK)) {
178261007b31SStefan Hajnoczi             /* Fall back to bounce buffer if write zeroes is unsupported */
1783465fe887SEric Blake             BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;
1784465fe887SEric Blake 
1785465fe887SEric Blake             if ((flags & BDRV_REQ_FUA) &&
1786465fe887SEric Blake                 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1787465fe887SEric Blake                 /* No need for bdrv_driver_pwrite() to do a fallback
1788465fe887SEric Blake                  * flush on each chunk; use just one at the end */
1789465fe887SEric Blake                 write_flags &= ~BDRV_REQ_FUA;
1790465fe887SEric Blake                 need_flush = true;
1791465fe887SEric Blake             }
17925def6b80SEric Blake             num = MIN(num, max_transfer);
17930d93ed08SVladimir Sementsov-Ogievskiy             if (buf == NULL) {
17940d93ed08SVladimir Sementsov-Ogievskiy                 buf = qemu_try_blockalign0(bs, num);
17950d93ed08SVladimir Sementsov-Ogievskiy                 if (buf == NULL) {
179661007b31SStefan Hajnoczi                     ret = -ENOMEM;
179761007b31SStefan Hajnoczi                     goto fail;
179861007b31SStefan Hajnoczi                 }
179961007b31SStefan Hajnoczi             }
18000d93ed08SVladimir Sementsov-Ogievskiy             qemu_iovec_init_buf(&qiov, buf, num);
180161007b31SStefan Hajnoczi 
1802ac850bf0SVladimir Sementsov-Ogievskiy             ret = bdrv_driver_pwritev(bs, offset, num, &qiov, 0, write_flags);
180361007b31SStefan Hajnoczi 
180461007b31SStefan Hajnoczi             /* Keep bounce buffer around if it is big enough for all
180561007b31SStefan Hajnoczi              * all future requests.
180661007b31SStefan Hajnoczi              */
18075def6b80SEric Blake             if (num < max_transfer) {
18080d93ed08SVladimir Sementsov-Ogievskiy                 qemu_vfree(buf);
18090d93ed08SVladimir Sementsov-Ogievskiy                 buf = NULL;
181061007b31SStefan Hajnoczi             }
181161007b31SStefan Hajnoczi         }
181261007b31SStefan Hajnoczi 
1813d05aa8bbSEric Blake         offset += num;
1814f5a5ca79SManos Pitsidianakis         bytes -= num;
181561007b31SStefan Hajnoczi     }
181661007b31SStefan Hajnoczi 
181761007b31SStefan Hajnoczi fail:
1818465fe887SEric Blake     if (ret == 0 && need_flush) {
1819465fe887SEric Blake         ret = bdrv_co_flush(bs);
1820465fe887SEric Blake     }
18210d93ed08SVladimir Sementsov-Ogievskiy     qemu_vfree(buf);
182261007b31SStefan Hajnoczi     return ret;
182361007b31SStefan Hajnoczi }
182461007b31SStefan Hajnoczi 
182585fe2479SFam Zheng static inline int coroutine_fn
182685fe2479SFam Zheng bdrv_co_write_req_prepare(BdrvChild *child, int64_t offset, uint64_t bytes,
182785fe2479SFam Zheng                           BdrvTrackedRequest *req, int flags)
182885fe2479SFam Zheng {
182985fe2479SFam Zheng     BlockDriverState *bs = child->bs;
183085fe2479SFam Zheng     bool waited;
183185fe2479SFam Zheng     int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
183285fe2479SFam Zheng 
183385fe2479SFam Zheng     if (bs->read_only) {
183485fe2479SFam Zheng         return -EPERM;
183585fe2479SFam Zheng     }
183685fe2479SFam Zheng 
183785fe2479SFam Zheng     assert(!(bs->open_flags & BDRV_O_INACTIVE));
183885fe2479SFam Zheng     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
183985fe2479SFam Zheng     assert(!(flags & ~BDRV_REQ_MASK));
184085fe2479SFam Zheng 
184185fe2479SFam Zheng     if (flags & BDRV_REQ_SERIALISING) {
184218fbd0deSPaolo Bonzini         waited = bdrv_mark_request_serialising(req, bdrv_get_cluster_size(bs));
184318fbd0deSPaolo Bonzini         /*
184418fbd0deSPaolo Bonzini          * For a misaligned request we should have already waited earlier,
184518fbd0deSPaolo Bonzini          * because we come after bdrv_padding_rmw_read which must be called
184618fbd0deSPaolo Bonzini          * with the request already marked as serialising.
184718fbd0deSPaolo Bonzini          */
184818fbd0deSPaolo Bonzini         assert(!waited ||
184918fbd0deSPaolo Bonzini                (req->offset == req->overlap_offset &&
185018fbd0deSPaolo Bonzini                 req->bytes == req->overlap_bytes));
185118fbd0deSPaolo Bonzini     } else {
185218fbd0deSPaolo Bonzini         bdrv_wait_serialising_requests(req);
185385fe2479SFam Zheng     }
185485fe2479SFam Zheng 
185585fe2479SFam Zheng     assert(req->overlap_offset <= offset);
185685fe2479SFam Zheng     assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
1857cd47d792SFam Zheng     assert(end_sector <= bs->total_sectors || child->perm & BLK_PERM_RESIZE);
185885fe2479SFam Zheng 
1859cd47d792SFam Zheng     switch (req->type) {
1860cd47d792SFam Zheng     case BDRV_TRACKED_WRITE:
1861cd47d792SFam Zheng     case BDRV_TRACKED_DISCARD:
186285fe2479SFam Zheng         if (flags & BDRV_REQ_WRITE_UNCHANGED) {
186385fe2479SFam Zheng             assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
186485fe2479SFam Zheng         } else {
186585fe2479SFam Zheng             assert(child->perm & BLK_PERM_WRITE);
186685fe2479SFam Zheng         }
1867cd47d792SFam Zheng         return notifier_with_return_list_notify(&bs->before_write_notifiers,
1868cd47d792SFam Zheng                                                 req);
1869cd47d792SFam Zheng     case BDRV_TRACKED_TRUNCATE:
1870cd47d792SFam Zheng         assert(child->perm & BLK_PERM_RESIZE);
1871cd47d792SFam Zheng         return 0;
1872cd47d792SFam Zheng     default:
1873cd47d792SFam Zheng         abort();
1874cd47d792SFam Zheng     }
187585fe2479SFam Zheng }
187685fe2479SFam Zheng 
187785fe2479SFam Zheng static inline void coroutine_fn
187885fe2479SFam Zheng bdrv_co_write_req_finish(BdrvChild *child, int64_t offset, uint64_t bytes,
187985fe2479SFam Zheng                          BdrvTrackedRequest *req, int ret)
188085fe2479SFam Zheng {
188185fe2479SFam Zheng     int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
188285fe2479SFam Zheng     BlockDriverState *bs = child->bs;
188385fe2479SFam Zheng 
1884d73415a3SStefan Hajnoczi     qatomic_inc(&bs->write_gen);
188585fe2479SFam Zheng 
188600695c27SFam Zheng     /*
188700695c27SFam Zheng      * Discard cannot extend the image, but in error handling cases, such as
188800695c27SFam Zheng      * when reverting a qcow2 cluster allocation, the discarded range can pass
188900695c27SFam Zheng      * the end of image file, so we cannot assert about BDRV_TRACKED_DISCARD
189000695c27SFam Zheng      * here. Instead, just skip it, since semantically a discard request
189100695c27SFam Zheng      * beyond EOF cannot expand the image anyway.
189200695c27SFam Zheng      */
18937f8f03efSFam Zheng     if (ret == 0 &&
1894cd47d792SFam Zheng         (req->type == BDRV_TRACKED_TRUNCATE ||
1895cd47d792SFam Zheng          end_sector > bs->total_sectors) &&
189600695c27SFam Zheng         req->type != BDRV_TRACKED_DISCARD) {
18977f8f03efSFam Zheng         bs->total_sectors = end_sector;
18987f8f03efSFam Zheng         bdrv_parent_cb_resize(bs);
18997f8f03efSFam Zheng         bdrv_dirty_bitmap_truncate(bs, end_sector << BDRV_SECTOR_BITS);
190085fe2479SFam Zheng     }
190100695c27SFam Zheng     if (req->bytes) {
190200695c27SFam Zheng         switch (req->type) {
190300695c27SFam Zheng         case BDRV_TRACKED_WRITE:
190400695c27SFam Zheng             stat64_max(&bs->wr_highest_offset, offset + bytes);
190500695c27SFam Zheng             /* fall through, to set dirty bits */
190600695c27SFam Zheng         case BDRV_TRACKED_DISCARD:
19077f8f03efSFam Zheng             bdrv_set_dirty(bs, offset, bytes);
190800695c27SFam Zheng             break;
190900695c27SFam Zheng         default:
191000695c27SFam Zheng             break;
191100695c27SFam Zheng         }
191200695c27SFam Zheng     }
191385fe2479SFam Zheng }
191485fe2479SFam Zheng 
191561007b31SStefan Hajnoczi /*
191604ed95f4SEric Blake  * Forwards an already correctly aligned write request to the BlockDriver,
191704ed95f4SEric Blake  * after possibly fragmenting it.
191861007b31SStefan Hajnoczi  */
191985c97ca7SKevin Wolf static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child,
192061007b31SStefan Hajnoczi     BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
192128c4da28SVladimir Sementsov-Ogievskiy     int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags)
192261007b31SStefan Hajnoczi {
192385c97ca7SKevin Wolf     BlockDriverState *bs = child->bs;
192461007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
192561007b31SStefan Hajnoczi     int ret;
192661007b31SStefan Hajnoczi 
192704ed95f4SEric Blake     uint64_t bytes_remaining = bytes;
192804ed95f4SEric Blake     int max_transfer;
192961007b31SStefan Hajnoczi 
1930d470ad42SMax Reitz     if (!drv) {
1931d470ad42SMax Reitz         return -ENOMEDIUM;
1932d470ad42SMax Reitz     }
1933d470ad42SMax Reitz 
1934d6883bc9SVladimir Sementsov-Ogievskiy     if (bdrv_has_readonly_bitmaps(bs)) {
1935d6883bc9SVladimir Sementsov-Ogievskiy         return -EPERM;
1936d6883bc9SVladimir Sementsov-Ogievskiy     }
1937d6883bc9SVladimir Sementsov-Ogievskiy 
1938cff86b38SEric Blake     assert(is_power_of_2(align));
1939cff86b38SEric Blake     assert((offset & (align - 1)) == 0);
1940cff86b38SEric Blake     assert((bytes & (align - 1)) == 0);
194128c4da28SVladimir Sementsov-Ogievskiy     assert(!qiov || qiov_offset + bytes <= qiov->size);
194204ed95f4SEric Blake     max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
194304ed95f4SEric Blake                                    align);
194461007b31SStefan Hajnoczi 
194585fe2479SFam Zheng     ret = bdrv_co_write_req_prepare(child, offset, bytes, req, flags);
194661007b31SStefan Hajnoczi 
194761007b31SStefan Hajnoczi     if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
1948c1499a5eSEric Blake         !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes &&
194928c4da28SVladimir Sementsov-Ogievskiy         qemu_iovec_is_zero(qiov, qiov_offset, bytes)) {
195061007b31SStefan Hajnoczi         flags |= BDRV_REQ_ZERO_WRITE;
195161007b31SStefan Hajnoczi         if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
195261007b31SStefan Hajnoczi             flags |= BDRV_REQ_MAY_UNMAP;
195361007b31SStefan Hajnoczi         }
195461007b31SStefan Hajnoczi     }
195561007b31SStefan Hajnoczi 
195661007b31SStefan Hajnoczi     if (ret < 0) {
195761007b31SStefan Hajnoczi         /* Do nothing, write notifier decided to fail this request */
195861007b31SStefan Hajnoczi     } else if (flags & BDRV_REQ_ZERO_WRITE) {
19599a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
19609896c876SKevin Wolf         ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags);
19613ea1a091SPavel Butsykin     } else if (flags & BDRV_REQ_WRITE_COMPRESSED) {
196228c4da28SVladimir Sementsov-Ogievskiy         ret = bdrv_driver_pwritev_compressed(bs, offset, bytes,
196328c4da28SVladimir Sementsov-Ogievskiy                                              qiov, qiov_offset);
196404ed95f4SEric Blake     } else if (bytes <= max_transfer) {
19659a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV);
196628c4da28SVladimir Sementsov-Ogievskiy         ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, qiov_offset, flags);
196704ed95f4SEric Blake     } else {
196804ed95f4SEric Blake         bdrv_debug_event(bs, BLKDBG_PWRITEV);
196904ed95f4SEric Blake         while (bytes_remaining) {
197004ed95f4SEric Blake             int num = MIN(bytes_remaining, max_transfer);
197104ed95f4SEric Blake             int local_flags = flags;
197204ed95f4SEric Blake 
197304ed95f4SEric Blake             assert(num);
197404ed95f4SEric Blake             if (num < bytes_remaining && (flags & BDRV_REQ_FUA) &&
197504ed95f4SEric Blake                 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
197604ed95f4SEric Blake                 /* If FUA is going to be emulated by flush, we only
197704ed95f4SEric Blake                  * need to flush on the last iteration */
197804ed95f4SEric Blake                 local_flags &= ~BDRV_REQ_FUA;
197904ed95f4SEric Blake             }
198004ed95f4SEric Blake 
198104ed95f4SEric Blake             ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining,
1982134b7decSMax Reitz                                       num, qiov,
1983134b7decSMax Reitz                                       qiov_offset + bytes - bytes_remaining,
198428c4da28SVladimir Sementsov-Ogievskiy                                       local_flags);
198504ed95f4SEric Blake             if (ret < 0) {
198604ed95f4SEric Blake                 break;
198704ed95f4SEric Blake             }
198804ed95f4SEric Blake             bytes_remaining -= num;
198904ed95f4SEric Blake         }
199061007b31SStefan Hajnoczi     }
19919a4f4c31SKevin Wolf     bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
199261007b31SStefan Hajnoczi 
199361007b31SStefan Hajnoczi     if (ret >= 0) {
199404ed95f4SEric Blake         ret = 0;
199561007b31SStefan Hajnoczi     }
199685fe2479SFam Zheng     bdrv_co_write_req_finish(child, offset, bytes, req, ret);
199761007b31SStefan Hajnoczi 
199861007b31SStefan Hajnoczi     return ret;
199961007b31SStefan Hajnoczi }
200061007b31SStefan Hajnoczi 
200185c97ca7SKevin Wolf static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child,
20029eeb6dd1SFam Zheng                                                 int64_t offset,
20039eeb6dd1SFam Zheng                                                 unsigned int bytes,
20049eeb6dd1SFam Zheng                                                 BdrvRequestFlags flags,
20059eeb6dd1SFam Zheng                                                 BdrvTrackedRequest *req)
20069eeb6dd1SFam Zheng {
200785c97ca7SKevin Wolf     BlockDriverState *bs = child->bs;
20089eeb6dd1SFam Zheng     QEMUIOVector local_qiov;
2009a5b8dd2cSEric Blake     uint64_t align = bs->bl.request_alignment;
20109eeb6dd1SFam Zheng     int ret = 0;
20117a3f542fSVladimir Sementsov-Ogievskiy     bool padding;
20127a3f542fSVladimir Sementsov-Ogievskiy     BdrvRequestPadding pad;
20139eeb6dd1SFam Zheng 
20147a3f542fSVladimir Sementsov-Ogievskiy     padding = bdrv_init_padding(bs, offset, bytes, &pad);
20157a3f542fSVladimir Sementsov-Ogievskiy     if (padding) {
2016304d9d7fSMax Reitz         bdrv_mark_request_serialising(req, align);
20179eeb6dd1SFam Zheng 
20187a3f542fSVladimir Sementsov-Ogievskiy         bdrv_padding_rmw_read(child, req, &pad, true);
20197a3f542fSVladimir Sementsov-Ogievskiy 
20207a3f542fSVladimir Sementsov-Ogievskiy         if (pad.head || pad.merge_reads) {
20217a3f542fSVladimir Sementsov-Ogievskiy             int64_t aligned_offset = offset & ~(align - 1);
20227a3f542fSVladimir Sementsov-Ogievskiy             int64_t write_bytes = pad.merge_reads ? pad.buf_len : align;
20237a3f542fSVladimir Sementsov-Ogievskiy 
20247a3f542fSVladimir Sementsov-Ogievskiy             qemu_iovec_init_buf(&local_qiov, pad.buf, write_bytes);
20257a3f542fSVladimir Sementsov-Ogievskiy             ret = bdrv_aligned_pwritev(child, req, aligned_offset, write_bytes,
202628c4da28SVladimir Sementsov-Ogievskiy                                        align, &local_qiov, 0,
20279eeb6dd1SFam Zheng                                        flags & ~BDRV_REQ_ZERO_WRITE);
20287a3f542fSVladimir Sementsov-Ogievskiy             if (ret < 0 || pad.merge_reads) {
20297a3f542fSVladimir Sementsov-Ogievskiy                 /* Error or all work is done */
20307a3f542fSVladimir Sementsov-Ogievskiy                 goto out;
20319eeb6dd1SFam Zheng             }
20327a3f542fSVladimir Sementsov-Ogievskiy             offset += write_bytes - pad.head;
20337a3f542fSVladimir Sementsov-Ogievskiy             bytes -= write_bytes - pad.head;
20347a3f542fSVladimir Sementsov-Ogievskiy         }
20359eeb6dd1SFam Zheng     }
20369eeb6dd1SFam Zheng 
20379eeb6dd1SFam Zheng     assert(!bytes || (offset & (align - 1)) == 0);
20389eeb6dd1SFam Zheng     if (bytes >= align) {
20399eeb6dd1SFam Zheng         /* Write the aligned part in the middle. */
20409eeb6dd1SFam Zheng         uint64_t aligned_bytes = bytes & ~(align - 1);
204185c97ca7SKevin Wolf         ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align,
204228c4da28SVladimir Sementsov-Ogievskiy                                    NULL, 0, flags);
20439eeb6dd1SFam Zheng         if (ret < 0) {
20447a3f542fSVladimir Sementsov-Ogievskiy             goto out;
20459eeb6dd1SFam Zheng         }
20469eeb6dd1SFam Zheng         bytes -= aligned_bytes;
20479eeb6dd1SFam Zheng         offset += aligned_bytes;
20489eeb6dd1SFam Zheng     }
20499eeb6dd1SFam Zheng 
20509eeb6dd1SFam Zheng     assert(!bytes || (offset & (align - 1)) == 0);
20519eeb6dd1SFam Zheng     if (bytes) {
20527a3f542fSVladimir Sementsov-Ogievskiy         assert(align == pad.tail + bytes);
20539eeb6dd1SFam Zheng 
20547a3f542fSVladimir Sementsov-Ogievskiy         qemu_iovec_init_buf(&local_qiov, pad.tail_buf, align);
205585c97ca7SKevin Wolf         ret = bdrv_aligned_pwritev(child, req, offset, align, align,
205628c4da28SVladimir Sementsov-Ogievskiy                                    &local_qiov, 0,
205728c4da28SVladimir Sementsov-Ogievskiy                                    flags & ~BDRV_REQ_ZERO_WRITE);
20589eeb6dd1SFam Zheng     }
20599eeb6dd1SFam Zheng 
20607a3f542fSVladimir Sementsov-Ogievskiy out:
20617a3f542fSVladimir Sementsov-Ogievskiy     bdrv_padding_destroy(&pad);
20627a3f542fSVladimir Sementsov-Ogievskiy 
20637a3f542fSVladimir Sementsov-Ogievskiy     return ret;
20649eeb6dd1SFam Zheng }
20659eeb6dd1SFam Zheng 
206661007b31SStefan Hajnoczi /*
206761007b31SStefan Hajnoczi  * Handle a write request in coroutine context
206861007b31SStefan Hajnoczi  */
2069a03ef88fSKevin Wolf int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
207061007b31SStefan Hajnoczi     int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
207161007b31SStefan Hajnoczi     BdrvRequestFlags flags)
207261007b31SStefan Hajnoczi {
20731acc3466SVladimir Sementsov-Ogievskiy     return bdrv_co_pwritev_part(child, offset, bytes, qiov, 0, flags);
20741acc3466SVladimir Sementsov-Ogievskiy }
20751acc3466SVladimir Sementsov-Ogievskiy 
20761acc3466SVladimir Sementsov-Ogievskiy int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child,
20771acc3466SVladimir Sementsov-Ogievskiy     int64_t offset, unsigned int bytes, QEMUIOVector *qiov, size_t qiov_offset,
20781acc3466SVladimir Sementsov-Ogievskiy     BdrvRequestFlags flags)
20791acc3466SVladimir Sementsov-Ogievskiy {
2080a03ef88fSKevin Wolf     BlockDriverState *bs = child->bs;
208161007b31SStefan Hajnoczi     BdrvTrackedRequest req;
2082a5b8dd2cSEric Blake     uint64_t align = bs->bl.request_alignment;
20837a3f542fSVladimir Sementsov-Ogievskiy     BdrvRequestPadding pad;
208461007b31SStefan Hajnoczi     int ret;
208561007b31SStefan Hajnoczi 
2086f42cf447SDaniel P. Berrange     trace_bdrv_co_pwritev(child->bs, offset, bytes, flags);
2087f42cf447SDaniel P. Berrange 
2088f4dad307SVladimir Sementsov-Ogievskiy     if (!bdrv_is_inserted(bs)) {
208961007b31SStefan Hajnoczi         return -ENOMEDIUM;
209061007b31SStefan Hajnoczi     }
209161007b31SStefan Hajnoczi 
20928b117001SVladimir Sementsov-Ogievskiy     ret = bdrv_check_request32(offset, bytes);
209361007b31SStefan Hajnoczi     if (ret < 0) {
209461007b31SStefan Hajnoczi         return ret;
209561007b31SStefan Hajnoczi     }
209661007b31SStefan Hajnoczi 
2097f2208fdcSAlberto Garcia     /* If the request is misaligned then we can't make it efficient */
2098f2208fdcSAlberto Garcia     if ((flags & BDRV_REQ_NO_FALLBACK) &&
2099f2208fdcSAlberto Garcia         !QEMU_IS_ALIGNED(offset | bytes, align))
2100f2208fdcSAlberto Garcia     {
2101f2208fdcSAlberto Garcia         return -ENOTSUP;
2102f2208fdcSAlberto Garcia     }
2103f2208fdcSAlberto Garcia 
2104ac9d00bfSVladimir Sementsov-Ogievskiy     if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) {
2105ac9d00bfSVladimir Sementsov-Ogievskiy         /*
2106ac9d00bfSVladimir Sementsov-Ogievskiy          * Aligning zero request is nonsense. Even if driver has special meaning
2107ac9d00bfSVladimir Sementsov-Ogievskiy          * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass
2108ac9d00bfSVladimir Sementsov-Ogievskiy          * it to driver due to request_alignment.
2109ac9d00bfSVladimir Sementsov-Ogievskiy          *
2110ac9d00bfSVladimir Sementsov-Ogievskiy          * Still, no reason to return an error if someone do unaligned
2111ac9d00bfSVladimir Sementsov-Ogievskiy          * zero-length write occasionally.
2112ac9d00bfSVladimir Sementsov-Ogievskiy          */
2113ac9d00bfSVladimir Sementsov-Ogievskiy         return 0;
2114ac9d00bfSVladimir Sementsov-Ogievskiy     }
2115ac9d00bfSVladimir Sementsov-Ogievskiy 
211699723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
211761007b31SStefan Hajnoczi     /*
211861007b31SStefan Hajnoczi      * Align write if necessary by performing a read-modify-write cycle.
211961007b31SStefan Hajnoczi      * Pad qiov with the read parts and be sure to have a tracked request not
212061007b31SStefan Hajnoczi      * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle.
212161007b31SStefan Hajnoczi      */
2122ebde595cSFam Zheng     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
212361007b31SStefan Hajnoczi 
212418a59f03SAnton Nefedov     if (flags & BDRV_REQ_ZERO_WRITE) {
212585c97ca7SKevin Wolf         ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req);
21269eeb6dd1SFam Zheng         goto out;
21279eeb6dd1SFam Zheng     }
21289eeb6dd1SFam Zheng 
21291acc3466SVladimir Sementsov-Ogievskiy     if (bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad)) {
2130304d9d7fSMax Reitz         bdrv_mark_request_serialising(&req, align);
21317a3f542fSVladimir Sementsov-Ogievskiy         bdrv_padding_rmw_read(child, &req, &pad, false);
213261007b31SStefan Hajnoczi     }
213361007b31SStefan Hajnoczi 
213485c97ca7SKevin Wolf     ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align,
21351acc3466SVladimir Sementsov-Ogievskiy                                qiov, qiov_offset, flags);
213661007b31SStefan Hajnoczi 
21377a3f542fSVladimir Sementsov-Ogievskiy     bdrv_padding_destroy(&pad);
213861007b31SStefan Hajnoczi 
21399eeb6dd1SFam Zheng out:
21409eeb6dd1SFam Zheng     tracked_request_end(&req);
214199723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
21427a3f542fSVladimir Sementsov-Ogievskiy 
214361007b31SStefan Hajnoczi     return ret;
214461007b31SStefan Hajnoczi }
214561007b31SStefan Hajnoczi 
2146a03ef88fSKevin Wolf int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset,
2147f5a5ca79SManos Pitsidianakis                                        int bytes, BdrvRequestFlags flags)
214861007b31SStefan Hajnoczi {
2149f5a5ca79SManos Pitsidianakis     trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags);
215061007b31SStefan Hajnoczi 
2151a03ef88fSKevin Wolf     if (!(child->bs->open_flags & BDRV_O_UNMAP)) {
215261007b31SStefan Hajnoczi         flags &= ~BDRV_REQ_MAY_UNMAP;
215361007b31SStefan Hajnoczi     }
215461007b31SStefan Hajnoczi 
2155f5a5ca79SManos Pitsidianakis     return bdrv_co_pwritev(child, offset, bytes, NULL,
215661007b31SStefan Hajnoczi                            BDRV_REQ_ZERO_WRITE | flags);
215761007b31SStefan Hajnoczi }
215861007b31SStefan Hajnoczi 
21594085f5c7SJohn Snow /*
21604085f5c7SJohn Snow  * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not.
21614085f5c7SJohn Snow  */
21624085f5c7SJohn Snow int bdrv_flush_all(void)
21634085f5c7SJohn Snow {
21644085f5c7SJohn Snow     BdrvNextIterator it;
21654085f5c7SJohn Snow     BlockDriverState *bs = NULL;
21664085f5c7SJohn Snow     int result = 0;
21674085f5c7SJohn Snow 
2168c8aa7895SPavel Dovgalyuk     /*
2169c8aa7895SPavel Dovgalyuk      * bdrv queue is managed by record/replay,
2170c8aa7895SPavel Dovgalyuk      * creating new flush request for stopping
2171c8aa7895SPavel Dovgalyuk      * the VM may break the determinism
2172c8aa7895SPavel Dovgalyuk      */
2173c8aa7895SPavel Dovgalyuk     if (replay_events_enabled()) {
2174c8aa7895SPavel Dovgalyuk         return result;
2175c8aa7895SPavel Dovgalyuk     }
2176c8aa7895SPavel Dovgalyuk 
21774085f5c7SJohn Snow     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
21784085f5c7SJohn Snow         AioContext *aio_context = bdrv_get_aio_context(bs);
21794085f5c7SJohn Snow         int ret;
21804085f5c7SJohn Snow 
21814085f5c7SJohn Snow         aio_context_acquire(aio_context);
21824085f5c7SJohn Snow         ret = bdrv_flush(bs);
21834085f5c7SJohn Snow         if (ret < 0 && !result) {
21844085f5c7SJohn Snow             result = ret;
21854085f5c7SJohn Snow         }
21864085f5c7SJohn Snow         aio_context_release(aio_context);
21874085f5c7SJohn Snow     }
21884085f5c7SJohn Snow 
21894085f5c7SJohn Snow     return result;
21904085f5c7SJohn Snow }
21914085f5c7SJohn Snow 
219261007b31SStefan Hajnoczi /*
219361007b31SStefan Hajnoczi  * Returns the allocation status of the specified sectors.
219461007b31SStefan Hajnoczi  * Drivers not implementing the functionality are assumed to not support
219561007b31SStefan Hajnoczi  * backing files, hence all their sectors are reported as allocated.
219661007b31SStefan Hajnoczi  *
219786a3d5c6SEric Blake  * If 'want_zero' is true, the caller is querying for mapping
219886a3d5c6SEric Blake  * purposes, with a focus on valid BDRV_BLOCK_OFFSET_VALID, _DATA, and
219986a3d5c6SEric Blake  * _ZERO where possible; otherwise, the result favors larger 'pnum',
220086a3d5c6SEric Blake  * with a focus on accurate BDRV_BLOCK_ALLOCATED.
2201c9ce8c4dSEric Blake  *
22022e8bc787SEric Blake  * If 'offset' is beyond the end of the disk image the return value is
2203fb0d8654SEric Blake  * BDRV_BLOCK_EOF and 'pnum' is set to 0.
220461007b31SStefan Hajnoczi  *
22052e8bc787SEric Blake  * 'bytes' is the max value 'pnum' should be set to.  If bytes goes
2206fb0d8654SEric Blake  * beyond the end of the disk image it will be clamped; if 'pnum' is set to
2207fb0d8654SEric Blake  * the end of the image, then the returned value will include BDRV_BLOCK_EOF.
220867a0fd2aSFam Zheng  *
22092e8bc787SEric Blake  * 'pnum' is set to the number of bytes (including and immediately
22102e8bc787SEric Blake  * following the specified offset) that are easily known to be in the
22112e8bc787SEric Blake  * same allocated/unallocated state.  Note that a second call starting
22122e8bc787SEric Blake  * at the original offset plus returned pnum may have the same status.
22132e8bc787SEric Blake  * The returned value is non-zero on success except at end-of-file.
22142e8bc787SEric Blake  *
22152e8bc787SEric Blake  * Returns negative errno on failure.  Otherwise, if the
22162e8bc787SEric Blake  * BDRV_BLOCK_OFFSET_VALID bit is set, 'map' and 'file' (if non-NULL) are
22172e8bc787SEric Blake  * set to the host mapping and BDS corresponding to the guest offset.
221861007b31SStefan Hajnoczi  */
22192e8bc787SEric Blake static int coroutine_fn bdrv_co_block_status(BlockDriverState *bs,
2220c9ce8c4dSEric Blake                                              bool want_zero,
22212e8bc787SEric Blake                                              int64_t offset, int64_t bytes,
22222e8bc787SEric Blake                                              int64_t *pnum, int64_t *map,
222367a0fd2aSFam Zheng                                              BlockDriverState **file)
222461007b31SStefan Hajnoczi {
22252e8bc787SEric Blake     int64_t total_size;
22262e8bc787SEric Blake     int64_t n; /* bytes */
2227efa6e2edSEric Blake     int ret;
22282e8bc787SEric Blake     int64_t local_map = 0;
2229298a1665SEric Blake     BlockDriverState *local_file = NULL;
2230efa6e2edSEric Blake     int64_t aligned_offset, aligned_bytes;
2231efa6e2edSEric Blake     uint32_t align;
2232549ec0d9SMax Reitz     bool has_filtered_child;
223361007b31SStefan Hajnoczi 
2234298a1665SEric Blake     assert(pnum);
2235298a1665SEric Blake     *pnum = 0;
22362e8bc787SEric Blake     total_size = bdrv_getlength(bs);
22372e8bc787SEric Blake     if (total_size < 0) {
22382e8bc787SEric Blake         ret = total_size;
2239298a1665SEric Blake         goto early_out;
224061007b31SStefan Hajnoczi     }
224161007b31SStefan Hajnoczi 
22422e8bc787SEric Blake     if (offset >= total_size) {
2243298a1665SEric Blake         ret = BDRV_BLOCK_EOF;
2244298a1665SEric Blake         goto early_out;
224561007b31SStefan Hajnoczi     }
22462e8bc787SEric Blake     if (!bytes) {
2247298a1665SEric Blake         ret = 0;
2248298a1665SEric Blake         goto early_out;
22499cdcfd9fSEric Blake     }
225061007b31SStefan Hajnoczi 
22512e8bc787SEric Blake     n = total_size - offset;
22522e8bc787SEric Blake     if (n < bytes) {
22532e8bc787SEric Blake         bytes = n;
225461007b31SStefan Hajnoczi     }
225561007b31SStefan Hajnoczi 
2256d470ad42SMax Reitz     /* Must be non-NULL or bdrv_getlength() would have failed */
2257d470ad42SMax Reitz     assert(bs->drv);
2258549ec0d9SMax Reitz     has_filtered_child = bdrv_filter_child(bs);
2259549ec0d9SMax Reitz     if (!bs->drv->bdrv_co_block_status && !has_filtered_child) {
22602e8bc787SEric Blake         *pnum = bytes;
226161007b31SStefan Hajnoczi         ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
22622e8bc787SEric Blake         if (offset + bytes == total_size) {
2263fb0d8654SEric Blake             ret |= BDRV_BLOCK_EOF;
2264fb0d8654SEric Blake         }
226561007b31SStefan Hajnoczi         if (bs->drv->protocol_name) {
22662e8bc787SEric Blake             ret |= BDRV_BLOCK_OFFSET_VALID;
22672e8bc787SEric Blake             local_map = offset;
2268298a1665SEric Blake             local_file = bs;
226961007b31SStefan Hajnoczi         }
2270298a1665SEric Blake         goto early_out;
227161007b31SStefan Hajnoczi     }
227261007b31SStefan Hajnoczi 
227399723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
2274efa6e2edSEric Blake 
2275efa6e2edSEric Blake     /* Round out to request_alignment boundaries */
227686a3d5c6SEric Blake     align = bs->bl.request_alignment;
2277efa6e2edSEric Blake     aligned_offset = QEMU_ALIGN_DOWN(offset, align);
2278efa6e2edSEric Blake     aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset;
2279efa6e2edSEric Blake 
2280549ec0d9SMax Reitz     if (bs->drv->bdrv_co_block_status) {
228186a3d5c6SEric Blake         ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset,
228286a3d5c6SEric Blake                                             aligned_bytes, pnum, &local_map,
228386a3d5c6SEric Blake                                             &local_file);
2284549ec0d9SMax Reitz     } else {
2285549ec0d9SMax Reitz         /* Default code for filters */
2286549ec0d9SMax Reitz 
2287549ec0d9SMax Reitz         local_file = bdrv_filter_bs(bs);
2288549ec0d9SMax Reitz         assert(local_file);
2289549ec0d9SMax Reitz 
2290549ec0d9SMax Reitz         *pnum = aligned_bytes;
2291549ec0d9SMax Reitz         local_map = aligned_offset;
2292549ec0d9SMax Reitz         ret = BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID;
2293549ec0d9SMax Reitz     }
229486a3d5c6SEric Blake     if (ret < 0) {
229586a3d5c6SEric Blake         *pnum = 0;
229686a3d5c6SEric Blake         goto out;
229786a3d5c6SEric Blake     }
2298efa6e2edSEric Blake 
2299efa6e2edSEric Blake     /*
2300636cb512SEric Blake      * The driver's result must be a non-zero multiple of request_alignment.
2301efa6e2edSEric Blake      * Clamp pnum and adjust map to original request.
2302efa6e2edSEric Blake      */
2303636cb512SEric Blake     assert(*pnum && QEMU_IS_ALIGNED(*pnum, align) &&
2304636cb512SEric Blake            align > offset - aligned_offset);
230569f47505SVladimir Sementsov-Ogievskiy     if (ret & BDRV_BLOCK_RECURSE) {
230669f47505SVladimir Sementsov-Ogievskiy         assert(ret & BDRV_BLOCK_DATA);
230769f47505SVladimir Sementsov-Ogievskiy         assert(ret & BDRV_BLOCK_OFFSET_VALID);
230869f47505SVladimir Sementsov-Ogievskiy         assert(!(ret & BDRV_BLOCK_ZERO));
230969f47505SVladimir Sementsov-Ogievskiy     }
231069f47505SVladimir Sementsov-Ogievskiy 
2311efa6e2edSEric Blake     *pnum -= offset - aligned_offset;
2312efa6e2edSEric Blake     if (*pnum > bytes) {
2313efa6e2edSEric Blake         *pnum = bytes;
2314efa6e2edSEric Blake     }
2315efa6e2edSEric Blake     if (ret & BDRV_BLOCK_OFFSET_VALID) {
2316efa6e2edSEric Blake         local_map += offset - aligned_offset;
2317efa6e2edSEric Blake     }
231861007b31SStefan Hajnoczi 
231961007b31SStefan Hajnoczi     if (ret & BDRV_BLOCK_RAW) {
2320298a1665SEric Blake         assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file);
23212e8bc787SEric Blake         ret = bdrv_co_block_status(local_file, want_zero, local_map,
23222e8bc787SEric Blake                                    *pnum, pnum, &local_map, &local_file);
232399723548SPaolo Bonzini         goto out;
232461007b31SStefan Hajnoczi     }
232561007b31SStefan Hajnoczi 
232661007b31SStefan Hajnoczi     if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
232761007b31SStefan Hajnoczi         ret |= BDRV_BLOCK_ALLOCATED;
2328d40f4a56SAlberto Garcia     } else if (bs->drv->supports_backing) {
2329cb850315SMax Reitz         BlockDriverState *cow_bs = bdrv_cow_bs(bs);
2330cb850315SMax Reitz 
2331d40f4a56SAlberto Garcia         if (!cow_bs) {
2332d40f4a56SAlberto Garcia             ret |= BDRV_BLOCK_ZERO;
2333d40f4a56SAlberto Garcia         } else if (want_zero) {
2334cb850315SMax Reitz             int64_t size2 = bdrv_getlength(cow_bs);
2335c9ce8c4dSEric Blake 
23362e8bc787SEric Blake             if (size2 >= 0 && offset >= size2) {
233761007b31SStefan Hajnoczi                 ret |= BDRV_BLOCK_ZERO;
233861007b31SStefan Hajnoczi             }
23397b1efe99SVladimir Sementsov-Ogievskiy         }
234061007b31SStefan Hajnoczi     }
234161007b31SStefan Hajnoczi 
234269f47505SVladimir Sementsov-Ogievskiy     if (want_zero && ret & BDRV_BLOCK_RECURSE &&
234369f47505SVladimir Sementsov-Ogievskiy         local_file && local_file != bs &&
234461007b31SStefan Hajnoczi         (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
234561007b31SStefan Hajnoczi         (ret & BDRV_BLOCK_OFFSET_VALID)) {
23462e8bc787SEric Blake         int64_t file_pnum;
23472e8bc787SEric Blake         int ret2;
234861007b31SStefan Hajnoczi 
23492e8bc787SEric Blake         ret2 = bdrv_co_block_status(local_file, want_zero, local_map,
23502e8bc787SEric Blake                                     *pnum, &file_pnum, NULL, NULL);
235161007b31SStefan Hajnoczi         if (ret2 >= 0) {
235261007b31SStefan Hajnoczi             /* Ignore errors.  This is just providing extra information, it
235361007b31SStefan Hajnoczi              * is useful but not necessary.
235461007b31SStefan Hajnoczi              */
2355c61e684eSEric Blake             if (ret2 & BDRV_BLOCK_EOF &&
2356c61e684eSEric Blake                 (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) {
2357c61e684eSEric Blake                 /*
2358c61e684eSEric Blake                  * It is valid for the format block driver to read
2359c61e684eSEric Blake                  * beyond the end of the underlying file's current
2360c61e684eSEric Blake                  * size; such areas read as zero.
2361c61e684eSEric Blake                  */
236261007b31SStefan Hajnoczi                 ret |= BDRV_BLOCK_ZERO;
236361007b31SStefan Hajnoczi             } else {
236461007b31SStefan Hajnoczi                 /* Limit request to the range reported by the protocol driver */
236561007b31SStefan Hajnoczi                 *pnum = file_pnum;
236661007b31SStefan Hajnoczi                 ret |= (ret2 & BDRV_BLOCK_ZERO);
236761007b31SStefan Hajnoczi             }
236861007b31SStefan Hajnoczi         }
236961007b31SStefan Hajnoczi     }
237061007b31SStefan Hajnoczi 
237199723548SPaolo Bonzini out:
237299723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
23732e8bc787SEric Blake     if (ret >= 0 && offset + *pnum == total_size) {
2374fb0d8654SEric Blake         ret |= BDRV_BLOCK_EOF;
2375fb0d8654SEric Blake     }
2376298a1665SEric Blake early_out:
2377298a1665SEric Blake     if (file) {
2378298a1665SEric Blake         *file = local_file;
2379298a1665SEric Blake     }
23802e8bc787SEric Blake     if (map) {
23812e8bc787SEric Blake         *map = local_map;
23822e8bc787SEric Blake     }
238361007b31SStefan Hajnoczi     return ret;
238461007b31SStefan Hajnoczi }
238561007b31SStefan Hajnoczi 
238621c2283eSVladimir Sementsov-Ogievskiy int coroutine_fn
2387f9e694cbSVladimir Sementsov-Ogievskiy bdrv_co_common_block_status_above(BlockDriverState *bs,
2388ba3f0e25SFam Zheng                                   BlockDriverState *base,
23893555a432SVladimir Sementsov-Ogievskiy                                   bool include_base,
2390c9ce8c4dSEric Blake                                   bool want_zero,
23915b648c67SEric Blake                                   int64_t offset,
23925b648c67SEric Blake                                   int64_t bytes,
23935b648c67SEric Blake                                   int64_t *pnum,
23945b648c67SEric Blake                                   int64_t *map,
2395a92b1b06SEric Blake                                   BlockDriverState **file,
2396a92b1b06SEric Blake                                   int *depth)
2397ba3f0e25SFam Zheng {
239867c095c8SVladimir Sementsov-Ogievskiy     int ret;
2399ba3f0e25SFam Zheng     BlockDriverState *p;
240067c095c8SVladimir Sementsov-Ogievskiy     int64_t eof = 0;
2401a92b1b06SEric Blake     int dummy;
2402ba3f0e25SFam Zheng 
24033555a432SVladimir Sementsov-Ogievskiy     assert(!include_base || base); /* Can't include NULL base */
240467c095c8SVladimir Sementsov-Ogievskiy 
2405a92b1b06SEric Blake     if (!depth) {
2406a92b1b06SEric Blake         depth = &dummy;
2407a92b1b06SEric Blake     }
2408a92b1b06SEric Blake     *depth = 0;
2409a92b1b06SEric Blake 
2410624f27bbSVladimir Sementsov-Ogievskiy     if (!include_base && bs == base) {
2411624f27bbSVladimir Sementsov-Ogievskiy         *pnum = bytes;
2412624f27bbSVladimir Sementsov-Ogievskiy         return 0;
2413624f27bbSVladimir Sementsov-Ogievskiy     }
2414624f27bbSVladimir Sementsov-Ogievskiy 
241567c095c8SVladimir Sementsov-Ogievskiy     ret = bdrv_co_block_status(bs, want_zero, offset, bytes, pnum, map, file);
2416a92b1b06SEric Blake     ++*depth;
24173555a432SVladimir Sementsov-Ogievskiy     if (ret < 0 || *pnum == 0 || ret & BDRV_BLOCK_ALLOCATED || bs == base) {
241867c095c8SVladimir Sementsov-Ogievskiy         return ret;
241967c095c8SVladimir Sementsov-Ogievskiy     }
242067c095c8SVladimir Sementsov-Ogievskiy 
242167c095c8SVladimir Sementsov-Ogievskiy     if (ret & BDRV_BLOCK_EOF) {
242267c095c8SVladimir Sementsov-Ogievskiy         eof = offset + *pnum;
242367c095c8SVladimir Sementsov-Ogievskiy     }
242467c095c8SVladimir Sementsov-Ogievskiy 
242567c095c8SVladimir Sementsov-Ogievskiy     assert(*pnum <= bytes);
242667c095c8SVladimir Sementsov-Ogievskiy     bytes = *pnum;
242767c095c8SVladimir Sementsov-Ogievskiy 
24283555a432SVladimir Sementsov-Ogievskiy     for (p = bdrv_filter_or_cow_bs(bs); include_base || p != base;
242967c095c8SVladimir Sementsov-Ogievskiy          p = bdrv_filter_or_cow_bs(p))
243067c095c8SVladimir Sementsov-Ogievskiy     {
24315b648c67SEric Blake         ret = bdrv_co_block_status(p, want_zero, offset, bytes, pnum, map,
24325b648c67SEric Blake                                    file);
2433a92b1b06SEric Blake         ++*depth;
2434c61e684eSEric Blake         if (ret < 0) {
243567c095c8SVladimir Sementsov-Ogievskiy             return ret;
2436c61e684eSEric Blake         }
243767c095c8SVladimir Sementsov-Ogievskiy         if (*pnum == 0) {
2438c61e684eSEric Blake             /*
243967c095c8SVladimir Sementsov-Ogievskiy              * The top layer deferred to this layer, and because this layer is
244067c095c8SVladimir Sementsov-Ogievskiy              * short, any zeroes that we synthesize beyond EOF behave as if they
244167c095c8SVladimir Sementsov-Ogievskiy              * were allocated at this layer.
244267c095c8SVladimir Sementsov-Ogievskiy              *
244367c095c8SVladimir Sementsov-Ogievskiy              * We don't include BDRV_BLOCK_EOF into ret, as upper layer may be
244467c095c8SVladimir Sementsov-Ogievskiy              * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see
244567c095c8SVladimir Sementsov-Ogievskiy              * below.
2446c61e684eSEric Blake              */
244767c095c8SVladimir Sementsov-Ogievskiy             assert(ret & BDRV_BLOCK_EOF);
24485b648c67SEric Blake             *pnum = bytes;
244967c095c8SVladimir Sementsov-Ogievskiy             if (file) {
245067c095c8SVladimir Sementsov-Ogievskiy                 *file = p;
2451c61e684eSEric Blake             }
245267c095c8SVladimir Sementsov-Ogievskiy             ret = BDRV_BLOCK_ZERO | BDRV_BLOCK_ALLOCATED;
2453ba3f0e25SFam Zheng             break;
2454ba3f0e25SFam Zheng         }
245567c095c8SVladimir Sementsov-Ogievskiy         if (ret & BDRV_BLOCK_ALLOCATED) {
245667c095c8SVladimir Sementsov-Ogievskiy             /*
245767c095c8SVladimir Sementsov-Ogievskiy              * We've found the node and the status, we must break.
245867c095c8SVladimir Sementsov-Ogievskiy              *
245967c095c8SVladimir Sementsov-Ogievskiy              * Drop BDRV_BLOCK_EOF, as it's not for upper layer, which may be
246067c095c8SVladimir Sementsov-Ogievskiy              * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see
246167c095c8SVladimir Sementsov-Ogievskiy              * below.
246267c095c8SVladimir Sementsov-Ogievskiy              */
246367c095c8SVladimir Sementsov-Ogievskiy             ret &= ~BDRV_BLOCK_EOF;
246467c095c8SVladimir Sementsov-Ogievskiy             break;
2465ba3f0e25SFam Zheng         }
246667c095c8SVladimir Sementsov-Ogievskiy 
24673555a432SVladimir Sementsov-Ogievskiy         if (p == base) {
24683555a432SVladimir Sementsov-Ogievskiy             assert(include_base);
24693555a432SVladimir Sementsov-Ogievskiy             break;
24703555a432SVladimir Sementsov-Ogievskiy         }
24713555a432SVladimir Sementsov-Ogievskiy 
247267c095c8SVladimir Sementsov-Ogievskiy         /*
247367c095c8SVladimir Sementsov-Ogievskiy          * OK, [offset, offset + *pnum) region is unallocated on this layer,
247467c095c8SVladimir Sementsov-Ogievskiy          * let's continue the diving.
247567c095c8SVladimir Sementsov-Ogievskiy          */
247667c095c8SVladimir Sementsov-Ogievskiy         assert(*pnum <= bytes);
247767c095c8SVladimir Sementsov-Ogievskiy         bytes = *pnum;
247867c095c8SVladimir Sementsov-Ogievskiy     }
247967c095c8SVladimir Sementsov-Ogievskiy 
248067c095c8SVladimir Sementsov-Ogievskiy     if (offset + *pnum == eof) {
248167c095c8SVladimir Sementsov-Ogievskiy         ret |= BDRV_BLOCK_EOF;
248267c095c8SVladimir Sementsov-Ogievskiy     }
248367c095c8SVladimir Sementsov-Ogievskiy 
2484ba3f0e25SFam Zheng     return ret;
2485ba3f0e25SFam Zheng }
2486ba3f0e25SFam Zheng 
248731826642SEric Blake int bdrv_block_status_above(BlockDriverState *bs, BlockDriverState *base,
248831826642SEric Blake                             int64_t offset, int64_t bytes, int64_t *pnum,
248931826642SEric Blake                             int64_t *map, BlockDriverState **file)
2490c9ce8c4dSEric Blake {
24913555a432SVladimir Sementsov-Ogievskiy     return bdrv_common_block_status_above(bs, base, false, true, offset, bytes,
2492a92b1b06SEric Blake                                           pnum, map, file, NULL);
2493c9ce8c4dSEric Blake }
2494c9ce8c4dSEric Blake 
2495237d78f8SEric Blake int bdrv_block_status(BlockDriverState *bs, int64_t offset, int64_t bytes,
2496237d78f8SEric Blake                       int64_t *pnum, int64_t *map, BlockDriverState **file)
2497ba3f0e25SFam Zheng {
2498cb850315SMax Reitz     return bdrv_block_status_above(bs, bdrv_filter_or_cow_bs(bs),
249931826642SEric Blake                                    offset, bytes, pnum, map, file);
2500ba3f0e25SFam Zheng }
2501ba3f0e25SFam Zheng 
250246cd1e8aSAlberto Garcia /*
250346cd1e8aSAlberto Garcia  * Check @bs (and its backing chain) to see if the range defined
250446cd1e8aSAlberto Garcia  * by @offset and @bytes is known to read as zeroes.
250546cd1e8aSAlberto Garcia  * Return 1 if that is the case, 0 otherwise and -errno on error.
250646cd1e8aSAlberto Garcia  * This test is meant to be fast rather than accurate so returning 0
250746cd1e8aSAlberto Garcia  * does not guarantee non-zero data.
250846cd1e8aSAlberto Garcia  */
250946cd1e8aSAlberto Garcia int coroutine_fn bdrv_co_is_zero_fast(BlockDriverState *bs, int64_t offset,
251046cd1e8aSAlberto Garcia                                       int64_t bytes)
251146cd1e8aSAlberto Garcia {
251246cd1e8aSAlberto Garcia     int ret;
251346cd1e8aSAlberto Garcia     int64_t pnum = bytes;
251446cd1e8aSAlberto Garcia 
251546cd1e8aSAlberto Garcia     if (!bytes) {
251646cd1e8aSAlberto Garcia         return 1;
251746cd1e8aSAlberto Garcia     }
251846cd1e8aSAlberto Garcia 
251946cd1e8aSAlberto Garcia     ret = bdrv_common_block_status_above(bs, NULL, false, false, offset,
2520a92b1b06SEric Blake                                          bytes, &pnum, NULL, NULL, NULL);
252146cd1e8aSAlberto Garcia 
252246cd1e8aSAlberto Garcia     if (ret < 0) {
252346cd1e8aSAlberto Garcia         return ret;
252446cd1e8aSAlberto Garcia     }
252546cd1e8aSAlberto Garcia 
252646cd1e8aSAlberto Garcia     return (pnum == bytes) && (ret & BDRV_BLOCK_ZERO);
252746cd1e8aSAlberto Garcia }
252846cd1e8aSAlberto Garcia 
2529d6a644bbSEric Blake int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t offset,
2530d6a644bbSEric Blake                                    int64_t bytes, int64_t *pnum)
253161007b31SStefan Hajnoczi {
25327ddb99b9SEric Blake     int ret;
25337ddb99b9SEric Blake     int64_t dummy;
2534d6a644bbSEric Blake 
25353555a432SVladimir Sementsov-Ogievskiy     ret = bdrv_common_block_status_above(bs, bs, true, false, offset,
25363555a432SVladimir Sementsov-Ogievskiy                                          bytes, pnum ? pnum : &dummy, NULL,
2537a92b1b06SEric Blake                                          NULL, NULL);
253861007b31SStefan Hajnoczi     if (ret < 0) {
253961007b31SStefan Hajnoczi         return ret;
254061007b31SStefan Hajnoczi     }
254161007b31SStefan Hajnoczi     return !!(ret & BDRV_BLOCK_ALLOCATED);
254261007b31SStefan Hajnoczi }
254361007b31SStefan Hajnoczi 
254461007b31SStefan Hajnoczi /*
254561007b31SStefan Hajnoczi  * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
254661007b31SStefan Hajnoczi  *
2547a92b1b06SEric Blake  * Return a positive depth if (a prefix of) the given range is allocated
2548a92b1b06SEric Blake  * in any image between BASE and TOP (BASE is only included if include_base
2549a92b1b06SEric Blake  * is set).  Depth 1 is TOP, 2 is the first backing layer, and so forth.
2550170d3bd3SAndrey Shinkevich  * BASE can be NULL to check if the given offset is allocated in any
2551170d3bd3SAndrey Shinkevich  * image of the chain.  Return 0 otherwise, or negative errno on
2552170d3bd3SAndrey Shinkevich  * failure.
255361007b31SStefan Hajnoczi  *
255451b0a488SEric Blake  * 'pnum' is set to the number of bytes (including and immediately
255551b0a488SEric Blake  * following the specified offset) that are known to be in the same
255651b0a488SEric Blake  * allocated/unallocated state.  Note that a subsequent call starting
255751b0a488SEric Blake  * at 'offset + *pnum' may return the same allocation status (in other
255851b0a488SEric Blake  * words, the result is not necessarily the maximum possible range);
255951b0a488SEric Blake  * but 'pnum' will only be 0 when end of file is reached.
256061007b31SStefan Hajnoczi  */
256161007b31SStefan Hajnoczi int bdrv_is_allocated_above(BlockDriverState *top,
256261007b31SStefan Hajnoczi                             BlockDriverState *base,
2563170d3bd3SAndrey Shinkevich                             bool include_base, int64_t offset,
2564170d3bd3SAndrey Shinkevich                             int64_t bytes, int64_t *pnum)
256561007b31SStefan Hajnoczi {
2566a92b1b06SEric Blake     int depth;
25677e7e5100SVladimir Sementsov-Ogievskiy     int ret = bdrv_common_block_status_above(top, base, include_base, false,
2568a92b1b06SEric Blake                                              offset, bytes, pnum, NULL, NULL,
2569a92b1b06SEric Blake                                              &depth);
257061007b31SStefan Hajnoczi     if (ret < 0) {
257161007b31SStefan Hajnoczi         return ret;
2572d6a644bbSEric Blake     }
257361007b31SStefan Hajnoczi 
2574a92b1b06SEric Blake     if (ret & BDRV_BLOCK_ALLOCATED) {
2575a92b1b06SEric Blake         return depth;
2576a92b1b06SEric Blake     }
2577a92b1b06SEric Blake     return 0;
257861007b31SStefan Hajnoczi }
257961007b31SStefan Hajnoczi 
258021c2283eSVladimir Sementsov-Ogievskiy int coroutine_fn
2581b33b354fSVladimir Sementsov-Ogievskiy bdrv_co_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
25821a8ae822SKevin Wolf {
25831a8ae822SKevin Wolf     BlockDriver *drv = bs->drv;
2584c4db2e25SMax Reitz     BlockDriverState *child_bs = bdrv_primary_bs(bs);
2585dc88a467SStefan Hajnoczi     int ret = -ENOTSUP;
2586dc88a467SStefan Hajnoczi 
2587b33b354fSVladimir Sementsov-Ogievskiy     if (!drv) {
2588b33b354fSVladimir Sementsov-Ogievskiy         return -ENOMEDIUM;
2589b33b354fSVladimir Sementsov-Ogievskiy     }
2590b33b354fSVladimir Sementsov-Ogievskiy 
2591dc88a467SStefan Hajnoczi     bdrv_inc_in_flight(bs);
25921a8ae822SKevin Wolf 
2593b33b354fSVladimir Sementsov-Ogievskiy     if (drv->bdrv_load_vmstate) {
2594dc88a467SStefan Hajnoczi         ret = drv->bdrv_load_vmstate(bs, qiov, pos);
2595c4db2e25SMax Reitz     } else if (child_bs) {
2596b33b354fSVladimir Sementsov-Ogievskiy         ret = bdrv_co_readv_vmstate(child_bs, qiov, pos);
25971a8ae822SKevin Wolf     }
25981a8ae822SKevin Wolf 
2599dc88a467SStefan Hajnoczi     bdrv_dec_in_flight(bs);
2600b33b354fSVladimir Sementsov-Ogievskiy 
2601b33b354fSVladimir Sementsov-Ogievskiy     return ret;
2602b33b354fSVladimir Sementsov-Ogievskiy }
2603b33b354fSVladimir Sementsov-Ogievskiy 
2604b33b354fSVladimir Sementsov-Ogievskiy int coroutine_fn
2605b33b354fSVladimir Sementsov-Ogievskiy bdrv_co_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
2606b33b354fSVladimir Sementsov-Ogievskiy {
2607b33b354fSVladimir Sementsov-Ogievskiy     BlockDriver *drv = bs->drv;
2608b33b354fSVladimir Sementsov-Ogievskiy     BlockDriverState *child_bs = bdrv_primary_bs(bs);
2609b33b354fSVladimir Sementsov-Ogievskiy     int ret = -ENOTSUP;
2610b33b354fSVladimir Sementsov-Ogievskiy 
2611b33b354fSVladimir Sementsov-Ogievskiy     if (!drv) {
2612b33b354fSVladimir Sementsov-Ogievskiy         return -ENOMEDIUM;
2613b33b354fSVladimir Sementsov-Ogievskiy     }
2614b33b354fSVladimir Sementsov-Ogievskiy 
2615b33b354fSVladimir Sementsov-Ogievskiy     bdrv_inc_in_flight(bs);
2616b33b354fSVladimir Sementsov-Ogievskiy 
2617b33b354fSVladimir Sementsov-Ogievskiy     if (drv->bdrv_save_vmstate) {
2618b33b354fSVladimir Sementsov-Ogievskiy         ret = drv->bdrv_save_vmstate(bs, qiov, pos);
2619b33b354fSVladimir Sementsov-Ogievskiy     } else if (child_bs) {
2620b33b354fSVladimir Sementsov-Ogievskiy         ret = bdrv_co_writev_vmstate(child_bs, qiov, pos);
2621b33b354fSVladimir Sementsov-Ogievskiy     }
2622b33b354fSVladimir Sementsov-Ogievskiy 
2623b33b354fSVladimir Sementsov-Ogievskiy     bdrv_dec_in_flight(bs);
2624b33b354fSVladimir Sementsov-Ogievskiy 
2625dc88a467SStefan Hajnoczi     return ret;
26261a8ae822SKevin Wolf }
26271a8ae822SKevin Wolf 
262861007b31SStefan Hajnoczi int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
262961007b31SStefan Hajnoczi                       int64_t pos, int size)
263061007b31SStefan Hajnoczi {
26310d93ed08SVladimir Sementsov-Ogievskiy     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size);
2632b33b354fSVladimir Sementsov-Ogievskiy     int ret = bdrv_writev_vmstate(bs, &qiov, pos);
263361007b31SStefan Hajnoczi 
2634b33b354fSVladimir Sementsov-Ogievskiy     return ret < 0 ? ret : size;
263561007b31SStefan Hajnoczi }
263661007b31SStefan Hajnoczi 
263761007b31SStefan Hajnoczi int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
263861007b31SStefan Hajnoczi                       int64_t pos, int size)
263961007b31SStefan Hajnoczi {
26400d93ed08SVladimir Sementsov-Ogievskiy     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size);
2641b33b354fSVladimir Sementsov-Ogievskiy     int ret = bdrv_readv_vmstate(bs, &qiov, pos);
26425ddda0b8SKevin Wolf 
2643b33b354fSVladimir Sementsov-Ogievskiy     return ret < 0 ? ret : size;
264461007b31SStefan Hajnoczi }
264561007b31SStefan Hajnoczi 
264661007b31SStefan Hajnoczi /**************************************************************/
264761007b31SStefan Hajnoczi /* async I/Os */
264861007b31SStefan Hajnoczi 
264961007b31SStefan Hajnoczi void bdrv_aio_cancel(BlockAIOCB *acb)
265061007b31SStefan Hajnoczi {
265161007b31SStefan Hajnoczi     qemu_aio_ref(acb);
265261007b31SStefan Hajnoczi     bdrv_aio_cancel_async(acb);
265361007b31SStefan Hajnoczi     while (acb->refcnt > 1) {
265461007b31SStefan Hajnoczi         if (acb->aiocb_info->get_aio_context) {
265561007b31SStefan Hajnoczi             aio_poll(acb->aiocb_info->get_aio_context(acb), true);
265661007b31SStefan Hajnoczi         } else if (acb->bs) {
26572f47da5fSPaolo Bonzini             /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so
26582f47da5fSPaolo Bonzini              * assert that we're not using an I/O thread.  Thread-safe
26592f47da5fSPaolo Bonzini              * code should use bdrv_aio_cancel_async exclusively.
26602f47da5fSPaolo Bonzini              */
26612f47da5fSPaolo Bonzini             assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context());
266261007b31SStefan Hajnoczi             aio_poll(bdrv_get_aio_context(acb->bs), true);
266361007b31SStefan Hajnoczi         } else {
266461007b31SStefan Hajnoczi             abort();
266561007b31SStefan Hajnoczi         }
266661007b31SStefan Hajnoczi     }
266761007b31SStefan Hajnoczi     qemu_aio_unref(acb);
266861007b31SStefan Hajnoczi }
266961007b31SStefan Hajnoczi 
267061007b31SStefan Hajnoczi /* Async version of aio cancel. The caller is not blocked if the acb implements
267161007b31SStefan Hajnoczi  * cancel_async, otherwise we do nothing and let the request normally complete.
267261007b31SStefan Hajnoczi  * In either case the completion callback must be called. */
267361007b31SStefan Hajnoczi void bdrv_aio_cancel_async(BlockAIOCB *acb)
267461007b31SStefan Hajnoczi {
267561007b31SStefan Hajnoczi     if (acb->aiocb_info->cancel_async) {
267661007b31SStefan Hajnoczi         acb->aiocb_info->cancel_async(acb);
267761007b31SStefan Hajnoczi     }
267861007b31SStefan Hajnoczi }
267961007b31SStefan Hajnoczi 
268061007b31SStefan Hajnoczi /**************************************************************/
268161007b31SStefan Hajnoczi /* Coroutine block device emulation */
268261007b31SStefan Hajnoczi 
268361007b31SStefan Hajnoczi int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
268461007b31SStefan Hajnoczi {
2685883833e2SMax Reitz     BdrvChild *primary_child = bdrv_primary_child(bs);
2686883833e2SMax Reitz     BdrvChild *child;
268749ca6259SFam Zheng     int current_gen;
268849ca6259SFam Zheng     int ret = 0;
268961007b31SStefan Hajnoczi 
269099723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
2691c32b82afSPavel Dovgalyuk 
2692e914404eSFam Zheng     if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
269349ca6259SFam Zheng         bdrv_is_sg(bs)) {
269449ca6259SFam Zheng         goto early_exit;
269549ca6259SFam Zheng     }
269649ca6259SFam Zheng 
26973783fa3dSPaolo Bonzini     qemu_co_mutex_lock(&bs->reqs_lock);
2698d73415a3SStefan Hajnoczi     current_gen = qatomic_read(&bs->write_gen);
26993ff2f67aSEvgeny Yakovlev 
27003ff2f67aSEvgeny Yakovlev     /* Wait until any previous flushes are completed */
270199723548SPaolo Bonzini     while (bs->active_flush_req) {
27023783fa3dSPaolo Bonzini         qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock);
27033ff2f67aSEvgeny Yakovlev     }
27043ff2f67aSEvgeny Yakovlev 
27053783fa3dSPaolo Bonzini     /* Flushes reach this point in nondecreasing current_gen order.  */
270699723548SPaolo Bonzini     bs->active_flush_req = true;
27073783fa3dSPaolo Bonzini     qemu_co_mutex_unlock(&bs->reqs_lock);
27083ff2f67aSEvgeny Yakovlev 
2709c32b82afSPavel Dovgalyuk     /* Write back all layers by calling one driver function */
2710c32b82afSPavel Dovgalyuk     if (bs->drv->bdrv_co_flush) {
2711c32b82afSPavel Dovgalyuk         ret = bs->drv->bdrv_co_flush(bs);
2712c32b82afSPavel Dovgalyuk         goto out;
2713c32b82afSPavel Dovgalyuk     }
2714c32b82afSPavel Dovgalyuk 
271561007b31SStefan Hajnoczi     /* Write back cached data to the OS even with cache=unsafe */
2716883833e2SMax Reitz     BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_OS);
271761007b31SStefan Hajnoczi     if (bs->drv->bdrv_co_flush_to_os) {
271861007b31SStefan Hajnoczi         ret = bs->drv->bdrv_co_flush_to_os(bs);
271961007b31SStefan Hajnoczi         if (ret < 0) {
2720cdb5e315SFam Zheng             goto out;
272161007b31SStefan Hajnoczi         }
272261007b31SStefan Hajnoczi     }
272361007b31SStefan Hajnoczi 
272461007b31SStefan Hajnoczi     /* But don't actually force it to the disk with cache=unsafe */
272561007b31SStefan Hajnoczi     if (bs->open_flags & BDRV_O_NO_FLUSH) {
2726883833e2SMax Reitz         goto flush_children;
272761007b31SStefan Hajnoczi     }
272861007b31SStefan Hajnoczi 
27293ff2f67aSEvgeny Yakovlev     /* Check if we really need to flush anything */
27303ff2f67aSEvgeny Yakovlev     if (bs->flushed_gen == current_gen) {
2731883833e2SMax Reitz         goto flush_children;
27323ff2f67aSEvgeny Yakovlev     }
27333ff2f67aSEvgeny Yakovlev 
2734883833e2SMax Reitz     BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_DISK);
2735d470ad42SMax Reitz     if (!bs->drv) {
2736d470ad42SMax Reitz         /* bs->drv->bdrv_co_flush() might have ejected the BDS
2737d470ad42SMax Reitz          * (even in case of apparent success) */
2738d470ad42SMax Reitz         ret = -ENOMEDIUM;
2739d470ad42SMax Reitz         goto out;
2740d470ad42SMax Reitz     }
274161007b31SStefan Hajnoczi     if (bs->drv->bdrv_co_flush_to_disk) {
274261007b31SStefan Hajnoczi         ret = bs->drv->bdrv_co_flush_to_disk(bs);
274361007b31SStefan Hajnoczi     } else if (bs->drv->bdrv_aio_flush) {
274461007b31SStefan Hajnoczi         BlockAIOCB *acb;
274561007b31SStefan Hajnoczi         CoroutineIOCompletion co = {
274661007b31SStefan Hajnoczi             .coroutine = qemu_coroutine_self(),
274761007b31SStefan Hajnoczi         };
274861007b31SStefan Hajnoczi 
274961007b31SStefan Hajnoczi         acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
275061007b31SStefan Hajnoczi         if (acb == NULL) {
275161007b31SStefan Hajnoczi             ret = -EIO;
275261007b31SStefan Hajnoczi         } else {
275361007b31SStefan Hajnoczi             qemu_coroutine_yield();
275461007b31SStefan Hajnoczi             ret = co.ret;
275561007b31SStefan Hajnoczi         }
275661007b31SStefan Hajnoczi     } else {
275761007b31SStefan Hajnoczi         /*
275861007b31SStefan Hajnoczi          * Some block drivers always operate in either writethrough or unsafe
275961007b31SStefan Hajnoczi          * mode and don't support bdrv_flush therefore. Usually qemu doesn't
276061007b31SStefan Hajnoczi          * know how the server works (because the behaviour is hardcoded or
276161007b31SStefan Hajnoczi          * depends on server-side configuration), so we can't ensure that
276261007b31SStefan Hajnoczi          * everything is safe on disk. Returning an error doesn't work because
276361007b31SStefan Hajnoczi          * that would break guests even if the server operates in writethrough
276461007b31SStefan Hajnoczi          * mode.
276561007b31SStefan Hajnoczi          *
276661007b31SStefan Hajnoczi          * Let's hope the user knows what he's doing.
276761007b31SStefan Hajnoczi          */
276861007b31SStefan Hajnoczi         ret = 0;
276961007b31SStefan Hajnoczi     }
27703ff2f67aSEvgeny Yakovlev 
277161007b31SStefan Hajnoczi     if (ret < 0) {
2772cdb5e315SFam Zheng         goto out;
277361007b31SStefan Hajnoczi     }
277461007b31SStefan Hajnoczi 
277561007b31SStefan Hajnoczi     /* Now flush the underlying protocol.  It will also have BDRV_O_NO_FLUSH
277661007b31SStefan Hajnoczi      * in the case of cache=unsafe, so there are no useless flushes.
277761007b31SStefan Hajnoczi      */
2778883833e2SMax Reitz flush_children:
2779883833e2SMax Reitz     ret = 0;
2780883833e2SMax Reitz     QLIST_FOREACH(child, &bs->children, next) {
2781883833e2SMax Reitz         if (child->perm & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED)) {
2782883833e2SMax Reitz             int this_child_ret = bdrv_co_flush(child->bs);
2783883833e2SMax Reitz             if (!ret) {
2784883833e2SMax Reitz                 ret = this_child_ret;
2785883833e2SMax Reitz             }
2786883833e2SMax Reitz         }
2787883833e2SMax Reitz     }
2788883833e2SMax Reitz 
2789cdb5e315SFam Zheng out:
27903ff2f67aSEvgeny Yakovlev     /* Notify any pending flushes that we have completed */
2791e6af1e08SKevin Wolf     if (ret == 0) {
27923ff2f67aSEvgeny Yakovlev         bs->flushed_gen = current_gen;
2793e6af1e08SKevin Wolf     }
27943783fa3dSPaolo Bonzini 
27953783fa3dSPaolo Bonzini     qemu_co_mutex_lock(&bs->reqs_lock);
279699723548SPaolo Bonzini     bs->active_flush_req = false;
2797156af3acSDenis V. Lunev     /* Return value is ignored - it's ok if wait queue is empty */
2798156af3acSDenis V. Lunev     qemu_co_queue_next(&bs->flush_queue);
27993783fa3dSPaolo Bonzini     qemu_co_mutex_unlock(&bs->reqs_lock);
28003ff2f67aSEvgeny Yakovlev 
280149ca6259SFam Zheng early_exit:
280299723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
2803cdb5e315SFam Zheng     return ret;
280461007b31SStefan Hajnoczi }
280561007b31SStefan Hajnoczi 
2806d93e5726SVladimir Sementsov-Ogievskiy int coroutine_fn bdrv_co_pdiscard(BdrvChild *child, int64_t offset,
2807d93e5726SVladimir Sementsov-Ogievskiy                                   int64_t bytes)
280861007b31SStefan Hajnoczi {
2809b1066c87SFam Zheng     BdrvTrackedRequest req;
28109f1963b3SEric Blake     int max_pdiscard, ret;
28113482b9bcSEric Blake     int head, tail, align;
28120b9fd3f4SFam Zheng     BlockDriverState *bs = child->bs;
281361007b31SStefan Hajnoczi 
2814d93e5726SVladimir Sementsov-Ogievskiy     if (!bs || !bs->drv || !bdrv_is_inserted(bs)) {
281561007b31SStefan Hajnoczi         return -ENOMEDIUM;
281661007b31SStefan Hajnoczi     }
281761007b31SStefan Hajnoczi 
2818d6883bc9SVladimir Sementsov-Ogievskiy     if (bdrv_has_readonly_bitmaps(bs)) {
2819d6883bc9SVladimir Sementsov-Ogievskiy         return -EPERM;
2820d6883bc9SVladimir Sementsov-Ogievskiy     }
2821d6883bc9SVladimir Sementsov-Ogievskiy 
28228b117001SVladimir Sementsov-Ogievskiy     ret = bdrv_check_request(offset, bytes);
28238b117001SVladimir Sementsov-Ogievskiy     if (ret < 0) {
28248b117001SVladimir Sementsov-Ogievskiy         return ret;
282561007b31SStefan Hajnoczi     }
282661007b31SStefan Hajnoczi 
282761007b31SStefan Hajnoczi     /* Do nothing if disabled.  */
282861007b31SStefan Hajnoczi     if (!(bs->open_flags & BDRV_O_UNMAP)) {
282961007b31SStefan Hajnoczi         return 0;
283061007b31SStefan Hajnoczi     }
283161007b31SStefan Hajnoczi 
283202aefe43SEric Blake     if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) {
283361007b31SStefan Hajnoczi         return 0;
283461007b31SStefan Hajnoczi     }
283561007b31SStefan Hajnoczi 
28363482b9bcSEric Blake     /* Discard is advisory, but some devices track and coalesce
28373482b9bcSEric Blake      * unaligned requests, so we must pass everything down rather than
28383482b9bcSEric Blake      * round here.  Still, most devices will just silently ignore
28393482b9bcSEric Blake      * unaligned requests (by returning -ENOTSUP), so we must fragment
28403482b9bcSEric Blake      * the request accordingly.  */
284102aefe43SEric Blake     align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment);
2842b8d0a980SEric Blake     assert(align % bs->bl.request_alignment == 0);
2843b8d0a980SEric Blake     head = offset % align;
2844f5a5ca79SManos Pitsidianakis     tail = (offset + bytes) % align;
28459f1963b3SEric Blake 
284699723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
2847f5a5ca79SManos Pitsidianakis     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD);
284850824995SFam Zheng 
284900695c27SFam Zheng     ret = bdrv_co_write_req_prepare(child, offset, bytes, &req, 0);
2850ec050f77SDenis V. Lunev     if (ret < 0) {
2851ec050f77SDenis V. Lunev         goto out;
2852ec050f77SDenis V. Lunev     }
2853ec050f77SDenis V. Lunev 
28549f1963b3SEric Blake     max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT_MAX),
28559f1963b3SEric Blake                                    align);
28563482b9bcSEric Blake     assert(max_pdiscard >= bs->bl.request_alignment);
28579f1963b3SEric Blake 
2858f5a5ca79SManos Pitsidianakis     while (bytes > 0) {
2859d93e5726SVladimir Sementsov-Ogievskiy         int64_t num = bytes;
28603482b9bcSEric Blake 
28613482b9bcSEric Blake         if (head) {
28623482b9bcSEric Blake             /* Make small requests to get to alignment boundaries. */
2863f5a5ca79SManos Pitsidianakis             num = MIN(bytes, align - head);
28643482b9bcSEric Blake             if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) {
28653482b9bcSEric Blake                 num %= bs->bl.request_alignment;
28663482b9bcSEric Blake             }
28673482b9bcSEric Blake             head = (head + num) % align;
28683482b9bcSEric Blake             assert(num < max_pdiscard);
28693482b9bcSEric Blake         } else if (tail) {
28703482b9bcSEric Blake             if (num > align) {
28713482b9bcSEric Blake                 /* Shorten the request to the last aligned cluster.  */
28723482b9bcSEric Blake                 num -= tail;
28733482b9bcSEric Blake             } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) &&
28743482b9bcSEric Blake                        tail > bs->bl.request_alignment) {
28753482b9bcSEric Blake                 tail %= bs->bl.request_alignment;
28763482b9bcSEric Blake                 num -= tail;
28773482b9bcSEric Blake             }
28783482b9bcSEric Blake         }
28793482b9bcSEric Blake         /* limit request size */
28803482b9bcSEric Blake         if (num > max_pdiscard) {
28813482b9bcSEric Blake             num = max_pdiscard;
28823482b9bcSEric Blake         }
288361007b31SStefan Hajnoczi 
2884d470ad42SMax Reitz         if (!bs->drv) {
2885d470ad42SMax Reitz             ret = -ENOMEDIUM;
2886d470ad42SMax Reitz             goto out;
2887d470ad42SMax Reitz         }
288847a5486dSEric Blake         if (bs->drv->bdrv_co_pdiscard) {
288947a5486dSEric Blake             ret = bs->drv->bdrv_co_pdiscard(bs, offset, num);
289061007b31SStefan Hajnoczi         } else {
289161007b31SStefan Hajnoczi             BlockAIOCB *acb;
289261007b31SStefan Hajnoczi             CoroutineIOCompletion co = {
289361007b31SStefan Hajnoczi                 .coroutine = qemu_coroutine_self(),
289461007b31SStefan Hajnoczi             };
289561007b31SStefan Hajnoczi 
28964da444a0SEric Blake             acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num,
289761007b31SStefan Hajnoczi                                              bdrv_co_io_em_complete, &co);
289861007b31SStefan Hajnoczi             if (acb == NULL) {
2899b1066c87SFam Zheng                 ret = -EIO;
2900b1066c87SFam Zheng                 goto out;
290161007b31SStefan Hajnoczi             } else {
290261007b31SStefan Hajnoczi                 qemu_coroutine_yield();
290361007b31SStefan Hajnoczi                 ret = co.ret;
290461007b31SStefan Hajnoczi             }
290561007b31SStefan Hajnoczi         }
290661007b31SStefan Hajnoczi         if (ret && ret != -ENOTSUP) {
2907b1066c87SFam Zheng             goto out;
290861007b31SStefan Hajnoczi         }
290961007b31SStefan Hajnoczi 
29109f1963b3SEric Blake         offset += num;
2911f5a5ca79SManos Pitsidianakis         bytes -= num;
291261007b31SStefan Hajnoczi     }
2913b1066c87SFam Zheng     ret = 0;
2914b1066c87SFam Zheng out:
291500695c27SFam Zheng     bdrv_co_write_req_finish(child, req.offset, req.bytes, &req, ret);
2916b1066c87SFam Zheng     tracked_request_end(&req);
291799723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
2918b1066c87SFam Zheng     return ret;
291961007b31SStefan Hajnoczi }
292061007b31SStefan Hajnoczi 
292148af776aSKevin Wolf int bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf)
292261007b31SStefan Hajnoczi {
292361007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
29245c5ae76aSFam Zheng     CoroutineIOCompletion co = {
29255c5ae76aSFam Zheng         .coroutine = qemu_coroutine_self(),
29265c5ae76aSFam Zheng     };
29275c5ae76aSFam Zheng     BlockAIOCB *acb;
292861007b31SStefan Hajnoczi 
292999723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
293016a389dcSKevin Wolf     if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) {
29315c5ae76aSFam Zheng         co.ret = -ENOTSUP;
29325c5ae76aSFam Zheng         goto out;
29335c5ae76aSFam Zheng     }
29345c5ae76aSFam Zheng 
293516a389dcSKevin Wolf     if (drv->bdrv_co_ioctl) {
293616a389dcSKevin Wolf         co.ret = drv->bdrv_co_ioctl(bs, req, buf);
293716a389dcSKevin Wolf     } else {
29385c5ae76aSFam Zheng         acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
29395c5ae76aSFam Zheng         if (!acb) {
2940c8a9fd80SFam Zheng             co.ret = -ENOTSUP;
2941c8a9fd80SFam Zheng             goto out;
29425c5ae76aSFam Zheng         }
29435c5ae76aSFam Zheng         qemu_coroutine_yield();
294416a389dcSKevin Wolf     }
29455c5ae76aSFam Zheng out:
294699723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
29475c5ae76aSFam Zheng     return co.ret;
29485c5ae76aSFam Zheng }
29495c5ae76aSFam Zheng 
295061007b31SStefan Hajnoczi void *qemu_blockalign(BlockDriverState *bs, size_t size)
295161007b31SStefan Hajnoczi {
295261007b31SStefan Hajnoczi     return qemu_memalign(bdrv_opt_mem_align(bs), size);
295361007b31SStefan Hajnoczi }
295461007b31SStefan Hajnoczi 
295561007b31SStefan Hajnoczi void *qemu_blockalign0(BlockDriverState *bs, size_t size)
295661007b31SStefan Hajnoczi {
295761007b31SStefan Hajnoczi     return memset(qemu_blockalign(bs, size), 0, size);
295861007b31SStefan Hajnoczi }
295961007b31SStefan Hajnoczi 
296061007b31SStefan Hajnoczi void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
296161007b31SStefan Hajnoczi {
296261007b31SStefan Hajnoczi     size_t align = bdrv_opt_mem_align(bs);
296361007b31SStefan Hajnoczi 
296461007b31SStefan Hajnoczi     /* Ensure that NULL is never returned on success */
296561007b31SStefan Hajnoczi     assert(align > 0);
296661007b31SStefan Hajnoczi     if (size == 0) {
296761007b31SStefan Hajnoczi         size = align;
296861007b31SStefan Hajnoczi     }
296961007b31SStefan Hajnoczi 
297061007b31SStefan Hajnoczi     return qemu_try_memalign(align, size);
297161007b31SStefan Hajnoczi }
297261007b31SStefan Hajnoczi 
297361007b31SStefan Hajnoczi void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
297461007b31SStefan Hajnoczi {
297561007b31SStefan Hajnoczi     void *mem = qemu_try_blockalign(bs, size);
297661007b31SStefan Hajnoczi 
297761007b31SStefan Hajnoczi     if (mem) {
297861007b31SStefan Hajnoczi         memset(mem, 0, size);
297961007b31SStefan Hajnoczi     }
298061007b31SStefan Hajnoczi 
298161007b31SStefan Hajnoczi     return mem;
298261007b31SStefan Hajnoczi }
298361007b31SStefan Hajnoczi 
298461007b31SStefan Hajnoczi /*
298561007b31SStefan Hajnoczi  * Check if all memory in this vector is sector aligned.
298661007b31SStefan Hajnoczi  */
298761007b31SStefan Hajnoczi bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
298861007b31SStefan Hajnoczi {
298961007b31SStefan Hajnoczi     int i;
29904196d2f0SDenis V. Lunev     size_t alignment = bdrv_min_mem_align(bs);
299161007b31SStefan Hajnoczi 
299261007b31SStefan Hajnoczi     for (i = 0; i < qiov->niov; i++) {
299361007b31SStefan Hajnoczi         if ((uintptr_t) qiov->iov[i].iov_base % alignment) {
299461007b31SStefan Hajnoczi             return false;
299561007b31SStefan Hajnoczi         }
299661007b31SStefan Hajnoczi         if (qiov->iov[i].iov_len % alignment) {
299761007b31SStefan Hajnoczi             return false;
299861007b31SStefan Hajnoczi         }
299961007b31SStefan Hajnoczi     }
300061007b31SStefan Hajnoczi 
300161007b31SStefan Hajnoczi     return true;
300261007b31SStefan Hajnoczi }
300361007b31SStefan Hajnoczi 
300461007b31SStefan Hajnoczi void bdrv_add_before_write_notifier(BlockDriverState *bs,
300561007b31SStefan Hajnoczi                                     NotifierWithReturn *notifier)
300661007b31SStefan Hajnoczi {
300761007b31SStefan Hajnoczi     notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
300861007b31SStefan Hajnoczi }
300961007b31SStefan Hajnoczi 
301061007b31SStefan Hajnoczi void bdrv_io_plug(BlockDriverState *bs)
301161007b31SStefan Hajnoczi {
30126b98bd64SPaolo Bonzini     BdrvChild *child;
30136b98bd64SPaolo Bonzini 
30146b98bd64SPaolo Bonzini     QLIST_FOREACH(child, &bs->children, next) {
30156b98bd64SPaolo Bonzini         bdrv_io_plug(child->bs);
30166b98bd64SPaolo Bonzini     }
30176b98bd64SPaolo Bonzini 
3018d73415a3SStefan Hajnoczi     if (qatomic_fetch_inc(&bs->io_plugged) == 0) {
301961007b31SStefan Hajnoczi         BlockDriver *drv = bs->drv;
302061007b31SStefan Hajnoczi         if (drv && drv->bdrv_io_plug) {
302161007b31SStefan Hajnoczi             drv->bdrv_io_plug(bs);
30226b98bd64SPaolo Bonzini         }
302361007b31SStefan Hajnoczi     }
302461007b31SStefan Hajnoczi }
302561007b31SStefan Hajnoczi 
302661007b31SStefan Hajnoczi void bdrv_io_unplug(BlockDriverState *bs)
302761007b31SStefan Hajnoczi {
30286b98bd64SPaolo Bonzini     BdrvChild *child;
30296b98bd64SPaolo Bonzini 
30306b98bd64SPaolo Bonzini     assert(bs->io_plugged);
3031d73415a3SStefan Hajnoczi     if (qatomic_fetch_dec(&bs->io_plugged) == 1) {
303261007b31SStefan Hajnoczi         BlockDriver *drv = bs->drv;
303361007b31SStefan Hajnoczi         if (drv && drv->bdrv_io_unplug) {
303461007b31SStefan Hajnoczi             drv->bdrv_io_unplug(bs);
303561007b31SStefan Hajnoczi         }
303661007b31SStefan Hajnoczi     }
303761007b31SStefan Hajnoczi 
30386b98bd64SPaolo Bonzini     QLIST_FOREACH(child, &bs->children, next) {
30396b98bd64SPaolo Bonzini         bdrv_io_unplug(child->bs);
30406b98bd64SPaolo Bonzini     }
30416b98bd64SPaolo Bonzini }
304223d0ba93SFam Zheng 
304323d0ba93SFam Zheng void bdrv_register_buf(BlockDriverState *bs, void *host, size_t size)
304423d0ba93SFam Zheng {
304523d0ba93SFam Zheng     BdrvChild *child;
304623d0ba93SFam Zheng 
304723d0ba93SFam Zheng     if (bs->drv && bs->drv->bdrv_register_buf) {
304823d0ba93SFam Zheng         bs->drv->bdrv_register_buf(bs, host, size);
304923d0ba93SFam Zheng     }
305023d0ba93SFam Zheng     QLIST_FOREACH(child, &bs->children, next) {
305123d0ba93SFam Zheng         bdrv_register_buf(child->bs, host, size);
305223d0ba93SFam Zheng     }
305323d0ba93SFam Zheng }
305423d0ba93SFam Zheng 
305523d0ba93SFam Zheng void bdrv_unregister_buf(BlockDriverState *bs, void *host)
305623d0ba93SFam Zheng {
305723d0ba93SFam Zheng     BdrvChild *child;
305823d0ba93SFam Zheng 
305923d0ba93SFam Zheng     if (bs->drv && bs->drv->bdrv_unregister_buf) {
306023d0ba93SFam Zheng         bs->drv->bdrv_unregister_buf(bs, host);
306123d0ba93SFam Zheng     }
306223d0ba93SFam Zheng     QLIST_FOREACH(child, &bs->children, next) {
306323d0ba93SFam Zheng         bdrv_unregister_buf(child->bs, host);
306423d0ba93SFam Zheng     }
306523d0ba93SFam Zheng }
3066fcc67678SFam Zheng 
306767b51fb9SVladimir Sementsov-Ogievskiy static int coroutine_fn bdrv_co_copy_range_internal(
306867b51fb9SVladimir Sementsov-Ogievskiy         BdrvChild *src, uint64_t src_offset, BdrvChild *dst,
306967b51fb9SVladimir Sementsov-Ogievskiy         uint64_t dst_offset, uint64_t bytes,
307067b51fb9SVladimir Sementsov-Ogievskiy         BdrvRequestFlags read_flags, BdrvRequestFlags write_flags,
3071fcc67678SFam Zheng         bool recurse_src)
3072fcc67678SFam Zheng {
3073999658a0SVladimir Sementsov-Ogievskiy     BdrvTrackedRequest req;
3074fcc67678SFam Zheng     int ret;
3075fcc67678SFam Zheng 
3076fe0480d6SKevin Wolf     /* TODO We can support BDRV_REQ_NO_FALLBACK here */
3077fe0480d6SKevin Wolf     assert(!(read_flags & BDRV_REQ_NO_FALLBACK));
3078fe0480d6SKevin Wolf     assert(!(write_flags & BDRV_REQ_NO_FALLBACK));
3079fe0480d6SKevin Wolf 
3080f4dad307SVladimir Sementsov-Ogievskiy     if (!dst || !dst->bs || !bdrv_is_inserted(dst->bs)) {
3081fcc67678SFam Zheng         return -ENOMEDIUM;
3082fcc67678SFam Zheng     }
30838b117001SVladimir Sementsov-Ogievskiy     ret = bdrv_check_request32(dst_offset, bytes);
3084fcc67678SFam Zheng     if (ret) {
3085fcc67678SFam Zheng         return ret;
3086fcc67678SFam Zheng     }
308767b51fb9SVladimir Sementsov-Ogievskiy     if (write_flags & BDRV_REQ_ZERO_WRITE) {
308867b51fb9SVladimir Sementsov-Ogievskiy         return bdrv_co_pwrite_zeroes(dst, dst_offset, bytes, write_flags);
3089fcc67678SFam Zheng     }
3090fcc67678SFam Zheng 
3091f4dad307SVladimir Sementsov-Ogievskiy     if (!src || !src->bs || !bdrv_is_inserted(src->bs)) {
3092d4d3e5a0SFam Zheng         return -ENOMEDIUM;
3093d4d3e5a0SFam Zheng     }
30948b117001SVladimir Sementsov-Ogievskiy     ret = bdrv_check_request32(src_offset, bytes);
3095d4d3e5a0SFam Zheng     if (ret) {
3096d4d3e5a0SFam Zheng         return ret;
3097d4d3e5a0SFam Zheng     }
3098d4d3e5a0SFam Zheng 
3099fcc67678SFam Zheng     if (!src->bs->drv->bdrv_co_copy_range_from
3100fcc67678SFam Zheng         || !dst->bs->drv->bdrv_co_copy_range_to
3101fcc67678SFam Zheng         || src->bs->encrypted || dst->bs->encrypted) {
3102fcc67678SFam Zheng         return -ENOTSUP;
3103fcc67678SFam Zheng     }
3104999658a0SVladimir Sementsov-Ogievskiy 
3105999658a0SVladimir Sementsov-Ogievskiy     if (recurse_src) {
3106d4d3e5a0SFam Zheng         bdrv_inc_in_flight(src->bs);
3107999658a0SVladimir Sementsov-Ogievskiy         tracked_request_begin(&req, src->bs, src_offset, bytes,
3108999658a0SVladimir Sementsov-Ogievskiy                               BDRV_TRACKED_READ);
310937aec7d7SFam Zheng 
311009d2f948SVladimir Sementsov-Ogievskiy         /* BDRV_REQ_SERIALISING is only for write operation */
311109d2f948SVladimir Sementsov-Ogievskiy         assert(!(read_flags & BDRV_REQ_SERIALISING));
3112304d9d7fSMax Reitz         bdrv_wait_serialising_requests(&req);
3113999658a0SVladimir Sementsov-Ogievskiy 
311437aec7d7SFam Zheng         ret = src->bs->drv->bdrv_co_copy_range_from(src->bs,
3115fcc67678SFam Zheng                                                     src, src_offset,
3116fcc67678SFam Zheng                                                     dst, dst_offset,
311767b51fb9SVladimir Sementsov-Ogievskiy                                                     bytes,
311867b51fb9SVladimir Sementsov-Ogievskiy                                                     read_flags, write_flags);
3119999658a0SVladimir Sementsov-Ogievskiy 
3120999658a0SVladimir Sementsov-Ogievskiy         tracked_request_end(&req);
3121999658a0SVladimir Sementsov-Ogievskiy         bdrv_dec_in_flight(src->bs);
3122fcc67678SFam Zheng     } else {
3123999658a0SVladimir Sementsov-Ogievskiy         bdrv_inc_in_flight(dst->bs);
3124999658a0SVladimir Sementsov-Ogievskiy         tracked_request_begin(&req, dst->bs, dst_offset, bytes,
3125999658a0SVladimir Sementsov-Ogievskiy                               BDRV_TRACKED_WRITE);
31260eb1e891SFam Zheng         ret = bdrv_co_write_req_prepare(dst, dst_offset, bytes, &req,
31270eb1e891SFam Zheng                                         write_flags);
31280eb1e891SFam Zheng         if (!ret) {
312937aec7d7SFam Zheng             ret = dst->bs->drv->bdrv_co_copy_range_to(dst->bs,
3130fcc67678SFam Zheng                                                       src, src_offset,
3131fcc67678SFam Zheng                                                       dst, dst_offset,
313267b51fb9SVladimir Sementsov-Ogievskiy                                                       bytes,
313367b51fb9SVladimir Sementsov-Ogievskiy                                                       read_flags, write_flags);
31340eb1e891SFam Zheng         }
31350eb1e891SFam Zheng         bdrv_co_write_req_finish(dst, dst_offset, bytes, &req, ret);
3136999658a0SVladimir Sementsov-Ogievskiy         tracked_request_end(&req);
3137d4d3e5a0SFam Zheng         bdrv_dec_in_flight(dst->bs);
3138999658a0SVladimir Sementsov-Ogievskiy     }
3139999658a0SVladimir Sementsov-Ogievskiy 
314037aec7d7SFam Zheng     return ret;
3141fcc67678SFam Zheng }
3142fcc67678SFam Zheng 
3143fcc67678SFam Zheng /* Copy range from @src to @dst.
3144fcc67678SFam Zheng  *
3145fcc67678SFam Zheng  * See the comment of bdrv_co_copy_range for the parameter and return value
3146fcc67678SFam Zheng  * semantics. */
3147fcc67678SFam Zheng int coroutine_fn bdrv_co_copy_range_from(BdrvChild *src, uint64_t src_offset,
3148fcc67678SFam Zheng                                          BdrvChild *dst, uint64_t dst_offset,
314967b51fb9SVladimir Sementsov-Ogievskiy                                          uint64_t bytes,
315067b51fb9SVladimir Sementsov-Ogievskiy                                          BdrvRequestFlags read_flags,
315167b51fb9SVladimir Sementsov-Ogievskiy                                          BdrvRequestFlags write_flags)
3152fcc67678SFam Zheng {
3153ecc983a5SFam Zheng     trace_bdrv_co_copy_range_from(src, src_offset, dst, dst_offset, bytes,
3154ecc983a5SFam Zheng                                   read_flags, write_flags);
3155fcc67678SFam Zheng     return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset,
315667b51fb9SVladimir Sementsov-Ogievskiy                                        bytes, read_flags, write_flags, true);
3157fcc67678SFam Zheng }
3158fcc67678SFam Zheng 
3159fcc67678SFam Zheng /* Copy range from @src to @dst.
3160fcc67678SFam Zheng  *
3161fcc67678SFam Zheng  * See the comment of bdrv_co_copy_range for the parameter and return value
3162fcc67678SFam Zheng  * semantics. */
3163fcc67678SFam Zheng int coroutine_fn bdrv_co_copy_range_to(BdrvChild *src, uint64_t src_offset,
3164fcc67678SFam Zheng                                        BdrvChild *dst, uint64_t dst_offset,
316567b51fb9SVladimir Sementsov-Ogievskiy                                        uint64_t bytes,
316667b51fb9SVladimir Sementsov-Ogievskiy                                        BdrvRequestFlags read_flags,
316767b51fb9SVladimir Sementsov-Ogievskiy                                        BdrvRequestFlags write_flags)
3168fcc67678SFam Zheng {
3169ecc983a5SFam Zheng     trace_bdrv_co_copy_range_to(src, src_offset, dst, dst_offset, bytes,
3170ecc983a5SFam Zheng                                 read_flags, write_flags);
3171fcc67678SFam Zheng     return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset,
317267b51fb9SVladimir Sementsov-Ogievskiy                                        bytes, read_flags, write_flags, false);
3173fcc67678SFam Zheng }
3174fcc67678SFam Zheng 
3175fcc67678SFam Zheng int coroutine_fn bdrv_co_copy_range(BdrvChild *src, uint64_t src_offset,
3176fcc67678SFam Zheng                                     BdrvChild *dst, uint64_t dst_offset,
317767b51fb9SVladimir Sementsov-Ogievskiy                                     uint64_t bytes, BdrvRequestFlags read_flags,
317867b51fb9SVladimir Sementsov-Ogievskiy                                     BdrvRequestFlags write_flags)
3179fcc67678SFam Zheng {
318037aec7d7SFam Zheng     return bdrv_co_copy_range_from(src, src_offset,
3181fcc67678SFam Zheng                                    dst, dst_offset,
318267b51fb9SVladimir Sementsov-Ogievskiy                                    bytes, read_flags, write_flags);
3183fcc67678SFam Zheng }
31843d9f2d2aSKevin Wolf 
31853d9f2d2aSKevin Wolf static void bdrv_parent_cb_resize(BlockDriverState *bs)
31863d9f2d2aSKevin Wolf {
31873d9f2d2aSKevin Wolf     BdrvChild *c;
31883d9f2d2aSKevin Wolf     QLIST_FOREACH(c, &bs->parents, next_parent) {
3189bd86fb99SMax Reitz         if (c->klass->resize) {
3190bd86fb99SMax Reitz             c->klass->resize(c);
31913d9f2d2aSKevin Wolf         }
31923d9f2d2aSKevin Wolf     }
31933d9f2d2aSKevin Wolf }
31943d9f2d2aSKevin Wolf 
31953d9f2d2aSKevin Wolf /**
31963d9f2d2aSKevin Wolf  * Truncate file to 'offset' bytes (needed only for file protocols)
3197c80d8b06SMax Reitz  *
3198c80d8b06SMax Reitz  * If 'exact' is true, the file must be resized to exactly the given
3199c80d8b06SMax Reitz  * 'offset'.  Otherwise, it is sufficient for the node to be at least
3200c80d8b06SMax Reitz  * 'offset' bytes in length.
32013d9f2d2aSKevin Wolf  */
3202c80d8b06SMax Reitz int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset, bool exact,
32037b8e4857SKevin Wolf                                   PreallocMode prealloc, BdrvRequestFlags flags,
32047b8e4857SKevin Wolf                                   Error **errp)
32053d9f2d2aSKevin Wolf {
32063d9f2d2aSKevin Wolf     BlockDriverState *bs = child->bs;
320723b93525SMax Reitz     BdrvChild *filtered, *backing;
32083d9f2d2aSKevin Wolf     BlockDriver *drv = bs->drv;
32091bc5f09fSKevin Wolf     BdrvTrackedRequest req;
32101bc5f09fSKevin Wolf     int64_t old_size, new_bytes;
32113d9f2d2aSKevin Wolf     int ret;
32123d9f2d2aSKevin Wolf 
32133d9f2d2aSKevin Wolf 
32143d9f2d2aSKevin Wolf     /* if bs->drv == NULL, bs is closed, so there's nothing to do here */
32153d9f2d2aSKevin Wolf     if (!drv) {
32163d9f2d2aSKevin Wolf         error_setg(errp, "No medium inserted");
32173d9f2d2aSKevin Wolf         return -ENOMEDIUM;
32183d9f2d2aSKevin Wolf     }
32193d9f2d2aSKevin Wolf     if (offset < 0) {
32203d9f2d2aSKevin Wolf         error_setg(errp, "Image size cannot be negative");
32213d9f2d2aSKevin Wolf         return -EINVAL;
32223d9f2d2aSKevin Wolf     }
32233d9f2d2aSKevin Wolf 
32248b117001SVladimir Sementsov-Ogievskiy     ret = bdrv_check_request(offset, 0);
32258b117001SVladimir Sementsov-Ogievskiy     if (ret < 0) {
32268b117001SVladimir Sementsov-Ogievskiy         error_setg(errp, "Required too big image size, it must be not greater "
32278b117001SVladimir Sementsov-Ogievskiy                    "than %" PRId64, BDRV_MAX_LENGTH);
32288b117001SVladimir Sementsov-Ogievskiy         return ret;
32298b117001SVladimir Sementsov-Ogievskiy     }
32308b117001SVladimir Sementsov-Ogievskiy 
32311bc5f09fSKevin Wolf     old_size = bdrv_getlength(bs);
32321bc5f09fSKevin Wolf     if (old_size < 0) {
32331bc5f09fSKevin Wolf         error_setg_errno(errp, -old_size, "Failed to get old image size");
32341bc5f09fSKevin Wolf         return old_size;
32351bc5f09fSKevin Wolf     }
32361bc5f09fSKevin Wolf 
32371bc5f09fSKevin Wolf     if (offset > old_size) {
32381bc5f09fSKevin Wolf         new_bytes = offset - old_size;
32391bc5f09fSKevin Wolf     } else {
32401bc5f09fSKevin Wolf         new_bytes = 0;
32411bc5f09fSKevin Wolf     }
32421bc5f09fSKevin Wolf 
32433d9f2d2aSKevin Wolf     bdrv_inc_in_flight(bs);
32445416a11eSFam Zheng     tracked_request_begin(&req, bs, offset - new_bytes, new_bytes,
32455416a11eSFam Zheng                           BDRV_TRACKED_TRUNCATE);
32461bc5f09fSKevin Wolf 
32471bc5f09fSKevin Wolf     /* If we are growing the image and potentially using preallocation for the
32481bc5f09fSKevin Wolf      * new area, we need to make sure that no write requests are made to it
32491bc5f09fSKevin Wolf      * concurrently or they might be overwritten by preallocation. */
32501bc5f09fSKevin Wolf     if (new_bytes) {
3251304d9d7fSMax Reitz         bdrv_mark_request_serialising(&req, 1);
3252cd47d792SFam Zheng     }
3253cd47d792SFam Zheng     if (bs->read_only) {
3254cd47d792SFam Zheng         error_setg(errp, "Image is read-only");
3255cd47d792SFam Zheng         ret = -EACCES;
3256cd47d792SFam Zheng         goto out;
3257cd47d792SFam Zheng     }
3258cd47d792SFam Zheng     ret = bdrv_co_write_req_prepare(child, offset - new_bytes, new_bytes, &req,
3259cd47d792SFam Zheng                                     0);
3260cd47d792SFam Zheng     if (ret < 0) {
3261cd47d792SFam Zheng         error_setg_errno(errp, -ret,
3262cd47d792SFam Zheng                          "Failed to prepare request for truncation");
3263cd47d792SFam Zheng         goto out;
32641bc5f09fSKevin Wolf     }
32653d9f2d2aSKevin Wolf 
326693393e69SMax Reitz     filtered = bdrv_filter_child(bs);
326723b93525SMax Reitz     backing = bdrv_cow_child(bs);
326893393e69SMax Reitz 
3269955c7d66SKevin Wolf     /*
3270955c7d66SKevin Wolf      * If the image has a backing file that is large enough that it would
3271955c7d66SKevin Wolf      * provide data for the new area, we cannot leave it unallocated because
3272955c7d66SKevin Wolf      * then the backing file content would become visible. Instead, zero-fill
3273955c7d66SKevin Wolf      * the new area.
3274955c7d66SKevin Wolf      *
3275955c7d66SKevin Wolf      * Note that if the image has a backing file, but was opened without the
3276955c7d66SKevin Wolf      * backing file, taking care of keeping things consistent with that backing
3277955c7d66SKevin Wolf      * file is the user's responsibility.
3278955c7d66SKevin Wolf      */
327923b93525SMax Reitz     if (new_bytes && backing) {
3280955c7d66SKevin Wolf         int64_t backing_len;
3281955c7d66SKevin Wolf 
328223b93525SMax Reitz         backing_len = bdrv_getlength(backing->bs);
3283955c7d66SKevin Wolf         if (backing_len < 0) {
3284955c7d66SKevin Wolf             ret = backing_len;
3285955c7d66SKevin Wolf             error_setg_errno(errp, -ret, "Could not get backing file size");
3286955c7d66SKevin Wolf             goto out;
3287955c7d66SKevin Wolf         }
3288955c7d66SKevin Wolf 
3289955c7d66SKevin Wolf         if (backing_len > old_size) {
3290955c7d66SKevin Wolf             flags |= BDRV_REQ_ZERO_WRITE;
3291955c7d66SKevin Wolf         }
3292955c7d66SKevin Wolf     }
3293955c7d66SKevin Wolf 
32946b7e8f8bSMax Reitz     if (drv->bdrv_co_truncate) {
329592b92799SKevin Wolf         if (flags & ~bs->supported_truncate_flags) {
329692b92799SKevin Wolf             error_setg(errp, "Block driver does not support requested flags");
329792b92799SKevin Wolf             ret = -ENOTSUP;
329892b92799SKevin Wolf             goto out;
329992b92799SKevin Wolf         }
330092b92799SKevin Wolf         ret = drv->bdrv_co_truncate(bs, offset, exact, prealloc, flags, errp);
330193393e69SMax Reitz     } else if (filtered) {
330293393e69SMax Reitz         ret = bdrv_co_truncate(filtered, offset, exact, prealloc, flags, errp);
33036b7e8f8bSMax Reitz     } else {
33043d9f2d2aSKevin Wolf         error_setg(errp, "Image format driver does not support resize");
33053d9f2d2aSKevin Wolf         ret = -ENOTSUP;
33063d9f2d2aSKevin Wolf         goto out;
33073d9f2d2aSKevin Wolf     }
33083d9f2d2aSKevin Wolf     if (ret < 0) {
33093d9f2d2aSKevin Wolf         goto out;
33103d9f2d2aSKevin Wolf     }
33116b7e8f8bSMax Reitz 
33123d9f2d2aSKevin Wolf     ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
33133d9f2d2aSKevin Wolf     if (ret < 0) {
33143d9f2d2aSKevin Wolf         error_setg_errno(errp, -ret, "Could not refresh total sector count");
33153d9f2d2aSKevin Wolf     } else {
33163d9f2d2aSKevin Wolf         offset = bs->total_sectors * BDRV_SECTOR_SIZE;
33173d9f2d2aSKevin Wolf     }
3318cd47d792SFam Zheng     /* It's possible that truncation succeeded but refresh_total_sectors
3319cd47d792SFam Zheng      * failed, but the latter doesn't affect how we should finish the request.
3320cd47d792SFam Zheng      * Pass 0 as the last parameter so that dirty bitmaps etc. are handled. */
3321cd47d792SFam Zheng     bdrv_co_write_req_finish(child, offset - new_bytes, new_bytes, &req, 0);
33223d9f2d2aSKevin Wolf 
33233d9f2d2aSKevin Wolf out:
33241bc5f09fSKevin Wolf     tracked_request_end(&req);
33253d9f2d2aSKevin Wolf     bdrv_dec_in_flight(bs);
33261bc5f09fSKevin Wolf 
33273d9f2d2aSKevin Wolf     return ret;
33283d9f2d2aSKevin Wolf }
3329