xref: /qemu/block/io.c (revision 4f384011c5a37f80dc6cadefffac61ffb1c3aa1e)
161007b31SStefan Hajnoczi /*
261007b31SStefan Hajnoczi  * Block layer I/O functions
361007b31SStefan Hajnoczi  *
461007b31SStefan Hajnoczi  * Copyright (c) 2003 Fabrice Bellard
561007b31SStefan Hajnoczi  *
661007b31SStefan Hajnoczi  * Permission is hereby granted, free of charge, to any person obtaining a copy
761007b31SStefan Hajnoczi  * of this software and associated documentation files (the "Software"), to deal
861007b31SStefan Hajnoczi  * in the Software without restriction, including without limitation the rights
961007b31SStefan Hajnoczi  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
1061007b31SStefan Hajnoczi  * copies of the Software, and to permit persons to whom the Software is
1161007b31SStefan Hajnoczi  * furnished to do so, subject to the following conditions:
1261007b31SStefan Hajnoczi  *
1361007b31SStefan Hajnoczi  * The above copyright notice and this permission notice shall be included in
1461007b31SStefan Hajnoczi  * all copies or substantial portions of the Software.
1561007b31SStefan Hajnoczi  *
1661007b31SStefan Hajnoczi  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
1761007b31SStefan Hajnoczi  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
1861007b31SStefan Hajnoczi  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
1961007b31SStefan Hajnoczi  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
2061007b31SStefan Hajnoczi  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
2161007b31SStefan Hajnoczi  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
2261007b31SStefan Hajnoczi  * THE SOFTWARE.
2361007b31SStefan Hajnoczi  */
2461007b31SStefan Hajnoczi 
2580c71a24SPeter Maydell #include "qemu/osdep.h"
2661007b31SStefan Hajnoczi #include "trace.h"
277f0e9da6SMax Reitz #include "sysemu/block-backend.h"
287719f3c9SStefan Hajnoczi #include "block/aio-wait.h"
2961007b31SStefan Hajnoczi #include "block/blockjob.h"
30f321dcb5SPaolo Bonzini #include "block/blockjob_int.h"
3161007b31SStefan Hajnoczi #include "block/block_int.h"
3221c2283eSVladimir Sementsov-Ogievskiy #include "block/coroutines.h"
3394783301SVladimir Sementsov-Ogievskiy #include "block/write-threshold.h"
34f348b6d1SVeronia Bahaa #include "qemu/cutils.h"
355df022cfSPeter Maydell #include "qemu/memalign.h"
36da34e65cSMarkus Armbruster #include "qapi/error.h"
37d49b6836SMarkus Armbruster #include "qemu/error-report.h"
38db725815SMarkus Armbruster #include "qemu/main-loop.h"
39c8aa7895SPavel Dovgalyuk #include "sysemu/replay.h"
4061007b31SStefan Hajnoczi 
41cb2e2878SEric Blake /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */
42cb2e2878SEric Blake #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS)
43cb2e2878SEric Blake 
447f8f03efSFam Zheng static void bdrv_parent_cb_resize(BlockDriverState *bs);
45d05aa8bbSEric Blake static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
465ae07b14SVladimir Sementsov-Ogievskiy     int64_t offset, int64_t bytes, BdrvRequestFlags flags);
4761007b31SStefan Hajnoczi 
48f4c8a43bSMax Reitz static void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore,
496cd5c9d7SKevin Wolf                                       bool ignore_bds_parents)
5061007b31SStefan Hajnoczi {
5102d21300SKevin Wolf     BdrvChild *c, *next;
5227ccdd52SKevin Wolf 
5302d21300SKevin Wolf     QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
54bd86fb99SMax Reitz         if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) {
550152bf40SKevin Wolf             continue;
560152bf40SKevin Wolf         }
574be6a6d1SKevin Wolf         bdrv_parent_drained_begin_single(c, false);
58ce0f1412SPaolo Bonzini     }
59ce0f1412SPaolo Bonzini }
60ce0f1412SPaolo Bonzini 
61e037c09cSMax Reitz static void bdrv_parent_drained_end_single_no_poll(BdrvChild *c,
62e037c09cSMax Reitz                                                    int *drained_end_counter)
63804db8eaSMax Reitz {
64804db8eaSMax Reitz     assert(c->parent_quiesce_counter > 0);
65804db8eaSMax Reitz     c->parent_quiesce_counter--;
66bd86fb99SMax Reitz     if (c->klass->drained_end) {
67bd86fb99SMax Reitz         c->klass->drained_end(c, drained_end_counter);
68804db8eaSMax Reitz     }
69804db8eaSMax Reitz }
70804db8eaSMax Reitz 
71e037c09cSMax Reitz void bdrv_parent_drained_end_single(BdrvChild *c)
72e037c09cSMax Reitz {
73e037c09cSMax Reitz     int drained_end_counter = 0;
74384a48fbSEmanuele Giuseppe Esposito     IO_OR_GS_CODE();
75e037c09cSMax Reitz     bdrv_parent_drained_end_single_no_poll(c, &drained_end_counter);
76d73415a3SStefan Hajnoczi     BDRV_POLL_WHILE(c->bs, qatomic_read(&drained_end_counter) > 0);
77e037c09cSMax Reitz }
78e037c09cSMax Reitz 
79f4c8a43bSMax Reitz static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
80e037c09cSMax Reitz                                     bool ignore_bds_parents,
81e037c09cSMax Reitz                                     int *drained_end_counter)
82ce0f1412SPaolo Bonzini {
8361ad631cSMax Reitz     BdrvChild *c;
8427ccdd52SKevin Wolf 
8561ad631cSMax Reitz     QLIST_FOREACH(c, &bs->parents, next_parent) {
86bd86fb99SMax Reitz         if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) {
870152bf40SKevin Wolf             continue;
880152bf40SKevin Wolf         }
89e037c09cSMax Reitz         bdrv_parent_drained_end_single_no_poll(c, drained_end_counter);
90c2066af0SKevin Wolf     }
9161007b31SStefan Hajnoczi }
9261007b31SStefan Hajnoczi 
934be6a6d1SKevin Wolf static bool bdrv_parent_drained_poll_single(BdrvChild *c)
944be6a6d1SKevin Wolf {
95bd86fb99SMax Reitz     if (c->klass->drained_poll) {
96bd86fb99SMax Reitz         return c->klass->drained_poll(c);
974be6a6d1SKevin Wolf     }
984be6a6d1SKevin Wolf     return false;
994be6a6d1SKevin Wolf }
1004be6a6d1SKevin Wolf 
1016cd5c9d7SKevin Wolf static bool bdrv_parent_drained_poll(BlockDriverState *bs, BdrvChild *ignore,
1026cd5c9d7SKevin Wolf                                      bool ignore_bds_parents)
10389bd0305SKevin Wolf {
10489bd0305SKevin Wolf     BdrvChild *c, *next;
10589bd0305SKevin Wolf     bool busy = false;
10689bd0305SKevin Wolf 
10789bd0305SKevin Wolf     QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
108bd86fb99SMax Reitz         if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) {
10989bd0305SKevin Wolf             continue;
11089bd0305SKevin Wolf         }
1114be6a6d1SKevin Wolf         busy |= bdrv_parent_drained_poll_single(c);
11289bd0305SKevin Wolf     }
11389bd0305SKevin Wolf 
11489bd0305SKevin Wolf     return busy;
11589bd0305SKevin Wolf }
11689bd0305SKevin Wolf 
1174be6a6d1SKevin Wolf void bdrv_parent_drained_begin_single(BdrvChild *c, bool poll)
1184be6a6d1SKevin Wolf {
119384a48fbSEmanuele Giuseppe Esposito     IO_OR_GS_CODE();
120804db8eaSMax Reitz     c->parent_quiesce_counter++;
121bd86fb99SMax Reitz     if (c->klass->drained_begin) {
122bd86fb99SMax Reitz         c->klass->drained_begin(c);
1234be6a6d1SKevin Wolf     }
1244be6a6d1SKevin Wolf     if (poll) {
1254be6a6d1SKevin Wolf         BDRV_POLL_WHILE(c->bs, bdrv_parent_drained_poll_single(c));
1264be6a6d1SKevin Wolf     }
1274be6a6d1SKevin Wolf }
1284be6a6d1SKevin Wolf 
129d9e0dfa2SEric Blake static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src)
130d9e0dfa2SEric Blake {
1319f460c64SAkihiko Odaki     dst->pdiscard_alignment = MAX(dst->pdiscard_alignment,
1329f460c64SAkihiko Odaki                                   src->pdiscard_alignment);
133d9e0dfa2SEric Blake     dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer);
134d9e0dfa2SEric Blake     dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer);
13524b36e98SPaolo Bonzini     dst->max_hw_transfer = MIN_NON_ZERO(dst->max_hw_transfer,
13624b36e98SPaolo Bonzini                                         src->max_hw_transfer);
137d9e0dfa2SEric Blake     dst->opt_mem_alignment = MAX(dst->opt_mem_alignment,
138d9e0dfa2SEric Blake                                  src->opt_mem_alignment);
139d9e0dfa2SEric Blake     dst->min_mem_alignment = MAX(dst->min_mem_alignment,
140d9e0dfa2SEric Blake                                  src->min_mem_alignment);
141d9e0dfa2SEric Blake     dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov);
142cc071629SPaolo Bonzini     dst->max_hw_iov = MIN_NON_ZERO(dst->max_hw_iov, src->max_hw_iov);
143d9e0dfa2SEric Blake }
144d9e0dfa2SEric Blake 
1451e4c797cSVladimir Sementsov-Ogievskiy typedef struct BdrvRefreshLimitsState {
1461e4c797cSVladimir Sementsov-Ogievskiy     BlockDriverState *bs;
1471e4c797cSVladimir Sementsov-Ogievskiy     BlockLimits old_bl;
1481e4c797cSVladimir Sementsov-Ogievskiy } BdrvRefreshLimitsState;
1491e4c797cSVladimir Sementsov-Ogievskiy 
1501e4c797cSVladimir Sementsov-Ogievskiy static void bdrv_refresh_limits_abort(void *opaque)
1511e4c797cSVladimir Sementsov-Ogievskiy {
1521e4c797cSVladimir Sementsov-Ogievskiy     BdrvRefreshLimitsState *s = opaque;
1531e4c797cSVladimir Sementsov-Ogievskiy 
1541e4c797cSVladimir Sementsov-Ogievskiy     s->bs->bl = s->old_bl;
1551e4c797cSVladimir Sementsov-Ogievskiy }
1561e4c797cSVladimir Sementsov-Ogievskiy 
1571e4c797cSVladimir Sementsov-Ogievskiy static TransactionActionDrv bdrv_refresh_limits_drv = {
1581e4c797cSVladimir Sementsov-Ogievskiy     .abort = bdrv_refresh_limits_abort,
1591e4c797cSVladimir Sementsov-Ogievskiy     .clean = g_free,
1601e4c797cSVladimir Sementsov-Ogievskiy };
1611e4c797cSVladimir Sementsov-Ogievskiy 
1621e4c797cSVladimir Sementsov-Ogievskiy /* @tran is allowed to be NULL, in this case no rollback is possible. */
1631e4c797cSVladimir Sementsov-Ogievskiy void bdrv_refresh_limits(BlockDriverState *bs, Transaction *tran, Error **errp)
16461007b31SStefan Hajnoczi {
16533985614SVladimir Sementsov-Ogievskiy     ERRP_GUARD();
16661007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
16766b129acSMax Reitz     BdrvChild *c;
16866b129acSMax Reitz     bool have_limits;
16961007b31SStefan Hajnoczi 
170f791bf7fSEmanuele Giuseppe Esposito     GLOBAL_STATE_CODE();
171f791bf7fSEmanuele Giuseppe Esposito 
1721e4c797cSVladimir Sementsov-Ogievskiy     if (tran) {
1731e4c797cSVladimir Sementsov-Ogievskiy         BdrvRefreshLimitsState *s = g_new(BdrvRefreshLimitsState, 1);
1741e4c797cSVladimir Sementsov-Ogievskiy         *s = (BdrvRefreshLimitsState) {
1751e4c797cSVladimir Sementsov-Ogievskiy             .bs = bs,
1761e4c797cSVladimir Sementsov-Ogievskiy             .old_bl = bs->bl,
1771e4c797cSVladimir Sementsov-Ogievskiy         };
1781e4c797cSVladimir Sementsov-Ogievskiy         tran_add(tran, &bdrv_refresh_limits_drv, s);
1791e4c797cSVladimir Sementsov-Ogievskiy     }
1801e4c797cSVladimir Sementsov-Ogievskiy 
18161007b31SStefan Hajnoczi     memset(&bs->bl, 0, sizeof(bs->bl));
18261007b31SStefan Hajnoczi 
18361007b31SStefan Hajnoczi     if (!drv) {
18461007b31SStefan Hajnoczi         return;
18561007b31SStefan Hajnoczi     }
18661007b31SStefan Hajnoczi 
18779ba8c98SEric Blake     /* Default alignment based on whether driver has byte interface */
188e31f6864SEric Blake     bs->bl.request_alignment = (drv->bdrv_co_preadv ||
189ac850bf0SVladimir Sementsov-Ogievskiy                                 drv->bdrv_aio_preadv ||
190ac850bf0SVladimir Sementsov-Ogievskiy                                 drv->bdrv_co_preadv_part) ? 1 : 512;
19179ba8c98SEric Blake 
19261007b31SStefan Hajnoczi     /* Take some limits from the children as a default */
19366b129acSMax Reitz     have_limits = false;
19466b129acSMax Reitz     QLIST_FOREACH(c, &bs->children, next) {
19566b129acSMax Reitz         if (c->role & (BDRV_CHILD_DATA | BDRV_CHILD_FILTERED | BDRV_CHILD_COW))
19666b129acSMax Reitz         {
19766b129acSMax Reitz             bdrv_merge_limits(&bs->bl, &c->bs->bl);
19866b129acSMax Reitz             have_limits = true;
19966b129acSMax Reitz         }
20066b129acSMax Reitz     }
20166b129acSMax Reitz 
20266b129acSMax Reitz     if (!have_limits) {
2034196d2f0SDenis V. Lunev         bs->bl.min_mem_alignment = 512;
2048e3b0cbbSMarc-André Lureau         bs->bl.opt_mem_alignment = qemu_real_host_page_size();
205bd44feb7SStefan Hajnoczi 
206bd44feb7SStefan Hajnoczi         /* Safe default since most protocols use readv()/writev()/etc */
207bd44feb7SStefan Hajnoczi         bs->bl.max_iov = IOV_MAX;
20861007b31SStefan Hajnoczi     }
20961007b31SStefan Hajnoczi 
21061007b31SStefan Hajnoczi     /* Then let the driver override it */
21161007b31SStefan Hajnoczi     if (drv->bdrv_refresh_limits) {
21261007b31SStefan Hajnoczi         drv->bdrv_refresh_limits(bs, errp);
2138b117001SVladimir Sementsov-Ogievskiy         if (*errp) {
2148b117001SVladimir Sementsov-Ogievskiy             return;
2158b117001SVladimir Sementsov-Ogievskiy         }
2168b117001SVladimir Sementsov-Ogievskiy     }
2178b117001SVladimir Sementsov-Ogievskiy 
2188b117001SVladimir Sementsov-Ogievskiy     if (bs->bl.request_alignment > BDRV_MAX_ALIGNMENT) {
2198b117001SVladimir Sementsov-Ogievskiy         error_setg(errp, "Driver requires too large request alignment");
22061007b31SStefan Hajnoczi     }
22161007b31SStefan Hajnoczi }
22261007b31SStefan Hajnoczi 
22361007b31SStefan Hajnoczi /**
22461007b31SStefan Hajnoczi  * The copy-on-read flag is actually a reference count so multiple users may
22561007b31SStefan Hajnoczi  * use the feature without worrying about clobbering its previous state.
22661007b31SStefan Hajnoczi  * Copy-on-read stays enabled until all users have called to disable it.
22761007b31SStefan Hajnoczi  */
22861007b31SStefan Hajnoczi void bdrv_enable_copy_on_read(BlockDriverState *bs)
22961007b31SStefan Hajnoczi {
230384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
231d73415a3SStefan Hajnoczi     qatomic_inc(&bs->copy_on_read);
23261007b31SStefan Hajnoczi }
23361007b31SStefan Hajnoczi 
23461007b31SStefan Hajnoczi void bdrv_disable_copy_on_read(BlockDriverState *bs)
23561007b31SStefan Hajnoczi {
236d73415a3SStefan Hajnoczi     int old = qatomic_fetch_dec(&bs->copy_on_read);
237384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
238d3faa13eSPaolo Bonzini     assert(old >= 1);
23961007b31SStefan Hajnoczi }
24061007b31SStefan Hajnoczi 
24161124f03SPaolo Bonzini typedef struct {
24261124f03SPaolo Bonzini     Coroutine *co;
24361124f03SPaolo Bonzini     BlockDriverState *bs;
24461124f03SPaolo Bonzini     bool done;
245481cad48SManos Pitsidianakis     bool begin;
246b0165585SKevin Wolf     bool recursive;
247fe4f0614SKevin Wolf     bool poll;
2480152bf40SKevin Wolf     BdrvChild *parent;
2496cd5c9d7SKevin Wolf     bool ignore_bds_parents;
2508e1da77eSMax Reitz     int *drained_end_counter;
25161124f03SPaolo Bonzini } BdrvCoDrainData;
25261124f03SPaolo Bonzini 
25361124f03SPaolo Bonzini static void coroutine_fn bdrv_drain_invoke_entry(void *opaque)
25461124f03SPaolo Bonzini {
25561124f03SPaolo Bonzini     BdrvCoDrainData *data = opaque;
25661124f03SPaolo Bonzini     BlockDriverState *bs = data->bs;
25761124f03SPaolo Bonzini 
258481cad48SManos Pitsidianakis     if (data->begin) {
259f8ea8dacSManos Pitsidianakis         bs->drv->bdrv_co_drain_begin(bs);
260481cad48SManos Pitsidianakis     } else {
261481cad48SManos Pitsidianakis         bs->drv->bdrv_co_drain_end(bs);
262481cad48SManos Pitsidianakis     }
26361124f03SPaolo Bonzini 
26465181d63SMax Reitz     /* Set data->done and decrement drained_end_counter before bdrv_wakeup() */
265d73415a3SStefan Hajnoczi     qatomic_mb_set(&data->done, true);
266e037c09cSMax Reitz     if (!data->begin) {
267d73415a3SStefan Hajnoczi         qatomic_dec(data->drained_end_counter);
2688e1da77eSMax Reitz     }
26965181d63SMax Reitz     bdrv_dec_in_flight(bs);
2708e1da77eSMax Reitz 
2710109e7e6SKevin Wolf     g_free(data);
2720109e7e6SKevin Wolf }
27361124f03SPaolo Bonzini 
274db0289b9SKevin Wolf /* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */
2758e1da77eSMax Reitz static void bdrv_drain_invoke(BlockDriverState *bs, bool begin,
2768e1da77eSMax Reitz                               int *drained_end_counter)
27761124f03SPaolo Bonzini {
2780109e7e6SKevin Wolf     BdrvCoDrainData *data;
27961124f03SPaolo Bonzini 
280f8ea8dacSManos Pitsidianakis     if (!bs->drv || (begin && !bs->drv->bdrv_co_drain_begin) ||
281481cad48SManos Pitsidianakis             (!begin && !bs->drv->bdrv_co_drain_end)) {
28261124f03SPaolo Bonzini         return;
28361124f03SPaolo Bonzini     }
28461124f03SPaolo Bonzini 
2850109e7e6SKevin Wolf     data = g_new(BdrvCoDrainData, 1);
2860109e7e6SKevin Wolf     *data = (BdrvCoDrainData) {
2870109e7e6SKevin Wolf         .bs = bs,
2880109e7e6SKevin Wolf         .done = false,
2898e1da77eSMax Reitz         .begin = begin,
2908e1da77eSMax Reitz         .drained_end_counter = drained_end_counter,
2910109e7e6SKevin Wolf     };
2920109e7e6SKevin Wolf 
293e037c09cSMax Reitz     if (!begin) {
294d73415a3SStefan Hajnoczi         qatomic_inc(drained_end_counter);
2958e1da77eSMax Reitz     }
2968e1da77eSMax Reitz 
2970109e7e6SKevin Wolf     /* Make sure the driver callback completes during the polling phase for
2980109e7e6SKevin Wolf      * drain_begin. */
2990109e7e6SKevin Wolf     bdrv_inc_in_flight(bs);
3000109e7e6SKevin Wolf     data->co = qemu_coroutine_create(bdrv_drain_invoke_entry, data);
3010109e7e6SKevin Wolf     aio_co_schedule(bdrv_get_aio_context(bs), data->co);
30261124f03SPaolo Bonzini }
30361124f03SPaolo Bonzini 
3041cc8e54aSKevin Wolf /* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */
305fe4f0614SKevin Wolf bool bdrv_drain_poll(BlockDriverState *bs, bool recursive,
3066cd5c9d7SKevin Wolf                      BdrvChild *ignore_parent, bool ignore_bds_parents)
30789bd0305SKevin Wolf {
308fe4f0614SKevin Wolf     BdrvChild *child, *next;
309384a48fbSEmanuele Giuseppe Esposito     IO_OR_GS_CODE();
310fe4f0614SKevin Wolf 
3116cd5c9d7SKevin Wolf     if (bdrv_parent_drained_poll(bs, ignore_parent, ignore_bds_parents)) {
31289bd0305SKevin Wolf         return true;
31389bd0305SKevin Wolf     }
31489bd0305SKevin Wolf 
315d73415a3SStefan Hajnoczi     if (qatomic_read(&bs->in_flight)) {
316fe4f0614SKevin Wolf         return true;
31789bd0305SKevin Wolf     }
31889bd0305SKevin Wolf 
319fe4f0614SKevin Wolf     if (recursive) {
3206cd5c9d7SKevin Wolf         assert(!ignore_bds_parents);
321fe4f0614SKevin Wolf         QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
3226cd5c9d7SKevin Wolf             if (bdrv_drain_poll(child->bs, recursive, child, false)) {
323fe4f0614SKevin Wolf                 return true;
324fe4f0614SKevin Wolf             }
325fe4f0614SKevin Wolf         }
326fe4f0614SKevin Wolf     }
327fe4f0614SKevin Wolf 
328fe4f0614SKevin Wolf     return false;
329fe4f0614SKevin Wolf }
330fe4f0614SKevin Wolf 
331fe4f0614SKevin Wolf static bool bdrv_drain_poll_top_level(BlockDriverState *bs, bool recursive,
33289bd0305SKevin Wolf                                       BdrvChild *ignore_parent)
3331cc8e54aSKevin Wolf {
3346cd5c9d7SKevin Wolf     return bdrv_drain_poll(bs, recursive, ignore_parent, false);
3351cc8e54aSKevin Wolf }
3361cc8e54aSKevin Wolf 
337b0165585SKevin Wolf static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
3386cd5c9d7SKevin Wolf                                   BdrvChild *parent, bool ignore_bds_parents,
3396cd5c9d7SKevin Wolf                                   bool poll);
340b0165585SKevin Wolf static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
3418e1da77eSMax Reitz                                 BdrvChild *parent, bool ignore_bds_parents,
3428e1da77eSMax Reitz                                 int *drained_end_counter);
3430152bf40SKevin Wolf 
344a77fd4bbSFam Zheng static void bdrv_co_drain_bh_cb(void *opaque)
345a77fd4bbSFam Zheng {
346a77fd4bbSFam Zheng     BdrvCoDrainData *data = opaque;
347a77fd4bbSFam Zheng     Coroutine *co = data->co;
34899723548SPaolo Bonzini     BlockDriverState *bs = data->bs;
349a77fd4bbSFam Zheng 
350c8ca33d0SKevin Wolf     if (bs) {
351aa1361d5SKevin Wolf         AioContext *ctx = bdrv_get_aio_context(bs);
352aa1361d5SKevin Wolf         aio_context_acquire(ctx);
35399723548SPaolo Bonzini         bdrv_dec_in_flight(bs);
354481cad48SManos Pitsidianakis         if (data->begin) {
355e037c09cSMax Reitz             assert(!data->drained_end_counter);
3566cd5c9d7SKevin Wolf             bdrv_do_drained_begin(bs, data->recursive, data->parent,
3576cd5c9d7SKevin Wolf                                   data->ignore_bds_parents, data->poll);
358481cad48SManos Pitsidianakis         } else {
359e037c09cSMax Reitz             assert(!data->poll);
3606cd5c9d7SKevin Wolf             bdrv_do_drained_end(bs, data->recursive, data->parent,
3618e1da77eSMax Reitz                                 data->ignore_bds_parents,
3628e1da77eSMax Reitz                                 data->drained_end_counter);
363481cad48SManos Pitsidianakis         }
364aa1361d5SKevin Wolf         aio_context_release(ctx);
365c8ca33d0SKevin Wolf     } else {
366c8ca33d0SKevin Wolf         assert(data->begin);
367c8ca33d0SKevin Wolf         bdrv_drain_all_begin();
368c8ca33d0SKevin Wolf     }
369481cad48SManos Pitsidianakis 
370a77fd4bbSFam Zheng     data->done = true;
3711919631eSPaolo Bonzini     aio_co_wake(co);
372a77fd4bbSFam Zheng }
373a77fd4bbSFam Zheng 
374481cad48SManos Pitsidianakis static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
375b0165585SKevin Wolf                                                 bool begin, bool recursive,
3766cd5c9d7SKevin Wolf                                                 BdrvChild *parent,
3776cd5c9d7SKevin Wolf                                                 bool ignore_bds_parents,
3788e1da77eSMax Reitz                                                 bool poll,
3798e1da77eSMax Reitz                                                 int *drained_end_counter)
380a77fd4bbSFam Zheng {
381a77fd4bbSFam Zheng     BdrvCoDrainData data;
382960d5fb3SKevin Wolf     Coroutine *self = qemu_coroutine_self();
383960d5fb3SKevin Wolf     AioContext *ctx = bdrv_get_aio_context(bs);
384960d5fb3SKevin Wolf     AioContext *co_ctx = qemu_coroutine_get_aio_context(self);
385a77fd4bbSFam Zheng 
386a77fd4bbSFam Zheng     /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
387c40a2545SStefan Hajnoczi      * other coroutines run if they were queued by aio_co_enter(). */
388a77fd4bbSFam Zheng 
389a77fd4bbSFam Zheng     assert(qemu_in_coroutine());
390a77fd4bbSFam Zheng     data = (BdrvCoDrainData) {
391960d5fb3SKevin Wolf         .co = self,
392a77fd4bbSFam Zheng         .bs = bs,
393a77fd4bbSFam Zheng         .done = false,
394481cad48SManos Pitsidianakis         .begin = begin,
395b0165585SKevin Wolf         .recursive = recursive,
3960152bf40SKevin Wolf         .parent = parent,
3976cd5c9d7SKevin Wolf         .ignore_bds_parents = ignore_bds_parents,
398fe4f0614SKevin Wolf         .poll = poll,
3998e1da77eSMax Reitz         .drained_end_counter = drained_end_counter,
400a77fd4bbSFam Zheng     };
4018e1da77eSMax Reitz 
402c8ca33d0SKevin Wolf     if (bs) {
40399723548SPaolo Bonzini         bdrv_inc_in_flight(bs);
404c8ca33d0SKevin Wolf     }
405960d5fb3SKevin Wolf 
406960d5fb3SKevin Wolf     /*
407960d5fb3SKevin Wolf      * Temporarily drop the lock across yield or we would get deadlocks.
408960d5fb3SKevin Wolf      * bdrv_co_drain_bh_cb() reaquires the lock as needed.
409960d5fb3SKevin Wolf      *
410960d5fb3SKevin Wolf      * When we yield below, the lock for the current context will be
411960d5fb3SKevin Wolf      * released, so if this is actually the lock that protects bs, don't drop
412960d5fb3SKevin Wolf      * it a second time.
413960d5fb3SKevin Wolf      */
414960d5fb3SKevin Wolf     if (ctx != co_ctx) {
415960d5fb3SKevin Wolf         aio_context_release(ctx);
416960d5fb3SKevin Wolf     }
417960d5fb3SKevin Wolf     replay_bh_schedule_oneshot_event(ctx, bdrv_co_drain_bh_cb, &data);
418a77fd4bbSFam Zheng 
419a77fd4bbSFam Zheng     qemu_coroutine_yield();
420a77fd4bbSFam Zheng     /* If we are resumed from some other event (such as an aio completion or a
421a77fd4bbSFam Zheng      * timer callback), it is a bug in the caller that should be fixed. */
422a77fd4bbSFam Zheng     assert(data.done);
423960d5fb3SKevin Wolf 
424960d5fb3SKevin Wolf     /* Reaquire the AioContext of bs if we dropped it */
425960d5fb3SKevin Wolf     if (ctx != co_ctx) {
426960d5fb3SKevin Wolf         aio_context_acquire(ctx);
427960d5fb3SKevin Wolf     }
428a77fd4bbSFam Zheng }
429a77fd4bbSFam Zheng 
430dcf94a23SKevin Wolf void bdrv_do_drained_begin_quiesce(BlockDriverState *bs,
4316cd5c9d7SKevin Wolf                                    BdrvChild *parent, bool ignore_bds_parents)
432dcf94a23SKevin Wolf {
433384a48fbSEmanuele Giuseppe Esposito     IO_OR_GS_CODE();
434dcf94a23SKevin Wolf     assert(!qemu_in_coroutine());
435dcf94a23SKevin Wolf 
436dcf94a23SKevin Wolf     /* Stop things in parent-to-child order */
437d73415a3SStefan Hajnoczi     if (qatomic_fetch_inc(&bs->quiesce_counter) == 0) {
438dcf94a23SKevin Wolf         aio_disable_external(bdrv_get_aio_context(bs));
439dcf94a23SKevin Wolf     }
440dcf94a23SKevin Wolf 
4416cd5c9d7SKevin Wolf     bdrv_parent_drained_begin(bs, parent, ignore_bds_parents);
4428e1da77eSMax Reitz     bdrv_drain_invoke(bs, true, NULL);
443dcf94a23SKevin Wolf }
444dcf94a23SKevin Wolf 
445dcf94a23SKevin Wolf static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
4466cd5c9d7SKevin Wolf                                   BdrvChild *parent, bool ignore_bds_parents,
4476cd5c9d7SKevin Wolf                                   bool poll)
4486820643fSKevin Wolf {
449b0165585SKevin Wolf     BdrvChild *child, *next;
450b0165585SKevin Wolf 
451d42cf288SPaolo Bonzini     if (qemu_in_coroutine()) {
4526cd5c9d7SKevin Wolf         bdrv_co_yield_to_drain(bs, true, recursive, parent, ignore_bds_parents,
4538e1da77eSMax Reitz                                poll, NULL);
454d42cf288SPaolo Bonzini         return;
455d42cf288SPaolo Bonzini     }
456d42cf288SPaolo Bonzini 
4576cd5c9d7SKevin Wolf     bdrv_do_drained_begin_quiesce(bs, parent, ignore_bds_parents);
458d30b8e64SKevin Wolf 
459b0165585SKevin Wolf     if (recursive) {
4606cd5c9d7SKevin Wolf         assert(!ignore_bds_parents);
461d736f119SKevin Wolf         bs->recursive_quiesce_counter++;
462b0165585SKevin Wolf         QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
4636cd5c9d7SKevin Wolf             bdrv_do_drained_begin(child->bs, true, child, ignore_bds_parents,
4646cd5c9d7SKevin Wolf                                   false);
465b0165585SKevin Wolf         }
466b0165585SKevin Wolf     }
467fe4f0614SKevin Wolf 
468fe4f0614SKevin Wolf     /*
469fe4f0614SKevin Wolf      * Wait for drained requests to finish.
470fe4f0614SKevin Wolf      *
471fe4f0614SKevin Wolf      * Calling BDRV_POLL_WHILE() only once for the top-level node is okay: The
472fe4f0614SKevin Wolf      * call is needed so things in this AioContext can make progress even
473fe4f0614SKevin Wolf      * though we don't return to the main AioContext loop - this automatically
474fe4f0614SKevin Wolf      * includes other nodes in the same AioContext and therefore all child
475fe4f0614SKevin Wolf      * nodes.
476fe4f0614SKevin Wolf      */
477fe4f0614SKevin Wolf     if (poll) {
4786cd5c9d7SKevin Wolf         assert(!ignore_bds_parents);
479fe4f0614SKevin Wolf         BDRV_POLL_WHILE(bs, bdrv_drain_poll_top_level(bs, recursive, parent));
480fe4f0614SKevin Wolf     }
4816820643fSKevin Wolf }
4826820643fSKevin Wolf 
4830152bf40SKevin Wolf void bdrv_drained_begin(BlockDriverState *bs)
4840152bf40SKevin Wolf {
485384a48fbSEmanuele Giuseppe Esposito     IO_OR_GS_CODE();
4866cd5c9d7SKevin Wolf     bdrv_do_drained_begin(bs, false, NULL, false, true);
4870152bf40SKevin Wolf }
4880152bf40SKevin Wolf 
489b0165585SKevin Wolf void bdrv_subtree_drained_begin(BlockDriverState *bs)
4906820643fSKevin Wolf {
491384a48fbSEmanuele Giuseppe Esposito     IO_OR_GS_CODE();
4926cd5c9d7SKevin Wolf     bdrv_do_drained_begin(bs, true, NULL, false, true);
493b0165585SKevin Wolf }
494b0165585SKevin Wolf 
495e037c09cSMax Reitz /**
496e037c09cSMax Reitz  * This function does not poll, nor must any of its recursively called
497e037c09cSMax Reitz  * functions.  The *drained_end_counter pointee will be incremented
498e037c09cSMax Reitz  * once for every background operation scheduled, and decremented once
499e037c09cSMax Reitz  * the operation settles.  Therefore, the pointer must remain valid
500e037c09cSMax Reitz  * until the pointee reaches 0.  That implies that whoever sets up the
501e037c09cSMax Reitz  * pointee has to poll until it is 0.
502e037c09cSMax Reitz  *
503e037c09cSMax Reitz  * We use atomic operations to access *drained_end_counter, because
504e037c09cSMax Reitz  * (1) when called from bdrv_set_aio_context_ignore(), the subgraph of
505e037c09cSMax Reitz  *     @bs may contain nodes in different AioContexts,
506e037c09cSMax Reitz  * (2) bdrv_drain_all_end() uses the same counter for all nodes,
507e037c09cSMax Reitz  *     regardless of which AioContext they are in.
508e037c09cSMax Reitz  */
5096cd5c9d7SKevin Wolf static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
5108e1da77eSMax Reitz                                 BdrvChild *parent, bool ignore_bds_parents,
5118e1da77eSMax Reitz                                 int *drained_end_counter)
512b0165585SKevin Wolf {
51361ad631cSMax Reitz     BdrvChild *child;
5140f115168SKevin Wolf     int old_quiesce_counter;
5150f115168SKevin Wolf 
516e037c09cSMax Reitz     assert(drained_end_counter != NULL);
517e037c09cSMax Reitz 
518481cad48SManos Pitsidianakis     if (qemu_in_coroutine()) {
5196cd5c9d7SKevin Wolf         bdrv_co_yield_to_drain(bs, false, recursive, parent, ignore_bds_parents,
5208e1da77eSMax Reitz                                false, drained_end_counter);
521481cad48SManos Pitsidianakis         return;
522481cad48SManos Pitsidianakis     }
5236820643fSKevin Wolf     assert(bs->quiesce_counter > 0);
5246820643fSKevin Wolf 
52560369b86SKevin Wolf     /* Re-enable things in child-to-parent order */
5268e1da77eSMax Reitz     bdrv_drain_invoke(bs, false, drained_end_counter);
527e037c09cSMax Reitz     bdrv_parent_drained_end(bs, parent, ignore_bds_parents,
528e037c09cSMax Reitz                             drained_end_counter);
5295cb2737eSMax Reitz 
530d73415a3SStefan Hajnoczi     old_quiesce_counter = qatomic_fetch_dec(&bs->quiesce_counter);
5310f115168SKevin Wolf     if (old_quiesce_counter == 1) {
5326820643fSKevin Wolf         aio_enable_external(bdrv_get_aio_context(bs));
5336820643fSKevin Wolf     }
534b0165585SKevin Wolf 
535b0165585SKevin Wolf     if (recursive) {
5366cd5c9d7SKevin Wolf         assert(!ignore_bds_parents);
537d736f119SKevin Wolf         bs->recursive_quiesce_counter--;
53861ad631cSMax Reitz         QLIST_FOREACH(child, &bs->children, next) {
5398e1da77eSMax Reitz             bdrv_do_drained_end(child->bs, true, child, ignore_bds_parents,
5408e1da77eSMax Reitz                                 drained_end_counter);
541b0165585SKevin Wolf         }
542b0165585SKevin Wolf     }
5430f115168SKevin Wolf }
5446820643fSKevin Wolf 
5450152bf40SKevin Wolf void bdrv_drained_end(BlockDriverState *bs)
5460152bf40SKevin Wolf {
547e037c09cSMax Reitz     int drained_end_counter = 0;
548384a48fbSEmanuele Giuseppe Esposito     IO_OR_GS_CODE();
549e037c09cSMax Reitz     bdrv_do_drained_end(bs, false, NULL, false, &drained_end_counter);
550d73415a3SStefan Hajnoczi     BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0);
551e037c09cSMax Reitz }
552e037c09cSMax Reitz 
553e037c09cSMax Reitz void bdrv_drained_end_no_poll(BlockDriverState *bs, int *drained_end_counter)
554e037c09cSMax Reitz {
555384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
556e037c09cSMax Reitz     bdrv_do_drained_end(bs, false, NULL, false, drained_end_counter);
557b0165585SKevin Wolf }
558b0165585SKevin Wolf 
559b0165585SKevin Wolf void bdrv_subtree_drained_end(BlockDriverState *bs)
560b0165585SKevin Wolf {
561e037c09cSMax Reitz     int drained_end_counter = 0;
562384a48fbSEmanuele Giuseppe Esposito     IO_OR_GS_CODE();
563e037c09cSMax Reitz     bdrv_do_drained_end(bs, true, NULL, false, &drained_end_counter);
564d73415a3SStefan Hajnoczi     BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0);
5650152bf40SKevin Wolf }
5660152bf40SKevin Wolf 
567d736f119SKevin Wolf void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent)
568d736f119SKevin Wolf {
569d736f119SKevin Wolf     int i;
570967d7905SEmanuele Giuseppe Esposito     IO_OR_GS_CODE();
571d736f119SKevin Wolf 
572d736f119SKevin Wolf     for (i = 0; i < new_parent->recursive_quiesce_counter; i++) {
5736cd5c9d7SKevin Wolf         bdrv_do_drained_begin(child->bs, true, child, false, true);
574d736f119SKevin Wolf     }
575d736f119SKevin Wolf }
576d736f119SKevin Wolf 
577d736f119SKevin Wolf void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent)
578d736f119SKevin Wolf {
579e037c09cSMax Reitz     int drained_end_counter = 0;
580d736f119SKevin Wolf     int i;
581967d7905SEmanuele Giuseppe Esposito     IO_OR_GS_CODE();
582d736f119SKevin Wolf 
583d736f119SKevin Wolf     for (i = 0; i < old_parent->recursive_quiesce_counter; i++) {
584e037c09cSMax Reitz         bdrv_do_drained_end(child->bs, true, child, false,
585e037c09cSMax Reitz                             &drained_end_counter);
586d736f119SKevin Wolf     }
587e037c09cSMax Reitz 
588d73415a3SStefan Hajnoczi     BDRV_POLL_WHILE(child->bs, qatomic_read(&drained_end_counter) > 0);
589d736f119SKevin Wolf }
590d736f119SKevin Wolf 
59161007b31SStefan Hajnoczi void bdrv_drain(BlockDriverState *bs)
59261007b31SStefan Hajnoczi {
593384a48fbSEmanuele Giuseppe Esposito     IO_OR_GS_CODE();
5946820643fSKevin Wolf     bdrv_drained_begin(bs);
5956820643fSKevin Wolf     bdrv_drained_end(bs);
59661007b31SStefan Hajnoczi }
59761007b31SStefan Hajnoczi 
598c13ad59fSKevin Wolf static void bdrv_drain_assert_idle(BlockDriverState *bs)
599c13ad59fSKevin Wolf {
600c13ad59fSKevin Wolf     BdrvChild *child, *next;
601c13ad59fSKevin Wolf 
602d73415a3SStefan Hajnoczi     assert(qatomic_read(&bs->in_flight) == 0);
603c13ad59fSKevin Wolf     QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
604c13ad59fSKevin Wolf         bdrv_drain_assert_idle(child->bs);
605c13ad59fSKevin Wolf     }
606c13ad59fSKevin Wolf }
607c13ad59fSKevin Wolf 
6080f12264eSKevin Wolf unsigned int bdrv_drain_all_count = 0;
6090f12264eSKevin Wolf 
6100f12264eSKevin Wolf static bool bdrv_drain_all_poll(void)
6110f12264eSKevin Wolf {
6120f12264eSKevin Wolf     BlockDriverState *bs = NULL;
6130f12264eSKevin Wolf     bool result = false;
614f791bf7fSEmanuele Giuseppe Esposito     GLOBAL_STATE_CODE();
6150f12264eSKevin Wolf 
6160f12264eSKevin Wolf     /* bdrv_drain_poll() can't make changes to the graph and we are holding the
6170f12264eSKevin Wolf      * main AioContext lock, so iterating bdrv_next_all_states() is safe. */
6180f12264eSKevin Wolf     while ((bs = bdrv_next_all_states(bs))) {
6190f12264eSKevin Wolf         AioContext *aio_context = bdrv_get_aio_context(bs);
6200f12264eSKevin Wolf         aio_context_acquire(aio_context);
6210f12264eSKevin Wolf         result |= bdrv_drain_poll(bs, false, NULL, true);
6220f12264eSKevin Wolf         aio_context_release(aio_context);
6230f12264eSKevin Wolf     }
6240f12264eSKevin Wolf 
6250f12264eSKevin Wolf     return result;
6260f12264eSKevin Wolf }
6270f12264eSKevin Wolf 
62861007b31SStefan Hajnoczi /*
62961007b31SStefan Hajnoczi  * Wait for pending requests to complete across all BlockDriverStates
63061007b31SStefan Hajnoczi  *
63161007b31SStefan Hajnoczi  * This function does not flush data to disk, use bdrv_flush_all() for that
63261007b31SStefan Hajnoczi  * after calling this function.
633c0778f66SAlberto Garcia  *
634c0778f66SAlberto Garcia  * This pauses all block jobs and disables external clients. It must
635c0778f66SAlberto Garcia  * be paired with bdrv_drain_all_end().
636c0778f66SAlberto Garcia  *
637c0778f66SAlberto Garcia  * NOTE: no new block jobs or BlockDriverStates can be created between
638c0778f66SAlberto Garcia  * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls.
63961007b31SStefan Hajnoczi  */
640c0778f66SAlberto Garcia void bdrv_drain_all_begin(void)
64161007b31SStefan Hajnoczi {
6420f12264eSKevin Wolf     BlockDriverState *bs = NULL;
643f791bf7fSEmanuele Giuseppe Esposito     GLOBAL_STATE_CODE();
64461007b31SStefan Hajnoczi 
645c8ca33d0SKevin Wolf     if (qemu_in_coroutine()) {
6468e1da77eSMax Reitz         bdrv_co_yield_to_drain(NULL, true, false, NULL, true, true, NULL);
647c8ca33d0SKevin Wolf         return;
648c8ca33d0SKevin Wolf     }
649c8ca33d0SKevin Wolf 
650c8aa7895SPavel Dovgalyuk     /*
651c8aa7895SPavel Dovgalyuk      * bdrv queue is managed by record/replay,
652c8aa7895SPavel Dovgalyuk      * waiting for finishing the I/O requests may
653c8aa7895SPavel Dovgalyuk      * be infinite
654c8aa7895SPavel Dovgalyuk      */
655c8aa7895SPavel Dovgalyuk     if (replay_events_enabled()) {
656c8aa7895SPavel Dovgalyuk         return;
657c8aa7895SPavel Dovgalyuk     }
658c8aa7895SPavel Dovgalyuk 
6590f12264eSKevin Wolf     /* AIO_WAIT_WHILE() with a NULL context can only be called from the main
6600f12264eSKevin Wolf      * loop AioContext, so make sure we're in the main context. */
6619a7e86c8SKevin Wolf     assert(qemu_get_current_aio_context() == qemu_get_aio_context());
6620f12264eSKevin Wolf     assert(bdrv_drain_all_count < INT_MAX);
6630f12264eSKevin Wolf     bdrv_drain_all_count++;
6649a7e86c8SKevin Wolf 
6650f12264eSKevin Wolf     /* Quiesce all nodes, without polling in-flight requests yet. The graph
6660f12264eSKevin Wolf      * cannot change during this loop. */
6670f12264eSKevin Wolf     while ((bs = bdrv_next_all_states(bs))) {
66861007b31SStefan Hajnoczi         AioContext *aio_context = bdrv_get_aio_context(bs);
66961007b31SStefan Hajnoczi 
67061007b31SStefan Hajnoczi         aio_context_acquire(aio_context);
6710f12264eSKevin Wolf         bdrv_do_drained_begin(bs, false, NULL, true, false);
67261007b31SStefan Hajnoczi         aio_context_release(aio_context);
67361007b31SStefan Hajnoczi     }
67461007b31SStefan Hajnoczi 
6750f12264eSKevin Wolf     /* Now poll the in-flight requests */
676cfe29d82SKevin Wolf     AIO_WAIT_WHILE(NULL, bdrv_drain_all_poll());
6770f12264eSKevin Wolf 
6780f12264eSKevin Wolf     while ((bs = bdrv_next_all_states(bs))) {
679c13ad59fSKevin Wolf         bdrv_drain_assert_idle(bs);
680f406c03cSAlexander Yarygin     }
681f406c03cSAlexander Yarygin }
682c0778f66SAlberto Garcia 
6831a6d3bd2SGreg Kurz void bdrv_drain_all_end_quiesce(BlockDriverState *bs)
6841a6d3bd2SGreg Kurz {
6851a6d3bd2SGreg Kurz     int drained_end_counter = 0;
686b4ad82aaSEmanuele Giuseppe Esposito     GLOBAL_STATE_CODE();
6871a6d3bd2SGreg Kurz 
6881a6d3bd2SGreg Kurz     g_assert(bs->quiesce_counter > 0);
6891a6d3bd2SGreg Kurz     g_assert(!bs->refcnt);
6901a6d3bd2SGreg Kurz 
6911a6d3bd2SGreg Kurz     while (bs->quiesce_counter) {
6921a6d3bd2SGreg Kurz         bdrv_do_drained_end(bs, false, NULL, true, &drained_end_counter);
6931a6d3bd2SGreg Kurz     }
6941a6d3bd2SGreg Kurz     BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0);
6951a6d3bd2SGreg Kurz }
6961a6d3bd2SGreg Kurz 
697c0778f66SAlberto Garcia void bdrv_drain_all_end(void)
698c0778f66SAlberto Garcia {
6990f12264eSKevin Wolf     BlockDriverState *bs = NULL;
700e037c09cSMax Reitz     int drained_end_counter = 0;
701f791bf7fSEmanuele Giuseppe Esposito     GLOBAL_STATE_CODE();
702c0778f66SAlberto Garcia 
703c8aa7895SPavel Dovgalyuk     /*
704c8aa7895SPavel Dovgalyuk      * bdrv queue is managed by record/replay,
705c8aa7895SPavel Dovgalyuk      * waiting for finishing the I/O requests may
706c8aa7895SPavel Dovgalyuk      * be endless
707c8aa7895SPavel Dovgalyuk      */
708c8aa7895SPavel Dovgalyuk     if (replay_events_enabled()) {
709c8aa7895SPavel Dovgalyuk         return;
710c8aa7895SPavel Dovgalyuk     }
711c8aa7895SPavel Dovgalyuk 
7120f12264eSKevin Wolf     while ((bs = bdrv_next_all_states(bs))) {
71361007b31SStefan Hajnoczi         AioContext *aio_context = bdrv_get_aio_context(bs);
71461007b31SStefan Hajnoczi 
71561007b31SStefan Hajnoczi         aio_context_acquire(aio_context);
716e037c09cSMax Reitz         bdrv_do_drained_end(bs, false, NULL, true, &drained_end_counter);
71761007b31SStefan Hajnoczi         aio_context_release(aio_context);
71861007b31SStefan Hajnoczi     }
7190f12264eSKevin Wolf 
720e037c09cSMax Reitz     assert(qemu_get_current_aio_context() == qemu_get_aio_context());
721d73415a3SStefan Hajnoczi     AIO_WAIT_WHILE(NULL, qatomic_read(&drained_end_counter) > 0);
722e037c09cSMax Reitz 
7230f12264eSKevin Wolf     assert(bdrv_drain_all_count > 0);
7240f12264eSKevin Wolf     bdrv_drain_all_count--;
72561007b31SStefan Hajnoczi }
72661007b31SStefan Hajnoczi 
727c0778f66SAlberto Garcia void bdrv_drain_all(void)
728c0778f66SAlberto Garcia {
729f791bf7fSEmanuele Giuseppe Esposito     GLOBAL_STATE_CODE();
730c0778f66SAlberto Garcia     bdrv_drain_all_begin();
731c0778f66SAlberto Garcia     bdrv_drain_all_end();
732c0778f66SAlberto Garcia }
733c0778f66SAlberto Garcia 
73461007b31SStefan Hajnoczi /**
73561007b31SStefan Hajnoczi  * Remove an active request from the tracked requests list
73661007b31SStefan Hajnoczi  *
73761007b31SStefan Hajnoczi  * This function should be called when a tracked request is completing.
73861007b31SStefan Hajnoczi  */
739f0d43b1eSPaolo Bonzini static void coroutine_fn tracked_request_end(BdrvTrackedRequest *req)
74061007b31SStefan Hajnoczi {
74161007b31SStefan Hajnoczi     if (req->serialising) {
742d73415a3SStefan Hajnoczi         qatomic_dec(&req->bs->serialising_in_flight);
74361007b31SStefan Hajnoczi     }
74461007b31SStefan Hajnoczi 
7453783fa3dSPaolo Bonzini     qemu_co_mutex_lock(&req->bs->reqs_lock);
74661007b31SStefan Hajnoczi     QLIST_REMOVE(req, list);
74761007b31SStefan Hajnoczi     qemu_co_queue_restart_all(&req->wait_queue);
7483783fa3dSPaolo Bonzini     qemu_co_mutex_unlock(&req->bs->reqs_lock);
74961007b31SStefan Hajnoczi }
75061007b31SStefan Hajnoczi 
75161007b31SStefan Hajnoczi /**
75261007b31SStefan Hajnoczi  * Add an active request to the tracked requests list
75361007b31SStefan Hajnoczi  */
754881a4c55SPaolo Bonzini static void coroutine_fn tracked_request_begin(BdrvTrackedRequest *req,
75561007b31SStefan Hajnoczi                                                BlockDriverState *bs,
75661007b31SStefan Hajnoczi                                                int64_t offset,
75780247264SEric Blake                                                int64_t bytes,
758ebde595cSFam Zheng                                                enum BdrvTrackedRequestType type)
75961007b31SStefan Hajnoczi {
76080247264SEric Blake     bdrv_check_request(offset, bytes, &error_abort);
76122931a15SFam Zheng 
76261007b31SStefan Hajnoczi     *req = (BdrvTrackedRequest){
76361007b31SStefan Hajnoczi         .bs = bs,
76461007b31SStefan Hajnoczi         .offset         = offset,
76561007b31SStefan Hajnoczi         .bytes          = bytes,
766ebde595cSFam Zheng         .type           = type,
76761007b31SStefan Hajnoczi         .co             = qemu_coroutine_self(),
76861007b31SStefan Hajnoczi         .serialising    = false,
76961007b31SStefan Hajnoczi         .overlap_offset = offset,
77061007b31SStefan Hajnoczi         .overlap_bytes  = bytes,
77161007b31SStefan Hajnoczi     };
77261007b31SStefan Hajnoczi 
77361007b31SStefan Hajnoczi     qemu_co_queue_init(&req->wait_queue);
77461007b31SStefan Hajnoczi 
7753783fa3dSPaolo Bonzini     qemu_co_mutex_lock(&bs->reqs_lock);
77661007b31SStefan Hajnoczi     QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
7773783fa3dSPaolo Bonzini     qemu_co_mutex_unlock(&bs->reqs_lock);
77861007b31SStefan Hajnoczi }
77961007b31SStefan Hajnoczi 
7803ba0e1a0SPaolo Bonzini static bool tracked_request_overlaps(BdrvTrackedRequest *req,
78180247264SEric Blake                                      int64_t offset, int64_t bytes)
7823ba0e1a0SPaolo Bonzini {
78380247264SEric Blake     bdrv_check_request(offset, bytes, &error_abort);
78480247264SEric Blake 
7853ba0e1a0SPaolo Bonzini     /*        aaaa   bbbb */
7863ba0e1a0SPaolo Bonzini     if (offset >= req->overlap_offset + req->overlap_bytes) {
7873ba0e1a0SPaolo Bonzini         return false;
7883ba0e1a0SPaolo Bonzini     }
7893ba0e1a0SPaolo Bonzini     /* bbbb   aaaa        */
7903ba0e1a0SPaolo Bonzini     if (req->overlap_offset >= offset + bytes) {
7913ba0e1a0SPaolo Bonzini         return false;
7923ba0e1a0SPaolo Bonzini     }
7933ba0e1a0SPaolo Bonzini     return true;
7943ba0e1a0SPaolo Bonzini }
7953ba0e1a0SPaolo Bonzini 
7963183937fSVladimir Sementsov-Ogievskiy /* Called with self->bs->reqs_lock held */
797881a4c55SPaolo Bonzini static coroutine_fn BdrvTrackedRequest *
7983183937fSVladimir Sementsov-Ogievskiy bdrv_find_conflicting_request(BdrvTrackedRequest *self)
7993ba0e1a0SPaolo Bonzini {
8003ba0e1a0SPaolo Bonzini     BdrvTrackedRequest *req;
8013ba0e1a0SPaolo Bonzini 
8023183937fSVladimir Sementsov-Ogievskiy     QLIST_FOREACH(req, &self->bs->tracked_requests, list) {
8033ba0e1a0SPaolo Bonzini         if (req == self || (!req->serialising && !self->serialising)) {
8043ba0e1a0SPaolo Bonzini             continue;
8053ba0e1a0SPaolo Bonzini         }
8063ba0e1a0SPaolo Bonzini         if (tracked_request_overlaps(req, self->overlap_offset,
8073ba0e1a0SPaolo Bonzini                                      self->overlap_bytes))
8083ba0e1a0SPaolo Bonzini         {
8093183937fSVladimir Sementsov-Ogievskiy             /*
8103183937fSVladimir Sementsov-Ogievskiy              * Hitting this means there was a reentrant request, for
8113ba0e1a0SPaolo Bonzini              * example, a block driver issuing nested requests.  This must
8123ba0e1a0SPaolo Bonzini              * never happen since it means deadlock.
8133ba0e1a0SPaolo Bonzini              */
8143ba0e1a0SPaolo Bonzini             assert(qemu_coroutine_self() != req->co);
8153ba0e1a0SPaolo Bonzini 
8163183937fSVladimir Sementsov-Ogievskiy             /*
8173183937fSVladimir Sementsov-Ogievskiy              * If the request is already (indirectly) waiting for us, or
8183ba0e1a0SPaolo Bonzini              * will wait for us as soon as it wakes up, then just go on
8193183937fSVladimir Sementsov-Ogievskiy              * (instead of producing a deadlock in the former case).
8203183937fSVladimir Sementsov-Ogievskiy              */
8213ba0e1a0SPaolo Bonzini             if (!req->waiting_for) {
8223183937fSVladimir Sementsov-Ogievskiy                 return req;
8233183937fSVladimir Sementsov-Ogievskiy             }
8243183937fSVladimir Sementsov-Ogievskiy         }
8253183937fSVladimir Sementsov-Ogievskiy     }
8263183937fSVladimir Sementsov-Ogievskiy 
8273183937fSVladimir Sementsov-Ogievskiy     return NULL;
8283183937fSVladimir Sementsov-Ogievskiy }
8293183937fSVladimir Sementsov-Ogievskiy 
830ec1c8868SVladimir Sementsov-Ogievskiy /* Called with self->bs->reqs_lock held */
831131498f7SDenis V. Lunev static void coroutine_fn
832ec1c8868SVladimir Sementsov-Ogievskiy bdrv_wait_serialising_requests_locked(BdrvTrackedRequest *self)
8333183937fSVladimir Sementsov-Ogievskiy {
8343183937fSVladimir Sementsov-Ogievskiy     BdrvTrackedRequest *req;
8353183937fSVladimir Sementsov-Ogievskiy 
8363183937fSVladimir Sementsov-Ogievskiy     while ((req = bdrv_find_conflicting_request(self))) {
8373ba0e1a0SPaolo Bonzini         self->waiting_for = req;
838ec1c8868SVladimir Sementsov-Ogievskiy         qemu_co_queue_wait(&req->wait_queue, &self->bs->reqs_lock);
8393ba0e1a0SPaolo Bonzini         self->waiting_for = NULL;
8403ba0e1a0SPaolo Bonzini     }
8413ba0e1a0SPaolo Bonzini }
8423ba0e1a0SPaolo Bonzini 
8438ac5aab2SVladimir Sementsov-Ogievskiy /* Called with req->bs->reqs_lock held */
8448ac5aab2SVladimir Sementsov-Ogievskiy static void tracked_request_set_serialising(BdrvTrackedRequest *req,
8458ac5aab2SVladimir Sementsov-Ogievskiy                                             uint64_t align)
84661007b31SStefan Hajnoczi {
84761007b31SStefan Hajnoczi     int64_t overlap_offset = req->offset & ~(align - 1);
84880247264SEric Blake     int64_t overlap_bytes =
84980247264SEric Blake         ROUND_UP(req->offset + req->bytes, align) - overlap_offset;
85080247264SEric Blake 
85180247264SEric Blake     bdrv_check_request(req->offset, req->bytes, &error_abort);
85261007b31SStefan Hajnoczi 
85361007b31SStefan Hajnoczi     if (!req->serialising) {
854d73415a3SStefan Hajnoczi         qatomic_inc(&req->bs->serialising_in_flight);
85561007b31SStefan Hajnoczi         req->serialising = true;
85661007b31SStefan Hajnoczi     }
85761007b31SStefan Hajnoczi 
85861007b31SStefan Hajnoczi     req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
85961007b31SStefan Hajnoczi     req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
86009d2f948SVladimir Sementsov-Ogievskiy }
86109d2f948SVladimir Sementsov-Ogievskiy 
86261007b31SStefan Hajnoczi /**
863c28107e9SMax Reitz  * Return the tracked request on @bs for the current coroutine, or
864c28107e9SMax Reitz  * NULL if there is none.
865c28107e9SMax Reitz  */
866c28107e9SMax Reitz BdrvTrackedRequest *coroutine_fn bdrv_co_get_self_request(BlockDriverState *bs)
867c28107e9SMax Reitz {
868c28107e9SMax Reitz     BdrvTrackedRequest *req;
869c28107e9SMax Reitz     Coroutine *self = qemu_coroutine_self();
870967d7905SEmanuele Giuseppe Esposito     IO_CODE();
871c28107e9SMax Reitz 
872c28107e9SMax Reitz     QLIST_FOREACH(req, &bs->tracked_requests, list) {
873c28107e9SMax Reitz         if (req->co == self) {
874c28107e9SMax Reitz             return req;
875c28107e9SMax Reitz         }
876c28107e9SMax Reitz     }
877c28107e9SMax Reitz 
878c28107e9SMax Reitz     return NULL;
879c28107e9SMax Reitz }
880c28107e9SMax Reitz 
881c28107e9SMax Reitz /**
882244483e6SKevin Wolf  * Round a region to cluster boundaries
883244483e6SKevin Wolf  */
884244483e6SKevin Wolf void bdrv_round_to_clusters(BlockDriverState *bs,
8857cfd5275SEric Blake                             int64_t offset, int64_t bytes,
886244483e6SKevin Wolf                             int64_t *cluster_offset,
8877cfd5275SEric Blake                             int64_t *cluster_bytes)
888244483e6SKevin Wolf {
889244483e6SKevin Wolf     BlockDriverInfo bdi;
890384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
891244483e6SKevin Wolf     if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
892244483e6SKevin Wolf         *cluster_offset = offset;
893244483e6SKevin Wolf         *cluster_bytes = bytes;
894244483e6SKevin Wolf     } else {
895244483e6SKevin Wolf         int64_t c = bdi.cluster_size;
896244483e6SKevin Wolf         *cluster_offset = QEMU_ALIGN_DOWN(offset, c);
897244483e6SKevin Wolf         *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c);
898244483e6SKevin Wolf     }
899244483e6SKevin Wolf }
900244483e6SKevin Wolf 
90161007b31SStefan Hajnoczi static int bdrv_get_cluster_size(BlockDriverState *bs)
90261007b31SStefan Hajnoczi {
90361007b31SStefan Hajnoczi     BlockDriverInfo bdi;
90461007b31SStefan Hajnoczi     int ret;
90561007b31SStefan Hajnoczi 
90661007b31SStefan Hajnoczi     ret = bdrv_get_info(bs, &bdi);
90761007b31SStefan Hajnoczi     if (ret < 0 || bdi.cluster_size == 0) {
908a5b8dd2cSEric Blake         return bs->bl.request_alignment;
90961007b31SStefan Hajnoczi     } else {
91061007b31SStefan Hajnoczi         return bdi.cluster_size;
91161007b31SStefan Hajnoczi     }
91261007b31SStefan Hajnoczi }
91361007b31SStefan Hajnoczi 
91499723548SPaolo Bonzini void bdrv_inc_in_flight(BlockDriverState *bs)
91599723548SPaolo Bonzini {
916967d7905SEmanuele Giuseppe Esposito     IO_CODE();
917d73415a3SStefan Hajnoczi     qatomic_inc(&bs->in_flight);
91899723548SPaolo Bonzini }
91999723548SPaolo Bonzini 
920c9d1a561SPaolo Bonzini void bdrv_wakeup(BlockDriverState *bs)
921c9d1a561SPaolo Bonzini {
922967d7905SEmanuele Giuseppe Esposito     IO_CODE();
923cfe29d82SKevin Wolf     aio_wait_kick();
924c9d1a561SPaolo Bonzini }
925c9d1a561SPaolo Bonzini 
92699723548SPaolo Bonzini void bdrv_dec_in_flight(BlockDriverState *bs)
92799723548SPaolo Bonzini {
928967d7905SEmanuele Giuseppe Esposito     IO_CODE();
929d73415a3SStefan Hajnoczi     qatomic_dec(&bs->in_flight);
930c9d1a561SPaolo Bonzini     bdrv_wakeup(bs);
93199723548SPaolo Bonzini }
93299723548SPaolo Bonzini 
933131498f7SDenis V. Lunev static void coroutine_fn
934131498f7SDenis V. Lunev bdrv_wait_serialising_requests(BdrvTrackedRequest *self)
93561007b31SStefan Hajnoczi {
93661007b31SStefan Hajnoczi     BlockDriverState *bs = self->bs;
93761007b31SStefan Hajnoczi 
938d73415a3SStefan Hajnoczi     if (!qatomic_read(&bs->serialising_in_flight)) {
939131498f7SDenis V. Lunev         return;
94061007b31SStefan Hajnoczi     }
94161007b31SStefan Hajnoczi 
9423783fa3dSPaolo Bonzini     qemu_co_mutex_lock(&bs->reqs_lock);
943131498f7SDenis V. Lunev     bdrv_wait_serialising_requests_locked(self);
9443783fa3dSPaolo Bonzini     qemu_co_mutex_unlock(&bs->reqs_lock);
94561007b31SStefan Hajnoczi }
94661007b31SStefan Hajnoczi 
947131498f7SDenis V. Lunev void coroutine_fn bdrv_make_request_serialising(BdrvTrackedRequest *req,
9488ac5aab2SVladimir Sementsov-Ogievskiy                                                 uint64_t align)
9498ac5aab2SVladimir Sementsov-Ogievskiy {
950967d7905SEmanuele Giuseppe Esposito     IO_CODE();
9518ac5aab2SVladimir Sementsov-Ogievskiy 
9528ac5aab2SVladimir Sementsov-Ogievskiy     qemu_co_mutex_lock(&req->bs->reqs_lock);
9538ac5aab2SVladimir Sementsov-Ogievskiy 
9548ac5aab2SVladimir Sementsov-Ogievskiy     tracked_request_set_serialising(req, align);
955131498f7SDenis V. Lunev     bdrv_wait_serialising_requests_locked(req);
9568ac5aab2SVladimir Sementsov-Ogievskiy 
9578ac5aab2SVladimir Sementsov-Ogievskiy     qemu_co_mutex_unlock(&req->bs->reqs_lock);
9588ac5aab2SVladimir Sementsov-Ogievskiy }
9598ac5aab2SVladimir Sementsov-Ogievskiy 
960558902ccSVladimir Sementsov-Ogievskiy int bdrv_check_qiov_request(int64_t offset, int64_t bytes,
96163f4ad11SVladimir Sementsov-Ogievskiy                             QEMUIOVector *qiov, size_t qiov_offset,
96263f4ad11SVladimir Sementsov-Ogievskiy                             Error **errp)
96361007b31SStefan Hajnoczi {
96463f4ad11SVladimir Sementsov-Ogievskiy     /*
96563f4ad11SVladimir Sementsov-Ogievskiy      * Check generic offset/bytes correctness
96663f4ad11SVladimir Sementsov-Ogievskiy      */
96763f4ad11SVladimir Sementsov-Ogievskiy 
96869b55e03SVladimir Sementsov-Ogievskiy     if (offset < 0) {
96969b55e03SVladimir Sementsov-Ogievskiy         error_setg(errp, "offset is negative: %" PRIi64, offset);
97069b55e03SVladimir Sementsov-Ogievskiy         return -EIO;
97169b55e03SVladimir Sementsov-Ogievskiy     }
97269b55e03SVladimir Sementsov-Ogievskiy 
97369b55e03SVladimir Sementsov-Ogievskiy     if (bytes < 0) {
97469b55e03SVladimir Sementsov-Ogievskiy         error_setg(errp, "bytes is negative: %" PRIi64, bytes);
97561007b31SStefan Hajnoczi         return -EIO;
97661007b31SStefan Hajnoczi     }
97761007b31SStefan Hajnoczi 
9788b117001SVladimir Sementsov-Ogievskiy     if (bytes > BDRV_MAX_LENGTH) {
97969b55e03SVladimir Sementsov-Ogievskiy         error_setg(errp, "bytes(%" PRIi64 ") exceeds maximum(%" PRIi64 ")",
98069b55e03SVladimir Sementsov-Ogievskiy                    bytes, BDRV_MAX_LENGTH);
98169b55e03SVladimir Sementsov-Ogievskiy         return -EIO;
98269b55e03SVladimir Sementsov-Ogievskiy     }
98369b55e03SVladimir Sementsov-Ogievskiy 
98469b55e03SVladimir Sementsov-Ogievskiy     if (offset > BDRV_MAX_LENGTH) {
98569b55e03SVladimir Sementsov-Ogievskiy         error_setg(errp, "offset(%" PRIi64 ") exceeds maximum(%" PRIi64 ")",
98669b55e03SVladimir Sementsov-Ogievskiy                    offset, BDRV_MAX_LENGTH);
9878b117001SVladimir Sementsov-Ogievskiy         return -EIO;
9888b117001SVladimir Sementsov-Ogievskiy     }
9898b117001SVladimir Sementsov-Ogievskiy 
9908b117001SVladimir Sementsov-Ogievskiy     if (offset > BDRV_MAX_LENGTH - bytes) {
99169b55e03SVladimir Sementsov-Ogievskiy         error_setg(errp, "sum of offset(%" PRIi64 ") and bytes(%" PRIi64 ") "
99269b55e03SVladimir Sementsov-Ogievskiy                    "exceeds maximum(%" PRIi64 ")", offset, bytes,
99369b55e03SVladimir Sementsov-Ogievskiy                    BDRV_MAX_LENGTH);
9948b117001SVladimir Sementsov-Ogievskiy         return -EIO;
9958b117001SVladimir Sementsov-Ogievskiy     }
9968b117001SVladimir Sementsov-Ogievskiy 
99763f4ad11SVladimir Sementsov-Ogievskiy     if (!qiov) {
9988b117001SVladimir Sementsov-Ogievskiy         return 0;
9998b117001SVladimir Sementsov-Ogievskiy     }
10008b117001SVladimir Sementsov-Ogievskiy 
100163f4ad11SVladimir Sementsov-Ogievskiy     /*
100263f4ad11SVladimir Sementsov-Ogievskiy      * Check qiov and qiov_offset
100363f4ad11SVladimir Sementsov-Ogievskiy      */
100463f4ad11SVladimir Sementsov-Ogievskiy 
100563f4ad11SVladimir Sementsov-Ogievskiy     if (qiov_offset > qiov->size) {
100663f4ad11SVladimir Sementsov-Ogievskiy         error_setg(errp, "qiov_offset(%zu) overflow io vector size(%zu)",
100763f4ad11SVladimir Sementsov-Ogievskiy                    qiov_offset, qiov->size);
100863f4ad11SVladimir Sementsov-Ogievskiy         return -EIO;
100963f4ad11SVladimir Sementsov-Ogievskiy     }
101063f4ad11SVladimir Sementsov-Ogievskiy 
101163f4ad11SVladimir Sementsov-Ogievskiy     if (bytes > qiov->size - qiov_offset) {
101263f4ad11SVladimir Sementsov-Ogievskiy         error_setg(errp, "bytes(%" PRIi64 ") + qiov_offset(%zu) overflow io "
101363f4ad11SVladimir Sementsov-Ogievskiy                    "vector size(%zu)", bytes, qiov_offset, qiov->size);
101463f4ad11SVladimir Sementsov-Ogievskiy         return -EIO;
101563f4ad11SVladimir Sementsov-Ogievskiy     }
101663f4ad11SVladimir Sementsov-Ogievskiy 
101763f4ad11SVladimir Sementsov-Ogievskiy     return 0;
101863f4ad11SVladimir Sementsov-Ogievskiy }
101963f4ad11SVladimir Sementsov-Ogievskiy 
102063f4ad11SVladimir Sementsov-Ogievskiy int bdrv_check_request(int64_t offset, int64_t bytes, Error **errp)
10218b117001SVladimir Sementsov-Ogievskiy {
102263f4ad11SVladimir Sementsov-Ogievskiy     return bdrv_check_qiov_request(offset, bytes, NULL, 0, errp);
102363f4ad11SVladimir Sementsov-Ogievskiy }
102463f4ad11SVladimir Sementsov-Ogievskiy 
102563f4ad11SVladimir Sementsov-Ogievskiy static int bdrv_check_request32(int64_t offset, int64_t bytes,
102663f4ad11SVladimir Sementsov-Ogievskiy                                 QEMUIOVector *qiov, size_t qiov_offset)
102763f4ad11SVladimir Sementsov-Ogievskiy {
102863f4ad11SVladimir Sementsov-Ogievskiy     int ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL);
10298b117001SVladimir Sementsov-Ogievskiy     if (ret < 0) {
10308b117001SVladimir Sementsov-Ogievskiy         return ret;
10318b117001SVladimir Sementsov-Ogievskiy     }
10328b117001SVladimir Sementsov-Ogievskiy 
10338b117001SVladimir Sementsov-Ogievskiy     if (bytes > BDRV_REQUEST_MAX_BYTES) {
103461007b31SStefan Hajnoczi         return -EIO;
103561007b31SStefan Hajnoczi     }
103661007b31SStefan Hajnoczi 
103761007b31SStefan Hajnoczi     return 0;
103861007b31SStefan Hajnoczi }
103961007b31SStefan Hajnoczi 
104061007b31SStefan Hajnoczi /*
104174021bc4SEric Blake  * Completely zero out a block device with the help of bdrv_pwrite_zeroes.
104261007b31SStefan Hajnoczi  * The operation is sped up by checking the block status and only writing
104361007b31SStefan Hajnoczi  * zeroes to the device if they currently do not return zeroes. Optional
104474021bc4SEric Blake  * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
1045465fe887SEric Blake  * BDRV_REQ_FUA).
104661007b31SStefan Hajnoczi  *
1047f4649069SEric Blake  * Returns < 0 on error, 0 on success. For error codes see bdrv_pwrite().
104861007b31SStefan Hajnoczi  */
1049720ff280SKevin Wolf int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags)
105061007b31SStefan Hajnoczi {
1051237d78f8SEric Blake     int ret;
1052237d78f8SEric Blake     int64_t target_size, bytes, offset = 0;
1053720ff280SKevin Wolf     BlockDriverState *bs = child->bs;
1054384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
105561007b31SStefan Hajnoczi 
10567286d610SEric Blake     target_size = bdrv_getlength(bs);
10577286d610SEric Blake     if (target_size < 0) {
10587286d610SEric Blake         return target_size;
105961007b31SStefan Hajnoczi     }
106061007b31SStefan Hajnoczi 
106161007b31SStefan Hajnoczi     for (;;) {
10627286d610SEric Blake         bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES);
10637286d610SEric Blake         if (bytes <= 0) {
106461007b31SStefan Hajnoczi             return 0;
106561007b31SStefan Hajnoczi         }
1066237d78f8SEric Blake         ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL);
106761007b31SStefan Hajnoczi         if (ret < 0) {
106861007b31SStefan Hajnoczi             return ret;
106961007b31SStefan Hajnoczi         }
107061007b31SStefan Hajnoczi         if (ret & BDRV_BLOCK_ZERO) {
1071237d78f8SEric Blake             offset += bytes;
107261007b31SStefan Hajnoczi             continue;
107361007b31SStefan Hajnoczi         }
1074237d78f8SEric Blake         ret = bdrv_pwrite_zeroes(child, offset, bytes, flags);
107561007b31SStefan Hajnoczi         if (ret < 0) {
107661007b31SStefan Hajnoczi             return ret;
107761007b31SStefan Hajnoczi         }
1078237d78f8SEric Blake         offset += bytes;
107961007b31SStefan Hajnoczi     }
108061007b31SStefan Hajnoczi }
108161007b31SStefan Hajnoczi 
108261007b31SStefan Hajnoczi /*
108361007b31SStefan Hajnoczi  * Writes to the file and ensures that no writes are reordered across this
108461007b31SStefan Hajnoczi  * request (acts as a barrier)
108561007b31SStefan Hajnoczi  *
108661007b31SStefan Hajnoczi  * Returns 0 on success, -errno in error cases.
108761007b31SStefan Hajnoczi  */
1088e97190a4SAlberto Faria int coroutine_fn bdrv_co_pwrite_sync(BdrvChild *child, int64_t offset,
1089e97190a4SAlberto Faria                                      int64_t bytes, const void *buf,
1090e97190a4SAlberto Faria                                      BdrvRequestFlags flags)
109161007b31SStefan Hajnoczi {
109261007b31SStefan Hajnoczi     int ret;
1093384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
109461007b31SStefan Hajnoczi 
1095e97190a4SAlberto Faria     ret = bdrv_co_pwrite(child, offset, bytes, buf, flags);
109661007b31SStefan Hajnoczi     if (ret < 0) {
109761007b31SStefan Hajnoczi         return ret;
109861007b31SStefan Hajnoczi     }
109961007b31SStefan Hajnoczi 
1100e97190a4SAlberto Faria     ret = bdrv_co_flush(child->bs);
1101855a6a93SKevin Wolf     if (ret < 0) {
1102855a6a93SKevin Wolf         return ret;
110361007b31SStefan Hajnoczi     }
110461007b31SStefan Hajnoczi 
110561007b31SStefan Hajnoczi     return 0;
110661007b31SStefan Hajnoczi }
110761007b31SStefan Hajnoczi 
110808844473SKevin Wolf typedef struct CoroutineIOCompletion {
110908844473SKevin Wolf     Coroutine *coroutine;
111008844473SKevin Wolf     int ret;
111108844473SKevin Wolf } CoroutineIOCompletion;
111208844473SKevin Wolf 
111308844473SKevin Wolf static void bdrv_co_io_em_complete(void *opaque, int ret)
111408844473SKevin Wolf {
111508844473SKevin Wolf     CoroutineIOCompletion *co = opaque;
111608844473SKevin Wolf 
111708844473SKevin Wolf     co->ret = ret;
1118b9e413ddSPaolo Bonzini     aio_co_wake(co->coroutine);
111908844473SKevin Wolf }
112008844473SKevin Wolf 
1121166fe960SKevin Wolf static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
112217abcbeeSVladimir Sementsov-Ogievskiy                                            int64_t offset, int64_t bytes,
1123ac850bf0SVladimir Sementsov-Ogievskiy                                            QEMUIOVector *qiov,
1124ac850bf0SVladimir Sementsov-Ogievskiy                                            size_t qiov_offset, int flags)
1125166fe960SKevin Wolf {
1126166fe960SKevin Wolf     BlockDriver *drv = bs->drv;
11273fb06697SKevin Wolf     int64_t sector_num;
11283fb06697SKevin Wolf     unsigned int nb_sectors;
1129ac850bf0SVladimir Sementsov-Ogievskiy     QEMUIOVector local_qiov;
1130ac850bf0SVladimir Sementsov-Ogievskiy     int ret;
11313fb06697SKevin Wolf 
113217abcbeeSVladimir Sementsov-Ogievskiy     bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1133fa166538SEric Blake     assert(!(flags & ~BDRV_REQ_MASK));
1134fe0480d6SKevin Wolf     assert(!(flags & BDRV_REQ_NO_FALLBACK));
1135fa166538SEric Blake 
1136d470ad42SMax Reitz     if (!drv) {
1137d470ad42SMax Reitz         return -ENOMEDIUM;
1138d470ad42SMax Reitz     }
1139d470ad42SMax Reitz 
1140ac850bf0SVladimir Sementsov-Ogievskiy     if (drv->bdrv_co_preadv_part) {
1141ac850bf0SVladimir Sementsov-Ogievskiy         return drv->bdrv_co_preadv_part(bs, offset, bytes, qiov, qiov_offset,
1142ac850bf0SVladimir Sementsov-Ogievskiy                                         flags);
1143ac850bf0SVladimir Sementsov-Ogievskiy     }
1144ac850bf0SVladimir Sementsov-Ogievskiy 
1145ac850bf0SVladimir Sementsov-Ogievskiy     if (qiov_offset > 0 || bytes != qiov->size) {
1146ac850bf0SVladimir Sementsov-Ogievskiy         qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1147ac850bf0SVladimir Sementsov-Ogievskiy         qiov = &local_qiov;
1148ac850bf0SVladimir Sementsov-Ogievskiy     }
1149ac850bf0SVladimir Sementsov-Ogievskiy 
11503fb06697SKevin Wolf     if (drv->bdrv_co_preadv) {
1151ac850bf0SVladimir Sementsov-Ogievskiy         ret = drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
1152ac850bf0SVladimir Sementsov-Ogievskiy         goto out;
11533fb06697SKevin Wolf     }
11543fb06697SKevin Wolf 
1155edfab6a0SEric Blake     if (drv->bdrv_aio_preadv) {
115608844473SKevin Wolf         BlockAIOCB *acb;
115708844473SKevin Wolf         CoroutineIOCompletion co = {
115808844473SKevin Wolf             .coroutine = qemu_coroutine_self(),
115908844473SKevin Wolf         };
116008844473SKevin Wolf 
1161e31f6864SEric Blake         acb = drv->bdrv_aio_preadv(bs, offset, bytes, qiov, flags,
116208844473SKevin Wolf                                    bdrv_co_io_em_complete, &co);
116308844473SKevin Wolf         if (acb == NULL) {
1164ac850bf0SVladimir Sementsov-Ogievskiy             ret = -EIO;
1165ac850bf0SVladimir Sementsov-Ogievskiy             goto out;
116608844473SKevin Wolf         } else {
116708844473SKevin Wolf             qemu_coroutine_yield();
1168ac850bf0SVladimir Sementsov-Ogievskiy             ret = co.ret;
1169ac850bf0SVladimir Sementsov-Ogievskiy             goto out;
117008844473SKevin Wolf         }
117108844473SKevin Wolf     }
1172edfab6a0SEric Blake 
1173edfab6a0SEric Blake     sector_num = offset >> BDRV_SECTOR_BITS;
1174edfab6a0SEric Blake     nb_sectors = bytes >> BDRV_SECTOR_BITS;
1175edfab6a0SEric Blake 
11761bbbf32dSNir Soffer     assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
11771bbbf32dSNir Soffer     assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
117841ae31e3SAlberto Garcia     assert(bytes <= BDRV_REQUEST_MAX_BYTES);
1179edfab6a0SEric Blake     assert(drv->bdrv_co_readv);
1180edfab6a0SEric Blake 
1181ac850bf0SVladimir Sementsov-Ogievskiy     ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
1182ac850bf0SVladimir Sementsov-Ogievskiy 
1183ac850bf0SVladimir Sementsov-Ogievskiy out:
1184ac850bf0SVladimir Sementsov-Ogievskiy     if (qiov == &local_qiov) {
1185ac850bf0SVladimir Sementsov-Ogievskiy         qemu_iovec_destroy(&local_qiov);
1186ac850bf0SVladimir Sementsov-Ogievskiy     }
1187ac850bf0SVladimir Sementsov-Ogievskiy 
1188ac850bf0SVladimir Sementsov-Ogievskiy     return ret;
1189166fe960SKevin Wolf }
1190166fe960SKevin Wolf 
119178a07294SKevin Wolf static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
119217abcbeeSVladimir Sementsov-Ogievskiy                                             int64_t offset, int64_t bytes,
1193ac850bf0SVladimir Sementsov-Ogievskiy                                             QEMUIOVector *qiov,
1194e75abedaSVladimir Sementsov-Ogievskiy                                             size_t qiov_offset,
1195e75abedaSVladimir Sementsov-Ogievskiy                                             BdrvRequestFlags flags)
119678a07294SKevin Wolf {
119778a07294SKevin Wolf     BlockDriver *drv = bs->drv;
11983fb06697SKevin Wolf     int64_t sector_num;
11993fb06697SKevin Wolf     unsigned int nb_sectors;
1200ac850bf0SVladimir Sementsov-Ogievskiy     QEMUIOVector local_qiov;
120178a07294SKevin Wolf     int ret;
120278a07294SKevin Wolf 
120317abcbeeSVladimir Sementsov-Ogievskiy     bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1204fa166538SEric Blake     assert(!(flags & ~BDRV_REQ_MASK));
1205fe0480d6SKevin Wolf     assert(!(flags & BDRV_REQ_NO_FALLBACK));
1206fa166538SEric Blake 
1207d470ad42SMax Reitz     if (!drv) {
1208d470ad42SMax Reitz         return -ENOMEDIUM;
1209d470ad42SMax Reitz     }
1210d470ad42SMax Reitz 
1211ac850bf0SVladimir Sementsov-Ogievskiy     if (drv->bdrv_co_pwritev_part) {
1212ac850bf0SVladimir Sementsov-Ogievskiy         ret = drv->bdrv_co_pwritev_part(bs, offset, bytes, qiov, qiov_offset,
1213ac850bf0SVladimir Sementsov-Ogievskiy                                         flags & bs->supported_write_flags);
1214ac850bf0SVladimir Sementsov-Ogievskiy         flags &= ~bs->supported_write_flags;
1215ac850bf0SVladimir Sementsov-Ogievskiy         goto emulate_flags;
1216ac850bf0SVladimir Sementsov-Ogievskiy     }
1217ac850bf0SVladimir Sementsov-Ogievskiy 
1218ac850bf0SVladimir Sementsov-Ogievskiy     if (qiov_offset > 0 || bytes != qiov->size) {
1219ac850bf0SVladimir Sementsov-Ogievskiy         qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1220ac850bf0SVladimir Sementsov-Ogievskiy         qiov = &local_qiov;
1221ac850bf0SVladimir Sementsov-Ogievskiy     }
1222ac850bf0SVladimir Sementsov-Ogievskiy 
12233fb06697SKevin Wolf     if (drv->bdrv_co_pwritev) {
1224515c2f43SKevin Wolf         ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov,
1225515c2f43SKevin Wolf                                    flags & bs->supported_write_flags);
1226515c2f43SKevin Wolf         flags &= ~bs->supported_write_flags;
12273fb06697SKevin Wolf         goto emulate_flags;
12283fb06697SKevin Wolf     }
12293fb06697SKevin Wolf 
1230edfab6a0SEric Blake     if (drv->bdrv_aio_pwritev) {
123108844473SKevin Wolf         BlockAIOCB *acb;
123208844473SKevin Wolf         CoroutineIOCompletion co = {
123308844473SKevin Wolf             .coroutine = qemu_coroutine_self(),
123408844473SKevin Wolf         };
123508844473SKevin Wolf 
1236e31f6864SEric Blake         acb = drv->bdrv_aio_pwritev(bs, offset, bytes, qiov,
1237e31f6864SEric Blake                                     flags & bs->supported_write_flags,
123808844473SKevin Wolf                                     bdrv_co_io_em_complete, &co);
1239e31f6864SEric Blake         flags &= ~bs->supported_write_flags;
124008844473SKevin Wolf         if (acb == NULL) {
12413fb06697SKevin Wolf             ret = -EIO;
124208844473SKevin Wolf         } else {
124308844473SKevin Wolf             qemu_coroutine_yield();
12443fb06697SKevin Wolf             ret = co.ret;
124508844473SKevin Wolf         }
1246edfab6a0SEric Blake         goto emulate_flags;
1247edfab6a0SEric Blake     }
1248edfab6a0SEric Blake 
1249edfab6a0SEric Blake     sector_num = offset >> BDRV_SECTOR_BITS;
1250edfab6a0SEric Blake     nb_sectors = bytes >> BDRV_SECTOR_BITS;
1251edfab6a0SEric Blake 
12521bbbf32dSNir Soffer     assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
12531bbbf32dSNir Soffer     assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
125441ae31e3SAlberto Garcia     assert(bytes <= BDRV_REQUEST_MAX_BYTES);
1255edfab6a0SEric Blake 
1256e18a58b4SEric Blake     assert(drv->bdrv_co_writev);
1257e18a58b4SEric Blake     ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov,
1258edfab6a0SEric Blake                               flags & bs->supported_write_flags);
1259edfab6a0SEric Blake     flags &= ~bs->supported_write_flags;
126078a07294SKevin Wolf 
12613fb06697SKevin Wolf emulate_flags:
12624df863f3SEric Blake     if (ret == 0 && (flags & BDRV_REQ_FUA)) {
126378a07294SKevin Wolf         ret = bdrv_co_flush(bs);
126478a07294SKevin Wolf     }
126578a07294SKevin Wolf 
1266ac850bf0SVladimir Sementsov-Ogievskiy     if (qiov == &local_qiov) {
1267ac850bf0SVladimir Sementsov-Ogievskiy         qemu_iovec_destroy(&local_qiov);
1268ac850bf0SVladimir Sementsov-Ogievskiy     }
1269ac850bf0SVladimir Sementsov-Ogievskiy 
127078a07294SKevin Wolf     return ret;
127178a07294SKevin Wolf }
127278a07294SKevin Wolf 
127329a298afSPavel Butsykin static int coroutine_fn
127417abcbeeSVladimir Sementsov-Ogievskiy bdrv_driver_pwritev_compressed(BlockDriverState *bs, int64_t offset,
127517abcbeeSVladimir Sementsov-Ogievskiy                                int64_t bytes, QEMUIOVector *qiov,
1276ac850bf0SVladimir Sementsov-Ogievskiy                                size_t qiov_offset)
127729a298afSPavel Butsykin {
127829a298afSPavel Butsykin     BlockDriver *drv = bs->drv;
1279ac850bf0SVladimir Sementsov-Ogievskiy     QEMUIOVector local_qiov;
1280ac850bf0SVladimir Sementsov-Ogievskiy     int ret;
128129a298afSPavel Butsykin 
128217abcbeeSVladimir Sementsov-Ogievskiy     bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
128317abcbeeSVladimir Sementsov-Ogievskiy 
1284d470ad42SMax Reitz     if (!drv) {
1285d470ad42SMax Reitz         return -ENOMEDIUM;
1286d470ad42SMax Reitz     }
1287d470ad42SMax Reitz 
1288ac850bf0SVladimir Sementsov-Ogievskiy     if (!block_driver_can_compress(drv)) {
128929a298afSPavel Butsykin         return -ENOTSUP;
129029a298afSPavel Butsykin     }
129129a298afSPavel Butsykin 
1292ac850bf0SVladimir Sementsov-Ogievskiy     if (drv->bdrv_co_pwritev_compressed_part) {
1293ac850bf0SVladimir Sementsov-Ogievskiy         return drv->bdrv_co_pwritev_compressed_part(bs, offset, bytes,
1294ac850bf0SVladimir Sementsov-Ogievskiy                                                     qiov, qiov_offset);
1295ac850bf0SVladimir Sementsov-Ogievskiy     }
1296ac850bf0SVladimir Sementsov-Ogievskiy 
1297ac850bf0SVladimir Sementsov-Ogievskiy     if (qiov_offset == 0) {
129829a298afSPavel Butsykin         return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov);
129929a298afSPavel Butsykin     }
130029a298afSPavel Butsykin 
1301ac850bf0SVladimir Sementsov-Ogievskiy     qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1302ac850bf0SVladimir Sementsov-Ogievskiy     ret = drv->bdrv_co_pwritev_compressed(bs, offset, bytes, &local_qiov);
1303ac850bf0SVladimir Sementsov-Ogievskiy     qemu_iovec_destroy(&local_qiov);
1304ac850bf0SVladimir Sementsov-Ogievskiy 
1305ac850bf0SVladimir Sementsov-Ogievskiy     return ret;
1306ac850bf0SVladimir Sementsov-Ogievskiy }
1307ac850bf0SVladimir Sementsov-Ogievskiy 
130885c97ca7SKevin Wolf static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child,
13099df5afbdSVladimir Sementsov-Ogievskiy         int64_t offset, int64_t bytes, QEMUIOVector *qiov,
13101143ec5eSVladimir Sementsov-Ogievskiy         size_t qiov_offset, int flags)
131161007b31SStefan Hajnoczi {
131285c97ca7SKevin Wolf     BlockDriverState *bs = child->bs;
131385c97ca7SKevin Wolf 
131461007b31SStefan Hajnoczi     /* Perform I/O through a temporary buffer so that users who scribble over
131561007b31SStefan Hajnoczi      * their read buffer while the operation is in progress do not end up
131661007b31SStefan Hajnoczi      * modifying the image file.  This is critical for zero-copy guest I/O
131761007b31SStefan Hajnoczi      * where anything might happen inside guest memory.
131861007b31SStefan Hajnoczi      */
13192275cc90SVladimir Sementsov-Ogievskiy     void *bounce_buffer = NULL;
132061007b31SStefan Hajnoczi 
132161007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
1322244483e6SKevin Wolf     int64_t cluster_offset;
13237cfd5275SEric Blake     int64_t cluster_bytes;
13249df5afbdSVladimir Sementsov-Ogievskiy     int64_t skip_bytes;
132561007b31SStefan Hajnoczi     int ret;
1326cb2e2878SEric Blake     int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer,
1327cb2e2878SEric Blake                                     BDRV_REQUEST_MAX_BYTES);
13289df5afbdSVladimir Sementsov-Ogievskiy     int64_t progress = 0;
13298644476eSMax Reitz     bool skip_write;
133061007b31SStefan Hajnoczi 
13319df5afbdSVladimir Sementsov-Ogievskiy     bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
13329df5afbdSVladimir Sementsov-Ogievskiy 
1333d470ad42SMax Reitz     if (!drv) {
1334d470ad42SMax Reitz         return -ENOMEDIUM;
1335d470ad42SMax Reitz     }
1336d470ad42SMax Reitz 
13378644476eSMax Reitz     /*
13388644476eSMax Reitz      * Do not write anything when the BDS is inactive.  That is not
13398644476eSMax Reitz      * allowed, and it would not help.
13408644476eSMax Reitz      */
13418644476eSMax Reitz     skip_write = (bs->open_flags & BDRV_O_INACTIVE);
13428644476eSMax Reitz 
13431bf03e66SKevin Wolf     /* FIXME We cannot require callers to have write permissions when all they
13441bf03e66SKevin Wolf      * are doing is a read request. If we did things right, write permissions
13451bf03e66SKevin Wolf      * would be obtained anyway, but internally by the copy-on-read code. As
1346765d9df9SEric Blake      * long as it is implemented here rather than in a separate filter driver,
13471bf03e66SKevin Wolf      * the copy-on-read code doesn't have its own BdrvChild, however, for which
13481bf03e66SKevin Wolf      * it could request permissions. Therefore we have to bypass the permission
13491bf03e66SKevin Wolf      * system for the moment. */
13501bf03e66SKevin Wolf     // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
1351afa4b293SKevin Wolf 
135261007b31SStefan Hajnoczi     /* Cover entire cluster so no additional backing file I/O is required when
1353cb2e2878SEric Blake      * allocating cluster in the image file.  Note that this value may exceed
1354cb2e2878SEric Blake      * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which
1355cb2e2878SEric Blake      * is one reason we loop rather than doing it all at once.
135661007b31SStefan Hajnoczi      */
1357244483e6SKevin Wolf     bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes);
1358cb2e2878SEric Blake     skip_bytes = offset - cluster_offset;
135961007b31SStefan Hajnoczi 
1360244483e6SKevin Wolf     trace_bdrv_co_do_copy_on_readv(bs, offset, bytes,
1361244483e6SKevin Wolf                                    cluster_offset, cluster_bytes);
136261007b31SStefan Hajnoczi 
1363cb2e2878SEric Blake     while (cluster_bytes) {
1364cb2e2878SEric Blake         int64_t pnum;
136561007b31SStefan Hajnoczi 
13668644476eSMax Reitz         if (skip_write) {
13678644476eSMax Reitz             ret = 1; /* "already allocated", so nothing will be copied */
13688644476eSMax Reitz             pnum = MIN(cluster_bytes, max_transfer);
13698644476eSMax Reitz         } else {
1370cb2e2878SEric Blake             ret = bdrv_is_allocated(bs, cluster_offset,
1371cb2e2878SEric Blake                                     MIN(cluster_bytes, max_transfer), &pnum);
1372cb2e2878SEric Blake             if (ret < 0) {
13738644476eSMax Reitz                 /*
13748644476eSMax Reitz                  * Safe to treat errors in querying allocation as if
1375cb2e2878SEric Blake                  * unallocated; we'll probably fail again soon on the
1376cb2e2878SEric Blake                  * read, but at least that will set a decent errno.
1377cb2e2878SEric Blake                  */
1378cb2e2878SEric Blake                 pnum = MIN(cluster_bytes, max_transfer);
1379cb2e2878SEric Blake             }
1380cb2e2878SEric Blake 
1381b0ddcbbbSKevin Wolf             /* Stop at EOF if the image ends in the middle of the cluster */
1382b0ddcbbbSKevin Wolf             if (ret == 0 && pnum == 0) {
1383b0ddcbbbSKevin Wolf                 assert(progress >= bytes);
1384b0ddcbbbSKevin Wolf                 break;
1385b0ddcbbbSKevin Wolf             }
1386b0ddcbbbSKevin Wolf 
1387cb2e2878SEric Blake             assert(skip_bytes < pnum);
13888644476eSMax Reitz         }
1389cb2e2878SEric Blake 
1390cb2e2878SEric Blake         if (ret <= 0) {
13911143ec5eSVladimir Sementsov-Ogievskiy             QEMUIOVector local_qiov;
13921143ec5eSVladimir Sementsov-Ogievskiy 
1393cb2e2878SEric Blake             /* Must copy-on-read; use the bounce buffer */
13940d93ed08SVladimir Sementsov-Ogievskiy             pnum = MIN(pnum, MAX_BOUNCE_BUFFER);
13952275cc90SVladimir Sementsov-Ogievskiy             if (!bounce_buffer) {
13962275cc90SVladimir Sementsov-Ogievskiy                 int64_t max_we_need = MAX(pnum, cluster_bytes - pnum);
13972275cc90SVladimir Sementsov-Ogievskiy                 int64_t max_allowed = MIN(max_transfer, MAX_BOUNCE_BUFFER);
13982275cc90SVladimir Sementsov-Ogievskiy                 int64_t bounce_buffer_len = MIN(max_we_need, max_allowed);
13992275cc90SVladimir Sementsov-Ogievskiy 
14002275cc90SVladimir Sementsov-Ogievskiy                 bounce_buffer = qemu_try_blockalign(bs, bounce_buffer_len);
14012275cc90SVladimir Sementsov-Ogievskiy                 if (!bounce_buffer) {
14022275cc90SVladimir Sementsov-Ogievskiy                     ret = -ENOMEM;
14032275cc90SVladimir Sementsov-Ogievskiy                     goto err;
14042275cc90SVladimir Sementsov-Ogievskiy                 }
14052275cc90SVladimir Sementsov-Ogievskiy             }
14060d93ed08SVladimir Sementsov-Ogievskiy             qemu_iovec_init_buf(&local_qiov, bounce_buffer, pnum);
1407cb2e2878SEric Blake 
1408cb2e2878SEric Blake             ret = bdrv_driver_preadv(bs, cluster_offset, pnum,
1409ac850bf0SVladimir Sementsov-Ogievskiy                                      &local_qiov, 0, 0);
141061007b31SStefan Hajnoczi             if (ret < 0) {
141161007b31SStefan Hajnoczi                 goto err;
141261007b31SStefan Hajnoczi             }
141361007b31SStefan Hajnoczi 
1414d855ebcdSEric Blake             bdrv_debug_event(bs, BLKDBG_COR_WRITE);
1415c1499a5eSEric Blake             if (drv->bdrv_co_pwrite_zeroes &&
1416cb2e2878SEric Blake                 buffer_is_zero(bounce_buffer, pnum)) {
1417a604fa2bSEric Blake                 /* FIXME: Should we (perhaps conditionally) be setting
1418a604fa2bSEric Blake                  * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy
1419a604fa2bSEric Blake                  * that still correctly reads as zero? */
14207adcf59fSMax Reitz                 ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, pnum,
14217adcf59fSMax Reitz                                                BDRV_REQ_WRITE_UNCHANGED);
142261007b31SStefan Hajnoczi             } else {
1423cb2e2878SEric Blake                 /* This does not change the data on the disk, it is not
1424cb2e2878SEric Blake                  * necessary to flush even in cache=writethrough mode.
142561007b31SStefan Hajnoczi                  */
1426cb2e2878SEric Blake                 ret = bdrv_driver_pwritev(bs, cluster_offset, pnum,
1427ac850bf0SVladimir Sementsov-Ogievskiy                                           &local_qiov, 0,
14287adcf59fSMax Reitz                                           BDRV_REQ_WRITE_UNCHANGED);
142961007b31SStefan Hajnoczi             }
143061007b31SStefan Hajnoczi 
143161007b31SStefan Hajnoczi             if (ret < 0) {
1432cb2e2878SEric Blake                 /* It might be okay to ignore write errors for guest
1433cb2e2878SEric Blake                  * requests.  If this is a deliberate copy-on-read
1434cb2e2878SEric Blake                  * then we don't want to ignore the error.  Simply
1435cb2e2878SEric Blake                  * report it in all cases.
143661007b31SStefan Hajnoczi                  */
143761007b31SStefan Hajnoczi                 goto err;
143861007b31SStefan Hajnoczi             }
143961007b31SStefan Hajnoczi 
14403299e5ecSVladimir Sementsov-Ogievskiy             if (!(flags & BDRV_REQ_PREFETCH)) {
14411143ec5eSVladimir Sementsov-Ogievskiy                 qemu_iovec_from_buf(qiov, qiov_offset + progress,
14421143ec5eSVladimir Sementsov-Ogievskiy                                     bounce_buffer + skip_bytes,
14434ab78b19SVladimir Sementsov-Ogievskiy                                     MIN(pnum - skip_bytes, bytes - progress));
14443299e5ecSVladimir Sementsov-Ogievskiy             }
14453299e5ecSVladimir Sementsov-Ogievskiy         } else if (!(flags & BDRV_REQ_PREFETCH)) {
1446cb2e2878SEric Blake             /* Read directly into the destination */
14471143ec5eSVladimir Sementsov-Ogievskiy             ret = bdrv_driver_preadv(bs, offset + progress,
14481143ec5eSVladimir Sementsov-Ogievskiy                                      MIN(pnum - skip_bytes, bytes - progress),
14491143ec5eSVladimir Sementsov-Ogievskiy                                      qiov, qiov_offset + progress, 0);
1450cb2e2878SEric Blake             if (ret < 0) {
1451cb2e2878SEric Blake                 goto err;
1452cb2e2878SEric Blake             }
1453cb2e2878SEric Blake         }
1454cb2e2878SEric Blake 
1455cb2e2878SEric Blake         cluster_offset += pnum;
1456cb2e2878SEric Blake         cluster_bytes -= pnum;
1457cb2e2878SEric Blake         progress += pnum - skip_bytes;
1458cb2e2878SEric Blake         skip_bytes = 0;
1459cb2e2878SEric Blake     }
1460cb2e2878SEric Blake     ret = 0;
146161007b31SStefan Hajnoczi 
146261007b31SStefan Hajnoczi err:
146361007b31SStefan Hajnoczi     qemu_vfree(bounce_buffer);
146461007b31SStefan Hajnoczi     return ret;
146561007b31SStefan Hajnoczi }
146661007b31SStefan Hajnoczi 
146761007b31SStefan Hajnoczi /*
146861007b31SStefan Hajnoczi  * Forwards an already correctly aligned request to the BlockDriver. This
14691a62d0acSEric Blake  * handles copy on read, zeroing after EOF, and fragmentation of large
14701a62d0acSEric Blake  * reads; any other features must be implemented by the caller.
147161007b31SStefan Hajnoczi  */
147285c97ca7SKevin Wolf static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,
14738b0c5d76SVladimir Sementsov-Ogievskiy     BdrvTrackedRequest *req, int64_t offset, int64_t bytes,
147465cd4424SVladimir Sementsov-Ogievskiy     int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags)
147561007b31SStefan Hajnoczi {
147685c97ca7SKevin Wolf     BlockDriverState *bs = child->bs;
1477c9d20029SKevin Wolf     int64_t total_bytes, max_bytes;
14781a62d0acSEric Blake     int ret = 0;
14798b0c5d76SVladimir Sementsov-Ogievskiy     int64_t bytes_remaining = bytes;
14801a62d0acSEric Blake     int max_transfer;
148161007b31SStefan Hajnoczi 
14828b0c5d76SVladimir Sementsov-Ogievskiy     bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
148349c07526SKevin Wolf     assert(is_power_of_2(align));
148449c07526SKevin Wolf     assert((offset & (align - 1)) == 0);
148549c07526SKevin Wolf     assert((bytes & (align - 1)) == 0);
1486abb06c5aSDaniel P. Berrange     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
14871a62d0acSEric Blake     max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
14881a62d0acSEric Blake                                    align);
1489a604fa2bSEric Blake 
1490a604fa2bSEric Blake     /* TODO: We would need a per-BDS .supported_read_flags and
1491a604fa2bSEric Blake      * potential fallback support, if we ever implement any read flags
1492a604fa2bSEric Blake      * to pass through to drivers.  For now, there aren't any
1493a604fa2bSEric Blake      * passthrough flags.  */
1494c53cb427SPaolo Bonzini     assert(!(flags & ~(BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH)));
149561007b31SStefan Hajnoczi 
149661007b31SStefan Hajnoczi     /* Handle Copy on Read and associated serialisation */
149761007b31SStefan Hajnoczi     if (flags & BDRV_REQ_COPY_ON_READ) {
149861007b31SStefan Hajnoczi         /* If we touch the same cluster it counts as an overlap.  This
149961007b31SStefan Hajnoczi          * guarantees that allocating writes will be serialized and not race
150061007b31SStefan Hajnoczi          * with each other for the same cluster.  For example, in copy-on-read
150161007b31SStefan Hajnoczi          * it ensures that the CoR read and write operations are atomic and
150261007b31SStefan Hajnoczi          * guest writes cannot interleave between them. */
15038ac5aab2SVladimir Sementsov-Ogievskiy         bdrv_make_request_serialising(req, bdrv_get_cluster_size(bs));
150418fbd0deSPaolo Bonzini     } else {
1505304d9d7fSMax Reitz         bdrv_wait_serialising_requests(req);
150618fbd0deSPaolo Bonzini     }
150761007b31SStefan Hajnoczi 
150861007b31SStefan Hajnoczi     if (flags & BDRV_REQ_COPY_ON_READ) {
1509d6a644bbSEric Blake         int64_t pnum;
151061007b31SStefan Hajnoczi 
1511897dd0ecSAndrey Shinkevich         /* The flag BDRV_REQ_COPY_ON_READ has reached its addressee */
1512897dd0ecSAndrey Shinkevich         flags &= ~BDRV_REQ_COPY_ON_READ;
1513897dd0ecSAndrey Shinkevich 
151488e63df2SEric Blake         ret = bdrv_is_allocated(bs, offset, bytes, &pnum);
151561007b31SStefan Hajnoczi         if (ret < 0) {
151661007b31SStefan Hajnoczi             goto out;
151761007b31SStefan Hajnoczi         }
151861007b31SStefan Hajnoczi 
151988e63df2SEric Blake         if (!ret || pnum != bytes) {
152065cd4424SVladimir Sementsov-Ogievskiy             ret = bdrv_co_do_copy_on_readv(child, offset, bytes,
152165cd4424SVladimir Sementsov-Ogievskiy                                            qiov, qiov_offset, flags);
15223299e5ecSVladimir Sementsov-Ogievskiy             goto out;
15233299e5ecSVladimir Sementsov-Ogievskiy         } else if (flags & BDRV_REQ_PREFETCH) {
152461007b31SStefan Hajnoczi             goto out;
152561007b31SStefan Hajnoczi         }
152661007b31SStefan Hajnoczi     }
152761007b31SStefan Hajnoczi 
15281a62d0acSEric Blake     /* Forward the request to the BlockDriver, possibly fragmenting it */
152949c07526SKevin Wolf     total_bytes = bdrv_getlength(bs);
153049c07526SKevin Wolf     if (total_bytes < 0) {
153149c07526SKevin Wolf         ret = total_bytes;
153261007b31SStefan Hajnoczi         goto out;
153361007b31SStefan Hajnoczi     }
153461007b31SStefan Hajnoczi 
1535897dd0ecSAndrey Shinkevich     assert(!(flags & ~bs->supported_read_flags));
1536897dd0ecSAndrey Shinkevich 
153749c07526SKevin Wolf     max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align);
15381a62d0acSEric Blake     if (bytes <= max_bytes && bytes <= max_transfer) {
1539897dd0ecSAndrey Shinkevich         ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, flags);
15401a62d0acSEric Blake         goto out;
154161007b31SStefan Hajnoczi     }
154261007b31SStefan Hajnoczi 
15431a62d0acSEric Blake     while (bytes_remaining) {
15448b0c5d76SVladimir Sementsov-Ogievskiy         int64_t num;
15451a62d0acSEric Blake 
15461a62d0acSEric Blake         if (max_bytes) {
15471a62d0acSEric Blake             num = MIN(bytes_remaining, MIN(max_bytes, max_transfer));
15481a62d0acSEric Blake             assert(num);
15491a62d0acSEric Blake 
15501a62d0acSEric Blake             ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining,
1551134b7decSMax Reitz                                      num, qiov,
1552897dd0ecSAndrey Shinkevich                                      qiov_offset + bytes - bytes_remaining,
1553897dd0ecSAndrey Shinkevich                                      flags);
15541a62d0acSEric Blake             max_bytes -= num;
15551a62d0acSEric Blake         } else {
15561a62d0acSEric Blake             num = bytes_remaining;
1557134b7decSMax Reitz             ret = qemu_iovec_memset(qiov, qiov_offset + bytes - bytes_remaining,
1558134b7decSMax Reitz                                     0, bytes_remaining);
15591a62d0acSEric Blake         }
15601a62d0acSEric Blake         if (ret < 0) {
15611a62d0acSEric Blake             goto out;
15621a62d0acSEric Blake         }
15631a62d0acSEric Blake         bytes_remaining -= num;
156461007b31SStefan Hajnoczi     }
156561007b31SStefan Hajnoczi 
156661007b31SStefan Hajnoczi out:
15671a62d0acSEric Blake     return ret < 0 ? ret : 0;
156861007b31SStefan Hajnoczi }
156961007b31SStefan Hajnoczi 
157061007b31SStefan Hajnoczi /*
15717a3f542fSVladimir Sementsov-Ogievskiy  * Request padding
15727a3f542fSVladimir Sementsov-Ogievskiy  *
15737a3f542fSVladimir Sementsov-Ogievskiy  *  |<---- align ----->|                     |<----- align ---->|
15747a3f542fSVladimir Sementsov-Ogievskiy  *  |<- head ->|<------------- bytes ------------->|<-- tail -->|
15757a3f542fSVladimir Sementsov-Ogievskiy  *  |          |       |                     |     |            |
15767a3f542fSVladimir Sementsov-Ogievskiy  * -*----------$-------*-------- ... --------*-----$------------*---
15777a3f542fSVladimir Sementsov-Ogievskiy  *  |          |       |                     |     |            |
15787a3f542fSVladimir Sementsov-Ogievskiy  *  |          offset  |                     |     end          |
15797a3f542fSVladimir Sementsov-Ogievskiy  *  ALIGN_DOWN(offset) ALIGN_UP(offset)      ALIGN_DOWN(end)   ALIGN_UP(end)
15807a3f542fSVladimir Sementsov-Ogievskiy  *  [buf   ... )                             [tail_buf          )
15817a3f542fSVladimir Sementsov-Ogievskiy  *
15827a3f542fSVladimir Sementsov-Ogievskiy  * @buf is an aligned allocation needed to store @head and @tail paddings. @head
15837a3f542fSVladimir Sementsov-Ogievskiy  * is placed at the beginning of @buf and @tail at the @end.
15847a3f542fSVladimir Sementsov-Ogievskiy  *
15857a3f542fSVladimir Sementsov-Ogievskiy  * @tail_buf is a pointer to sub-buffer, corresponding to align-sized chunk
15867a3f542fSVladimir Sementsov-Ogievskiy  * around tail, if tail exists.
15877a3f542fSVladimir Sementsov-Ogievskiy  *
15887a3f542fSVladimir Sementsov-Ogievskiy  * @merge_reads is true for small requests,
15897a3f542fSVladimir Sementsov-Ogievskiy  * if @buf_len == @head + bytes + @tail. In this case it is possible that both
15907a3f542fSVladimir Sementsov-Ogievskiy  * head and tail exist but @buf_len == align and @tail_buf == @buf.
159161007b31SStefan Hajnoczi  */
15927a3f542fSVladimir Sementsov-Ogievskiy typedef struct BdrvRequestPadding {
15937a3f542fSVladimir Sementsov-Ogievskiy     uint8_t *buf;
15947a3f542fSVladimir Sementsov-Ogievskiy     size_t buf_len;
15957a3f542fSVladimir Sementsov-Ogievskiy     uint8_t *tail_buf;
15967a3f542fSVladimir Sementsov-Ogievskiy     size_t head;
15977a3f542fSVladimir Sementsov-Ogievskiy     size_t tail;
15987a3f542fSVladimir Sementsov-Ogievskiy     bool merge_reads;
15997a3f542fSVladimir Sementsov-Ogievskiy     QEMUIOVector local_qiov;
16007a3f542fSVladimir Sementsov-Ogievskiy } BdrvRequestPadding;
16017a3f542fSVladimir Sementsov-Ogievskiy 
16027a3f542fSVladimir Sementsov-Ogievskiy static bool bdrv_init_padding(BlockDriverState *bs,
16037a3f542fSVladimir Sementsov-Ogievskiy                               int64_t offset, int64_t bytes,
16047a3f542fSVladimir Sementsov-Ogievskiy                               BdrvRequestPadding *pad)
16057a3f542fSVladimir Sementsov-Ogievskiy {
1606a56ed80cSVladimir Sementsov-Ogievskiy     int64_t align = bs->bl.request_alignment;
1607a56ed80cSVladimir Sementsov-Ogievskiy     int64_t sum;
1608a56ed80cSVladimir Sementsov-Ogievskiy 
1609a56ed80cSVladimir Sementsov-Ogievskiy     bdrv_check_request(offset, bytes, &error_abort);
1610a56ed80cSVladimir Sementsov-Ogievskiy     assert(align <= INT_MAX); /* documented in block/block_int.h */
1611a56ed80cSVladimir Sementsov-Ogievskiy     assert(align <= SIZE_MAX / 2); /* so we can allocate the buffer */
16127a3f542fSVladimir Sementsov-Ogievskiy 
16137a3f542fSVladimir Sementsov-Ogievskiy     memset(pad, 0, sizeof(*pad));
16147a3f542fSVladimir Sementsov-Ogievskiy 
16157a3f542fSVladimir Sementsov-Ogievskiy     pad->head = offset & (align - 1);
16167a3f542fSVladimir Sementsov-Ogievskiy     pad->tail = ((offset + bytes) & (align - 1));
16177a3f542fSVladimir Sementsov-Ogievskiy     if (pad->tail) {
16187a3f542fSVladimir Sementsov-Ogievskiy         pad->tail = align - pad->tail;
16197a3f542fSVladimir Sementsov-Ogievskiy     }
16207a3f542fSVladimir Sementsov-Ogievskiy 
1621ac9d00bfSVladimir Sementsov-Ogievskiy     if (!pad->head && !pad->tail) {
16227a3f542fSVladimir Sementsov-Ogievskiy         return false;
16237a3f542fSVladimir Sementsov-Ogievskiy     }
16247a3f542fSVladimir Sementsov-Ogievskiy 
1625ac9d00bfSVladimir Sementsov-Ogievskiy     assert(bytes); /* Nothing good in aligning zero-length requests */
1626ac9d00bfSVladimir Sementsov-Ogievskiy 
16277a3f542fSVladimir Sementsov-Ogievskiy     sum = pad->head + bytes + pad->tail;
16287a3f542fSVladimir Sementsov-Ogievskiy     pad->buf_len = (sum > align && pad->head && pad->tail) ? 2 * align : align;
16297a3f542fSVladimir Sementsov-Ogievskiy     pad->buf = qemu_blockalign(bs, pad->buf_len);
16307a3f542fSVladimir Sementsov-Ogievskiy     pad->merge_reads = sum == pad->buf_len;
16317a3f542fSVladimir Sementsov-Ogievskiy     if (pad->tail) {
16327a3f542fSVladimir Sementsov-Ogievskiy         pad->tail_buf = pad->buf + pad->buf_len - align;
16337a3f542fSVladimir Sementsov-Ogievskiy     }
16347a3f542fSVladimir Sementsov-Ogievskiy 
16357a3f542fSVladimir Sementsov-Ogievskiy     return true;
16367a3f542fSVladimir Sementsov-Ogievskiy }
16377a3f542fSVladimir Sementsov-Ogievskiy 
1638881a4c55SPaolo Bonzini static coroutine_fn int bdrv_padding_rmw_read(BdrvChild *child,
16397a3f542fSVladimir Sementsov-Ogievskiy                                               BdrvTrackedRequest *req,
16407a3f542fSVladimir Sementsov-Ogievskiy                                               BdrvRequestPadding *pad,
16417a3f542fSVladimir Sementsov-Ogievskiy                                               bool zero_middle)
16427a3f542fSVladimir Sementsov-Ogievskiy {
16437a3f542fSVladimir Sementsov-Ogievskiy     QEMUIOVector local_qiov;
16447a3f542fSVladimir Sementsov-Ogievskiy     BlockDriverState *bs = child->bs;
16457a3f542fSVladimir Sementsov-Ogievskiy     uint64_t align = bs->bl.request_alignment;
16467a3f542fSVladimir Sementsov-Ogievskiy     int ret;
16477a3f542fSVladimir Sementsov-Ogievskiy 
16487a3f542fSVladimir Sementsov-Ogievskiy     assert(req->serialising && pad->buf);
16497a3f542fSVladimir Sementsov-Ogievskiy 
16507a3f542fSVladimir Sementsov-Ogievskiy     if (pad->head || pad->merge_reads) {
16518b0c5d76SVladimir Sementsov-Ogievskiy         int64_t bytes = pad->merge_reads ? pad->buf_len : align;
16527a3f542fSVladimir Sementsov-Ogievskiy 
16537a3f542fSVladimir Sementsov-Ogievskiy         qemu_iovec_init_buf(&local_qiov, pad->buf, bytes);
16547a3f542fSVladimir Sementsov-Ogievskiy 
16557a3f542fSVladimir Sementsov-Ogievskiy         if (pad->head) {
16567a3f542fSVladimir Sementsov-Ogievskiy             bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
16577a3f542fSVladimir Sementsov-Ogievskiy         }
16587a3f542fSVladimir Sementsov-Ogievskiy         if (pad->merge_reads && pad->tail) {
16597a3f542fSVladimir Sementsov-Ogievskiy             bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
16607a3f542fSVladimir Sementsov-Ogievskiy         }
16617a3f542fSVladimir Sementsov-Ogievskiy         ret = bdrv_aligned_preadv(child, req, req->overlap_offset, bytes,
166265cd4424SVladimir Sementsov-Ogievskiy                                   align, &local_qiov, 0, 0);
16637a3f542fSVladimir Sementsov-Ogievskiy         if (ret < 0) {
16647a3f542fSVladimir Sementsov-Ogievskiy             return ret;
16657a3f542fSVladimir Sementsov-Ogievskiy         }
16667a3f542fSVladimir Sementsov-Ogievskiy         if (pad->head) {
16677a3f542fSVladimir Sementsov-Ogievskiy             bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
16687a3f542fSVladimir Sementsov-Ogievskiy         }
16697a3f542fSVladimir Sementsov-Ogievskiy         if (pad->merge_reads && pad->tail) {
16707a3f542fSVladimir Sementsov-Ogievskiy             bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
16717a3f542fSVladimir Sementsov-Ogievskiy         }
16727a3f542fSVladimir Sementsov-Ogievskiy 
16737a3f542fSVladimir Sementsov-Ogievskiy         if (pad->merge_reads) {
16747a3f542fSVladimir Sementsov-Ogievskiy             goto zero_mem;
16757a3f542fSVladimir Sementsov-Ogievskiy         }
16767a3f542fSVladimir Sementsov-Ogievskiy     }
16777a3f542fSVladimir Sementsov-Ogievskiy 
16787a3f542fSVladimir Sementsov-Ogievskiy     if (pad->tail) {
16797a3f542fSVladimir Sementsov-Ogievskiy         qemu_iovec_init_buf(&local_qiov, pad->tail_buf, align);
16807a3f542fSVladimir Sementsov-Ogievskiy 
16817a3f542fSVladimir Sementsov-Ogievskiy         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
16827a3f542fSVladimir Sementsov-Ogievskiy         ret = bdrv_aligned_preadv(
16837a3f542fSVladimir Sementsov-Ogievskiy                 child, req,
16847a3f542fSVladimir Sementsov-Ogievskiy                 req->overlap_offset + req->overlap_bytes - align,
168565cd4424SVladimir Sementsov-Ogievskiy                 align, align, &local_qiov, 0, 0);
16867a3f542fSVladimir Sementsov-Ogievskiy         if (ret < 0) {
16877a3f542fSVladimir Sementsov-Ogievskiy             return ret;
16887a3f542fSVladimir Sementsov-Ogievskiy         }
16897a3f542fSVladimir Sementsov-Ogievskiy         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
16907a3f542fSVladimir Sementsov-Ogievskiy     }
16917a3f542fSVladimir Sementsov-Ogievskiy 
16927a3f542fSVladimir Sementsov-Ogievskiy zero_mem:
16937a3f542fSVladimir Sementsov-Ogievskiy     if (zero_middle) {
16947a3f542fSVladimir Sementsov-Ogievskiy         memset(pad->buf + pad->head, 0, pad->buf_len - pad->head - pad->tail);
16957a3f542fSVladimir Sementsov-Ogievskiy     }
16967a3f542fSVladimir Sementsov-Ogievskiy 
16977a3f542fSVladimir Sementsov-Ogievskiy     return 0;
16987a3f542fSVladimir Sementsov-Ogievskiy }
16997a3f542fSVladimir Sementsov-Ogievskiy 
17007a3f542fSVladimir Sementsov-Ogievskiy static void bdrv_padding_destroy(BdrvRequestPadding *pad)
17017a3f542fSVladimir Sementsov-Ogievskiy {
17027a3f542fSVladimir Sementsov-Ogievskiy     if (pad->buf) {
17037a3f542fSVladimir Sementsov-Ogievskiy         qemu_vfree(pad->buf);
17047a3f542fSVladimir Sementsov-Ogievskiy         qemu_iovec_destroy(&pad->local_qiov);
17057a3f542fSVladimir Sementsov-Ogievskiy     }
170698ca4549SVladimir Sementsov-Ogievskiy     memset(pad, 0, sizeof(*pad));
17077a3f542fSVladimir Sementsov-Ogievskiy }
17087a3f542fSVladimir Sementsov-Ogievskiy 
17097a3f542fSVladimir Sementsov-Ogievskiy /*
17107a3f542fSVladimir Sementsov-Ogievskiy  * bdrv_pad_request
17117a3f542fSVladimir Sementsov-Ogievskiy  *
17127a3f542fSVladimir Sementsov-Ogievskiy  * Exchange request parameters with padded request if needed. Don't include RMW
17137a3f542fSVladimir Sementsov-Ogievskiy  * read of padding, bdrv_padding_rmw_read() should be called separately if
17147a3f542fSVladimir Sementsov-Ogievskiy  * needed.
17157a3f542fSVladimir Sementsov-Ogievskiy  *
171698ca4549SVladimir Sementsov-Ogievskiy  * Request parameters (@qiov, &qiov_offset, &offset, &bytes) are in-out:
171798ca4549SVladimir Sementsov-Ogievskiy  *  - on function start they represent original request
171898ca4549SVladimir Sementsov-Ogievskiy  *  - on failure or when padding is not needed they are unchanged
171998ca4549SVladimir Sementsov-Ogievskiy  *  - on success when padding is needed they represent padded request
17207a3f542fSVladimir Sementsov-Ogievskiy  */
172198ca4549SVladimir Sementsov-Ogievskiy static int bdrv_pad_request(BlockDriverState *bs,
17221acc3466SVladimir Sementsov-Ogievskiy                             QEMUIOVector **qiov, size_t *qiov_offset,
172337e9403eSVladimir Sementsov-Ogievskiy                             int64_t *offset, int64_t *bytes,
172498ca4549SVladimir Sementsov-Ogievskiy                             BdrvRequestPadding *pad, bool *padded)
17257a3f542fSVladimir Sementsov-Ogievskiy {
17264c002cefSVladimir Sementsov-Ogievskiy     int ret;
17274c002cefSVladimir Sementsov-Ogievskiy 
172837e9403eSVladimir Sementsov-Ogievskiy     bdrv_check_qiov_request(*offset, *bytes, *qiov, *qiov_offset, &error_abort);
172937e9403eSVladimir Sementsov-Ogievskiy 
17307a3f542fSVladimir Sementsov-Ogievskiy     if (!bdrv_init_padding(bs, *offset, *bytes, pad)) {
173198ca4549SVladimir Sementsov-Ogievskiy         if (padded) {
173298ca4549SVladimir Sementsov-Ogievskiy             *padded = false;
173398ca4549SVladimir Sementsov-Ogievskiy         }
173498ca4549SVladimir Sementsov-Ogievskiy         return 0;
17357a3f542fSVladimir Sementsov-Ogievskiy     }
17367a3f542fSVladimir Sementsov-Ogievskiy 
17374c002cefSVladimir Sementsov-Ogievskiy     ret = qemu_iovec_init_extended(&pad->local_qiov, pad->buf, pad->head,
17381acc3466SVladimir Sementsov-Ogievskiy                                    *qiov, *qiov_offset, *bytes,
17394c002cefSVladimir Sementsov-Ogievskiy                                    pad->buf + pad->buf_len - pad->tail,
17404c002cefSVladimir Sementsov-Ogievskiy                                    pad->tail);
174198ca4549SVladimir Sementsov-Ogievskiy     if (ret < 0) {
174298ca4549SVladimir Sementsov-Ogievskiy         bdrv_padding_destroy(pad);
174398ca4549SVladimir Sementsov-Ogievskiy         return ret;
174498ca4549SVladimir Sementsov-Ogievskiy     }
17457a3f542fSVladimir Sementsov-Ogievskiy     *bytes += pad->head + pad->tail;
17467a3f542fSVladimir Sementsov-Ogievskiy     *offset -= pad->head;
17477a3f542fSVladimir Sementsov-Ogievskiy     *qiov = &pad->local_qiov;
17481acc3466SVladimir Sementsov-Ogievskiy     *qiov_offset = 0;
174998ca4549SVladimir Sementsov-Ogievskiy     if (padded) {
175098ca4549SVladimir Sementsov-Ogievskiy         *padded = true;
175198ca4549SVladimir Sementsov-Ogievskiy     }
17527a3f542fSVladimir Sementsov-Ogievskiy 
175398ca4549SVladimir Sementsov-Ogievskiy     return 0;
17547a3f542fSVladimir Sementsov-Ogievskiy }
17557a3f542fSVladimir Sementsov-Ogievskiy 
1756a03ef88fSKevin Wolf int coroutine_fn bdrv_co_preadv(BdrvChild *child,
1757e9e52efdSVladimir Sementsov-Ogievskiy     int64_t offset, int64_t bytes, QEMUIOVector *qiov,
175861007b31SStefan Hajnoczi     BdrvRequestFlags flags)
175961007b31SStefan Hajnoczi {
1760967d7905SEmanuele Giuseppe Esposito     IO_CODE();
17611acc3466SVladimir Sementsov-Ogievskiy     return bdrv_co_preadv_part(child, offset, bytes, qiov, 0, flags);
17621acc3466SVladimir Sementsov-Ogievskiy }
17631acc3466SVladimir Sementsov-Ogievskiy 
17641acc3466SVladimir Sementsov-Ogievskiy int coroutine_fn bdrv_co_preadv_part(BdrvChild *child,
176537e9403eSVladimir Sementsov-Ogievskiy     int64_t offset, int64_t bytes,
17661acc3466SVladimir Sementsov-Ogievskiy     QEMUIOVector *qiov, size_t qiov_offset,
17671acc3466SVladimir Sementsov-Ogievskiy     BdrvRequestFlags flags)
17681acc3466SVladimir Sementsov-Ogievskiy {
1769a03ef88fSKevin Wolf     BlockDriverState *bs = child->bs;
177061007b31SStefan Hajnoczi     BdrvTrackedRequest req;
17717a3f542fSVladimir Sementsov-Ogievskiy     BdrvRequestPadding pad;
177261007b31SStefan Hajnoczi     int ret;
1773967d7905SEmanuele Giuseppe Esposito     IO_CODE();
177461007b31SStefan Hajnoczi 
177537e9403eSVladimir Sementsov-Ogievskiy     trace_bdrv_co_preadv_part(bs, offset, bytes, flags);
177661007b31SStefan Hajnoczi 
1777f4dad307SVladimir Sementsov-Ogievskiy     if (!bdrv_is_inserted(bs)) {
1778f4dad307SVladimir Sementsov-Ogievskiy         return -ENOMEDIUM;
1779f4dad307SVladimir Sementsov-Ogievskiy     }
1780f4dad307SVladimir Sementsov-Ogievskiy 
178163f4ad11SVladimir Sementsov-Ogievskiy     ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset);
178261007b31SStefan Hajnoczi     if (ret < 0) {
178361007b31SStefan Hajnoczi         return ret;
178461007b31SStefan Hajnoczi     }
178561007b31SStefan Hajnoczi 
1786ac9d00bfSVladimir Sementsov-Ogievskiy     if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) {
1787ac9d00bfSVladimir Sementsov-Ogievskiy         /*
1788ac9d00bfSVladimir Sementsov-Ogievskiy          * Aligning zero request is nonsense. Even if driver has special meaning
1789ac9d00bfSVladimir Sementsov-Ogievskiy          * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass
1790ac9d00bfSVladimir Sementsov-Ogievskiy          * it to driver due to request_alignment.
1791ac9d00bfSVladimir Sementsov-Ogievskiy          *
1792ac9d00bfSVladimir Sementsov-Ogievskiy          * Still, no reason to return an error if someone do unaligned
1793ac9d00bfSVladimir Sementsov-Ogievskiy          * zero-length read occasionally.
1794ac9d00bfSVladimir Sementsov-Ogievskiy          */
1795ac9d00bfSVladimir Sementsov-Ogievskiy         return 0;
1796ac9d00bfSVladimir Sementsov-Ogievskiy     }
1797ac9d00bfSVladimir Sementsov-Ogievskiy 
179899723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
179999723548SPaolo Bonzini 
18009568b511SWen Congyang     /* Don't do copy-on-read if we read data before write operation */
1801d73415a3SStefan Hajnoczi     if (qatomic_read(&bs->copy_on_read)) {
180261007b31SStefan Hajnoczi         flags |= BDRV_REQ_COPY_ON_READ;
180361007b31SStefan Hajnoczi     }
180461007b31SStefan Hajnoczi 
180598ca4549SVladimir Sementsov-Ogievskiy     ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad,
180698ca4549SVladimir Sementsov-Ogievskiy                            NULL);
180798ca4549SVladimir Sementsov-Ogievskiy     if (ret < 0) {
180887ab8802SKevin Wolf         goto fail;
180998ca4549SVladimir Sementsov-Ogievskiy     }
181061007b31SStefan Hajnoczi 
1811ebde595cSFam Zheng     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
18127a3f542fSVladimir Sementsov-Ogievskiy     ret = bdrv_aligned_preadv(child, &req, offset, bytes,
18137a3f542fSVladimir Sementsov-Ogievskiy                               bs->bl.request_alignment,
18141acc3466SVladimir Sementsov-Ogievskiy                               qiov, qiov_offset, flags);
181561007b31SStefan Hajnoczi     tracked_request_end(&req);
18167a3f542fSVladimir Sementsov-Ogievskiy     bdrv_padding_destroy(&pad);
181761007b31SStefan Hajnoczi 
181887ab8802SKevin Wolf fail:
181987ab8802SKevin Wolf     bdrv_dec_in_flight(bs);
182087ab8802SKevin Wolf 
182161007b31SStefan Hajnoczi     return ret;
182261007b31SStefan Hajnoczi }
182361007b31SStefan Hajnoczi 
1824d05aa8bbSEric Blake static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
18255ae07b14SVladimir Sementsov-Ogievskiy     int64_t offset, int64_t bytes, BdrvRequestFlags flags)
182661007b31SStefan Hajnoczi {
182761007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
182861007b31SStefan Hajnoczi     QEMUIOVector qiov;
18290d93ed08SVladimir Sementsov-Ogievskiy     void *buf = NULL;
183061007b31SStefan Hajnoczi     int ret = 0;
1831465fe887SEric Blake     bool need_flush = false;
1832443668caSDenis V. Lunev     int head = 0;
1833443668caSDenis V. Lunev     int tail = 0;
183461007b31SStefan Hajnoczi 
18352aaa3f9bSVladimir Sementsov-Ogievskiy     int64_t max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes,
18362aaa3f9bSVladimir Sementsov-Ogievskiy                                             INT64_MAX);
1837a5b8dd2cSEric Blake     int alignment = MAX(bs->bl.pwrite_zeroes_alignment,
1838a5b8dd2cSEric Blake                         bs->bl.request_alignment);
1839cb2e2878SEric Blake     int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER);
1840cf081fcaSEric Blake 
18415ae07b14SVladimir Sementsov-Ogievskiy     bdrv_check_request(offset, bytes, &error_abort);
18425ae07b14SVladimir Sementsov-Ogievskiy 
1843d470ad42SMax Reitz     if (!drv) {
1844d470ad42SMax Reitz         return -ENOMEDIUM;
1845d470ad42SMax Reitz     }
1846d470ad42SMax Reitz 
1847fe0480d6SKevin Wolf     if ((flags & ~bs->supported_zero_flags) & BDRV_REQ_NO_FALLBACK) {
1848fe0480d6SKevin Wolf         return -ENOTSUP;
1849fe0480d6SKevin Wolf     }
1850fe0480d6SKevin Wolf 
18510bc329fbSHanna Reitz     /* Invalidate the cached block-status data range if this write overlaps */
18520bc329fbSHanna Reitz     bdrv_bsc_invalidate_range(bs, offset, bytes);
18530bc329fbSHanna Reitz 
1854b8d0a980SEric Blake     assert(alignment % bs->bl.request_alignment == 0);
1855b8d0a980SEric Blake     head = offset % alignment;
1856f5a5ca79SManos Pitsidianakis     tail = (offset + bytes) % alignment;
1857b8d0a980SEric Blake     max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment);
1858b8d0a980SEric Blake     assert(max_write_zeroes >= bs->bl.request_alignment);
185961007b31SStefan Hajnoczi 
1860f5a5ca79SManos Pitsidianakis     while (bytes > 0 && !ret) {
18615ae07b14SVladimir Sementsov-Ogievskiy         int64_t num = bytes;
186261007b31SStefan Hajnoczi 
186361007b31SStefan Hajnoczi         /* Align request.  Block drivers can expect the "bulk" of the request
1864443668caSDenis V. Lunev          * to be aligned, and that unaligned requests do not cross cluster
1865443668caSDenis V. Lunev          * boundaries.
186661007b31SStefan Hajnoczi          */
1867443668caSDenis V. Lunev         if (head) {
1868b2f95feeSEric Blake             /* Make a small request up to the first aligned sector. For
1869b2f95feeSEric Blake              * convenience, limit this request to max_transfer even if
1870b2f95feeSEric Blake              * we don't need to fall back to writes.  */
1871f5a5ca79SManos Pitsidianakis             num = MIN(MIN(bytes, max_transfer), alignment - head);
1872b2f95feeSEric Blake             head = (head + num) % alignment;
1873b2f95feeSEric Blake             assert(num < max_write_zeroes);
1874d05aa8bbSEric Blake         } else if (tail && num > alignment) {
1875443668caSDenis V. Lunev             /* Shorten the request to the last aligned sector.  */
1876443668caSDenis V. Lunev             num -= tail;
187761007b31SStefan Hajnoczi         }
187861007b31SStefan Hajnoczi 
187961007b31SStefan Hajnoczi         /* limit request size */
188061007b31SStefan Hajnoczi         if (num > max_write_zeroes) {
188161007b31SStefan Hajnoczi             num = max_write_zeroes;
188261007b31SStefan Hajnoczi         }
188361007b31SStefan Hajnoczi 
188461007b31SStefan Hajnoczi         ret = -ENOTSUP;
188561007b31SStefan Hajnoczi         /* First try the efficient write zeroes operation */
1886d05aa8bbSEric Blake         if (drv->bdrv_co_pwrite_zeroes) {
1887d05aa8bbSEric Blake             ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num,
1888d05aa8bbSEric Blake                                              flags & bs->supported_zero_flags);
1889d05aa8bbSEric Blake             if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) &&
1890d05aa8bbSEric Blake                 !(bs->supported_zero_flags & BDRV_REQ_FUA)) {
1891d05aa8bbSEric Blake                 need_flush = true;
1892d05aa8bbSEric Blake             }
1893465fe887SEric Blake         } else {
1894465fe887SEric Blake             assert(!bs->supported_zero_flags);
189561007b31SStefan Hajnoczi         }
189661007b31SStefan Hajnoczi 
1897294682ccSAndrey Shinkevich         if (ret == -ENOTSUP && !(flags & BDRV_REQ_NO_FALLBACK)) {
189861007b31SStefan Hajnoczi             /* Fall back to bounce buffer if write zeroes is unsupported */
1899465fe887SEric Blake             BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;
1900465fe887SEric Blake 
1901465fe887SEric Blake             if ((flags & BDRV_REQ_FUA) &&
1902465fe887SEric Blake                 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1903465fe887SEric Blake                 /* No need for bdrv_driver_pwrite() to do a fallback
1904465fe887SEric Blake                  * flush on each chunk; use just one at the end */
1905465fe887SEric Blake                 write_flags &= ~BDRV_REQ_FUA;
1906465fe887SEric Blake                 need_flush = true;
1907465fe887SEric Blake             }
19085def6b80SEric Blake             num = MIN(num, max_transfer);
19090d93ed08SVladimir Sementsov-Ogievskiy             if (buf == NULL) {
19100d93ed08SVladimir Sementsov-Ogievskiy                 buf = qemu_try_blockalign0(bs, num);
19110d93ed08SVladimir Sementsov-Ogievskiy                 if (buf == NULL) {
191261007b31SStefan Hajnoczi                     ret = -ENOMEM;
191361007b31SStefan Hajnoczi                     goto fail;
191461007b31SStefan Hajnoczi                 }
191561007b31SStefan Hajnoczi             }
19160d93ed08SVladimir Sementsov-Ogievskiy             qemu_iovec_init_buf(&qiov, buf, num);
191761007b31SStefan Hajnoczi 
1918ac850bf0SVladimir Sementsov-Ogievskiy             ret = bdrv_driver_pwritev(bs, offset, num, &qiov, 0, write_flags);
191961007b31SStefan Hajnoczi 
192061007b31SStefan Hajnoczi             /* Keep bounce buffer around if it is big enough for all
192161007b31SStefan Hajnoczi              * all future requests.
192261007b31SStefan Hajnoczi              */
19235def6b80SEric Blake             if (num < max_transfer) {
19240d93ed08SVladimir Sementsov-Ogievskiy                 qemu_vfree(buf);
19250d93ed08SVladimir Sementsov-Ogievskiy                 buf = NULL;
192661007b31SStefan Hajnoczi             }
192761007b31SStefan Hajnoczi         }
192861007b31SStefan Hajnoczi 
1929d05aa8bbSEric Blake         offset += num;
1930f5a5ca79SManos Pitsidianakis         bytes -= num;
193161007b31SStefan Hajnoczi     }
193261007b31SStefan Hajnoczi 
193361007b31SStefan Hajnoczi fail:
1934465fe887SEric Blake     if (ret == 0 && need_flush) {
1935465fe887SEric Blake         ret = bdrv_co_flush(bs);
1936465fe887SEric Blake     }
19370d93ed08SVladimir Sementsov-Ogievskiy     qemu_vfree(buf);
193861007b31SStefan Hajnoczi     return ret;
193961007b31SStefan Hajnoczi }
194061007b31SStefan Hajnoczi 
194185fe2479SFam Zheng static inline int coroutine_fn
1942fcfd9adeSVladimir Sementsov-Ogievskiy bdrv_co_write_req_prepare(BdrvChild *child, int64_t offset, int64_t bytes,
194385fe2479SFam Zheng                           BdrvTrackedRequest *req, int flags)
194485fe2479SFam Zheng {
194585fe2479SFam Zheng     BlockDriverState *bs = child->bs;
1946fcfd9adeSVladimir Sementsov-Ogievskiy 
1947fcfd9adeSVladimir Sementsov-Ogievskiy     bdrv_check_request(offset, bytes, &error_abort);
194885fe2479SFam Zheng 
1949307261b2SVladimir Sementsov-Ogievskiy     if (bdrv_is_read_only(bs)) {
195085fe2479SFam Zheng         return -EPERM;
195185fe2479SFam Zheng     }
195285fe2479SFam Zheng 
195385fe2479SFam Zheng     assert(!(bs->open_flags & BDRV_O_INACTIVE));
195485fe2479SFam Zheng     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
195585fe2479SFam Zheng     assert(!(flags & ~BDRV_REQ_MASK));
1956d1a764d1SVladimir Sementsov-Ogievskiy     assert(!((flags & BDRV_REQ_NO_WAIT) && !(flags & BDRV_REQ_SERIALISING)));
195785fe2479SFam Zheng 
195885fe2479SFam Zheng     if (flags & BDRV_REQ_SERIALISING) {
1959d1a764d1SVladimir Sementsov-Ogievskiy         QEMU_LOCK_GUARD(&bs->reqs_lock);
1960d1a764d1SVladimir Sementsov-Ogievskiy 
1961d1a764d1SVladimir Sementsov-Ogievskiy         tracked_request_set_serialising(req, bdrv_get_cluster_size(bs));
1962d1a764d1SVladimir Sementsov-Ogievskiy 
1963d1a764d1SVladimir Sementsov-Ogievskiy         if ((flags & BDRV_REQ_NO_WAIT) && bdrv_find_conflicting_request(req)) {
1964d1a764d1SVladimir Sementsov-Ogievskiy             return -EBUSY;
1965d1a764d1SVladimir Sementsov-Ogievskiy         }
1966d1a764d1SVladimir Sementsov-Ogievskiy 
1967d1a764d1SVladimir Sementsov-Ogievskiy         bdrv_wait_serialising_requests_locked(req);
196818fbd0deSPaolo Bonzini     } else {
196918fbd0deSPaolo Bonzini         bdrv_wait_serialising_requests(req);
197085fe2479SFam Zheng     }
197185fe2479SFam Zheng 
197285fe2479SFam Zheng     assert(req->overlap_offset <= offset);
197385fe2479SFam Zheng     assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
1974fcfd9adeSVladimir Sementsov-Ogievskiy     assert(offset + bytes <= bs->total_sectors * BDRV_SECTOR_SIZE ||
1975fcfd9adeSVladimir Sementsov-Ogievskiy            child->perm & BLK_PERM_RESIZE);
197685fe2479SFam Zheng 
1977cd47d792SFam Zheng     switch (req->type) {
1978cd47d792SFam Zheng     case BDRV_TRACKED_WRITE:
1979cd47d792SFam Zheng     case BDRV_TRACKED_DISCARD:
198085fe2479SFam Zheng         if (flags & BDRV_REQ_WRITE_UNCHANGED) {
198185fe2479SFam Zheng             assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
198285fe2479SFam Zheng         } else {
198385fe2479SFam Zheng             assert(child->perm & BLK_PERM_WRITE);
198485fe2479SFam Zheng         }
198594783301SVladimir Sementsov-Ogievskiy         bdrv_write_threshold_check_write(bs, offset, bytes);
198694783301SVladimir Sementsov-Ogievskiy         return 0;
1987cd47d792SFam Zheng     case BDRV_TRACKED_TRUNCATE:
1988cd47d792SFam Zheng         assert(child->perm & BLK_PERM_RESIZE);
1989cd47d792SFam Zheng         return 0;
1990cd47d792SFam Zheng     default:
1991cd47d792SFam Zheng         abort();
1992cd47d792SFam Zheng     }
199385fe2479SFam Zheng }
199485fe2479SFam Zheng 
199585fe2479SFam Zheng static inline void coroutine_fn
1996fcfd9adeSVladimir Sementsov-Ogievskiy bdrv_co_write_req_finish(BdrvChild *child, int64_t offset, int64_t bytes,
199785fe2479SFam Zheng                          BdrvTrackedRequest *req, int ret)
199885fe2479SFam Zheng {
199985fe2479SFam Zheng     int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
200085fe2479SFam Zheng     BlockDriverState *bs = child->bs;
200185fe2479SFam Zheng 
2002fcfd9adeSVladimir Sementsov-Ogievskiy     bdrv_check_request(offset, bytes, &error_abort);
2003fcfd9adeSVladimir Sementsov-Ogievskiy 
2004d73415a3SStefan Hajnoczi     qatomic_inc(&bs->write_gen);
200585fe2479SFam Zheng 
200600695c27SFam Zheng     /*
200700695c27SFam Zheng      * Discard cannot extend the image, but in error handling cases, such as
200800695c27SFam Zheng      * when reverting a qcow2 cluster allocation, the discarded range can pass
200900695c27SFam Zheng      * the end of image file, so we cannot assert about BDRV_TRACKED_DISCARD
201000695c27SFam Zheng      * here. Instead, just skip it, since semantically a discard request
201100695c27SFam Zheng      * beyond EOF cannot expand the image anyway.
201200695c27SFam Zheng      */
20137f8f03efSFam Zheng     if (ret == 0 &&
2014cd47d792SFam Zheng         (req->type == BDRV_TRACKED_TRUNCATE ||
2015cd47d792SFam Zheng          end_sector > bs->total_sectors) &&
201600695c27SFam Zheng         req->type != BDRV_TRACKED_DISCARD) {
20177f8f03efSFam Zheng         bs->total_sectors = end_sector;
20187f8f03efSFam Zheng         bdrv_parent_cb_resize(bs);
20197f8f03efSFam Zheng         bdrv_dirty_bitmap_truncate(bs, end_sector << BDRV_SECTOR_BITS);
202085fe2479SFam Zheng     }
202100695c27SFam Zheng     if (req->bytes) {
202200695c27SFam Zheng         switch (req->type) {
202300695c27SFam Zheng         case BDRV_TRACKED_WRITE:
202400695c27SFam Zheng             stat64_max(&bs->wr_highest_offset, offset + bytes);
202500695c27SFam Zheng             /* fall through, to set dirty bits */
202600695c27SFam Zheng         case BDRV_TRACKED_DISCARD:
20277f8f03efSFam Zheng             bdrv_set_dirty(bs, offset, bytes);
202800695c27SFam Zheng             break;
202900695c27SFam Zheng         default:
203000695c27SFam Zheng             break;
203100695c27SFam Zheng         }
203200695c27SFam Zheng     }
203385fe2479SFam Zheng }
203485fe2479SFam Zheng 
203561007b31SStefan Hajnoczi /*
203604ed95f4SEric Blake  * Forwards an already correctly aligned write request to the BlockDriver,
203704ed95f4SEric Blake  * after possibly fragmenting it.
203861007b31SStefan Hajnoczi  */
203985c97ca7SKevin Wolf static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child,
2040fcfd9adeSVladimir Sementsov-Ogievskiy     BdrvTrackedRequest *req, int64_t offset, int64_t bytes,
2041e75abedaSVladimir Sementsov-Ogievskiy     int64_t align, QEMUIOVector *qiov, size_t qiov_offset,
2042e75abedaSVladimir Sementsov-Ogievskiy     BdrvRequestFlags flags)
204361007b31SStefan Hajnoczi {
204485c97ca7SKevin Wolf     BlockDriverState *bs = child->bs;
204561007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
204661007b31SStefan Hajnoczi     int ret;
204761007b31SStefan Hajnoczi 
2048fcfd9adeSVladimir Sementsov-Ogievskiy     int64_t bytes_remaining = bytes;
204904ed95f4SEric Blake     int max_transfer;
205061007b31SStefan Hajnoczi 
2051fcfd9adeSVladimir Sementsov-Ogievskiy     bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
2052fcfd9adeSVladimir Sementsov-Ogievskiy 
2053d470ad42SMax Reitz     if (!drv) {
2054d470ad42SMax Reitz         return -ENOMEDIUM;
2055d470ad42SMax Reitz     }
2056d470ad42SMax Reitz 
2057d6883bc9SVladimir Sementsov-Ogievskiy     if (bdrv_has_readonly_bitmaps(bs)) {
2058d6883bc9SVladimir Sementsov-Ogievskiy         return -EPERM;
2059d6883bc9SVladimir Sementsov-Ogievskiy     }
2060d6883bc9SVladimir Sementsov-Ogievskiy 
2061cff86b38SEric Blake     assert(is_power_of_2(align));
2062cff86b38SEric Blake     assert((offset & (align - 1)) == 0);
2063cff86b38SEric Blake     assert((bytes & (align - 1)) == 0);
206404ed95f4SEric Blake     max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
206504ed95f4SEric Blake                                    align);
206661007b31SStefan Hajnoczi 
206785fe2479SFam Zheng     ret = bdrv_co_write_req_prepare(child, offset, bytes, req, flags);
206861007b31SStefan Hajnoczi 
206961007b31SStefan Hajnoczi     if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
2070c1499a5eSEric Blake         !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes &&
207128c4da28SVladimir Sementsov-Ogievskiy         qemu_iovec_is_zero(qiov, qiov_offset, bytes)) {
207261007b31SStefan Hajnoczi         flags |= BDRV_REQ_ZERO_WRITE;
207361007b31SStefan Hajnoczi         if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
207461007b31SStefan Hajnoczi             flags |= BDRV_REQ_MAY_UNMAP;
207561007b31SStefan Hajnoczi         }
207661007b31SStefan Hajnoczi     }
207761007b31SStefan Hajnoczi 
207861007b31SStefan Hajnoczi     if (ret < 0) {
207961007b31SStefan Hajnoczi         /* Do nothing, write notifier decided to fail this request */
208061007b31SStefan Hajnoczi     } else if (flags & BDRV_REQ_ZERO_WRITE) {
20819a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
20829896c876SKevin Wolf         ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags);
20833ea1a091SPavel Butsykin     } else if (flags & BDRV_REQ_WRITE_COMPRESSED) {
208428c4da28SVladimir Sementsov-Ogievskiy         ret = bdrv_driver_pwritev_compressed(bs, offset, bytes,
208528c4da28SVladimir Sementsov-Ogievskiy                                              qiov, qiov_offset);
208604ed95f4SEric Blake     } else if (bytes <= max_transfer) {
20879a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV);
208828c4da28SVladimir Sementsov-Ogievskiy         ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, qiov_offset, flags);
208904ed95f4SEric Blake     } else {
209004ed95f4SEric Blake         bdrv_debug_event(bs, BLKDBG_PWRITEV);
209104ed95f4SEric Blake         while (bytes_remaining) {
209204ed95f4SEric Blake             int num = MIN(bytes_remaining, max_transfer);
209304ed95f4SEric Blake             int local_flags = flags;
209404ed95f4SEric Blake 
209504ed95f4SEric Blake             assert(num);
209604ed95f4SEric Blake             if (num < bytes_remaining && (flags & BDRV_REQ_FUA) &&
209704ed95f4SEric Blake                 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
209804ed95f4SEric Blake                 /* If FUA is going to be emulated by flush, we only
209904ed95f4SEric Blake                  * need to flush on the last iteration */
210004ed95f4SEric Blake                 local_flags &= ~BDRV_REQ_FUA;
210104ed95f4SEric Blake             }
210204ed95f4SEric Blake 
210304ed95f4SEric Blake             ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining,
2104134b7decSMax Reitz                                       num, qiov,
2105134b7decSMax Reitz                                       qiov_offset + bytes - bytes_remaining,
210628c4da28SVladimir Sementsov-Ogievskiy                                       local_flags);
210704ed95f4SEric Blake             if (ret < 0) {
210804ed95f4SEric Blake                 break;
210904ed95f4SEric Blake             }
211004ed95f4SEric Blake             bytes_remaining -= num;
211104ed95f4SEric Blake         }
211261007b31SStefan Hajnoczi     }
21139a4f4c31SKevin Wolf     bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
211461007b31SStefan Hajnoczi 
211561007b31SStefan Hajnoczi     if (ret >= 0) {
211604ed95f4SEric Blake         ret = 0;
211761007b31SStefan Hajnoczi     }
211885fe2479SFam Zheng     bdrv_co_write_req_finish(child, offset, bytes, req, ret);
211961007b31SStefan Hajnoczi 
212061007b31SStefan Hajnoczi     return ret;
212161007b31SStefan Hajnoczi }
212261007b31SStefan Hajnoczi 
212385c97ca7SKevin Wolf static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child,
21249eeb6dd1SFam Zheng                                                 int64_t offset,
212537e9403eSVladimir Sementsov-Ogievskiy                                                 int64_t bytes,
21269eeb6dd1SFam Zheng                                                 BdrvRequestFlags flags,
21279eeb6dd1SFam Zheng                                                 BdrvTrackedRequest *req)
21289eeb6dd1SFam Zheng {
212985c97ca7SKevin Wolf     BlockDriverState *bs = child->bs;
21309eeb6dd1SFam Zheng     QEMUIOVector local_qiov;
2131a5b8dd2cSEric Blake     uint64_t align = bs->bl.request_alignment;
21329eeb6dd1SFam Zheng     int ret = 0;
21337a3f542fSVladimir Sementsov-Ogievskiy     bool padding;
21347a3f542fSVladimir Sementsov-Ogievskiy     BdrvRequestPadding pad;
21359eeb6dd1SFam Zheng 
21367a3f542fSVladimir Sementsov-Ogievskiy     padding = bdrv_init_padding(bs, offset, bytes, &pad);
21377a3f542fSVladimir Sementsov-Ogievskiy     if (padding) {
213845e62b46SVladimir Sementsov-Ogievskiy         assert(!(flags & BDRV_REQ_NO_WAIT));
21398ac5aab2SVladimir Sementsov-Ogievskiy         bdrv_make_request_serialising(req, align);
21409eeb6dd1SFam Zheng 
21417a3f542fSVladimir Sementsov-Ogievskiy         bdrv_padding_rmw_read(child, req, &pad, true);
21427a3f542fSVladimir Sementsov-Ogievskiy 
21437a3f542fSVladimir Sementsov-Ogievskiy         if (pad.head || pad.merge_reads) {
21447a3f542fSVladimir Sementsov-Ogievskiy             int64_t aligned_offset = offset & ~(align - 1);
21457a3f542fSVladimir Sementsov-Ogievskiy             int64_t write_bytes = pad.merge_reads ? pad.buf_len : align;
21467a3f542fSVladimir Sementsov-Ogievskiy 
21477a3f542fSVladimir Sementsov-Ogievskiy             qemu_iovec_init_buf(&local_qiov, pad.buf, write_bytes);
21487a3f542fSVladimir Sementsov-Ogievskiy             ret = bdrv_aligned_pwritev(child, req, aligned_offset, write_bytes,
214928c4da28SVladimir Sementsov-Ogievskiy                                        align, &local_qiov, 0,
21509eeb6dd1SFam Zheng                                        flags & ~BDRV_REQ_ZERO_WRITE);
21517a3f542fSVladimir Sementsov-Ogievskiy             if (ret < 0 || pad.merge_reads) {
21527a3f542fSVladimir Sementsov-Ogievskiy                 /* Error or all work is done */
21537a3f542fSVladimir Sementsov-Ogievskiy                 goto out;
21549eeb6dd1SFam Zheng             }
21557a3f542fSVladimir Sementsov-Ogievskiy             offset += write_bytes - pad.head;
21567a3f542fSVladimir Sementsov-Ogievskiy             bytes -= write_bytes - pad.head;
21577a3f542fSVladimir Sementsov-Ogievskiy         }
21589eeb6dd1SFam Zheng     }
21599eeb6dd1SFam Zheng 
21609eeb6dd1SFam Zheng     assert(!bytes || (offset & (align - 1)) == 0);
21619eeb6dd1SFam Zheng     if (bytes >= align) {
21629eeb6dd1SFam Zheng         /* Write the aligned part in the middle. */
2163fcfd9adeSVladimir Sementsov-Ogievskiy         int64_t aligned_bytes = bytes & ~(align - 1);
216485c97ca7SKevin Wolf         ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align,
216528c4da28SVladimir Sementsov-Ogievskiy                                    NULL, 0, flags);
21669eeb6dd1SFam Zheng         if (ret < 0) {
21677a3f542fSVladimir Sementsov-Ogievskiy             goto out;
21689eeb6dd1SFam Zheng         }
21699eeb6dd1SFam Zheng         bytes -= aligned_bytes;
21709eeb6dd1SFam Zheng         offset += aligned_bytes;
21719eeb6dd1SFam Zheng     }
21729eeb6dd1SFam Zheng 
21739eeb6dd1SFam Zheng     assert(!bytes || (offset & (align - 1)) == 0);
21749eeb6dd1SFam Zheng     if (bytes) {
21757a3f542fSVladimir Sementsov-Ogievskiy         assert(align == pad.tail + bytes);
21769eeb6dd1SFam Zheng 
21777a3f542fSVladimir Sementsov-Ogievskiy         qemu_iovec_init_buf(&local_qiov, pad.tail_buf, align);
217885c97ca7SKevin Wolf         ret = bdrv_aligned_pwritev(child, req, offset, align, align,
217928c4da28SVladimir Sementsov-Ogievskiy                                    &local_qiov, 0,
218028c4da28SVladimir Sementsov-Ogievskiy                                    flags & ~BDRV_REQ_ZERO_WRITE);
21819eeb6dd1SFam Zheng     }
21829eeb6dd1SFam Zheng 
21837a3f542fSVladimir Sementsov-Ogievskiy out:
21847a3f542fSVladimir Sementsov-Ogievskiy     bdrv_padding_destroy(&pad);
21857a3f542fSVladimir Sementsov-Ogievskiy 
21867a3f542fSVladimir Sementsov-Ogievskiy     return ret;
21879eeb6dd1SFam Zheng }
21889eeb6dd1SFam Zheng 
218961007b31SStefan Hajnoczi /*
219061007b31SStefan Hajnoczi  * Handle a write request in coroutine context
219161007b31SStefan Hajnoczi  */
2192a03ef88fSKevin Wolf int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
2193e9e52efdSVladimir Sementsov-Ogievskiy     int64_t offset, int64_t bytes, QEMUIOVector *qiov,
219461007b31SStefan Hajnoczi     BdrvRequestFlags flags)
219561007b31SStefan Hajnoczi {
2196967d7905SEmanuele Giuseppe Esposito     IO_CODE();
21971acc3466SVladimir Sementsov-Ogievskiy     return bdrv_co_pwritev_part(child, offset, bytes, qiov, 0, flags);
21981acc3466SVladimir Sementsov-Ogievskiy }
21991acc3466SVladimir Sementsov-Ogievskiy 
22001acc3466SVladimir Sementsov-Ogievskiy int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child,
220137e9403eSVladimir Sementsov-Ogievskiy     int64_t offset, int64_t bytes, QEMUIOVector *qiov, size_t qiov_offset,
22021acc3466SVladimir Sementsov-Ogievskiy     BdrvRequestFlags flags)
22031acc3466SVladimir Sementsov-Ogievskiy {
2204a03ef88fSKevin Wolf     BlockDriverState *bs = child->bs;
220561007b31SStefan Hajnoczi     BdrvTrackedRequest req;
2206a5b8dd2cSEric Blake     uint64_t align = bs->bl.request_alignment;
22077a3f542fSVladimir Sementsov-Ogievskiy     BdrvRequestPadding pad;
220861007b31SStefan Hajnoczi     int ret;
2209f0deecffSVladimir Sementsov-Ogievskiy     bool padded = false;
2210967d7905SEmanuele Giuseppe Esposito     IO_CODE();
221161007b31SStefan Hajnoczi 
221237e9403eSVladimir Sementsov-Ogievskiy     trace_bdrv_co_pwritev_part(child->bs, offset, bytes, flags);
2213f42cf447SDaniel P. Berrange 
2214f4dad307SVladimir Sementsov-Ogievskiy     if (!bdrv_is_inserted(bs)) {
221561007b31SStefan Hajnoczi         return -ENOMEDIUM;
221661007b31SStefan Hajnoczi     }
221761007b31SStefan Hajnoczi 
22182aaa3f9bSVladimir Sementsov-Ogievskiy     if (flags & BDRV_REQ_ZERO_WRITE) {
22192aaa3f9bSVladimir Sementsov-Ogievskiy         ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL);
22202aaa3f9bSVladimir Sementsov-Ogievskiy     } else {
222163f4ad11SVladimir Sementsov-Ogievskiy         ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset);
22222aaa3f9bSVladimir Sementsov-Ogievskiy     }
222361007b31SStefan Hajnoczi     if (ret < 0) {
222461007b31SStefan Hajnoczi         return ret;
222561007b31SStefan Hajnoczi     }
222661007b31SStefan Hajnoczi 
2227f2208fdcSAlberto Garcia     /* If the request is misaligned then we can't make it efficient */
2228f2208fdcSAlberto Garcia     if ((flags & BDRV_REQ_NO_FALLBACK) &&
2229f2208fdcSAlberto Garcia         !QEMU_IS_ALIGNED(offset | bytes, align))
2230f2208fdcSAlberto Garcia     {
2231f2208fdcSAlberto Garcia         return -ENOTSUP;
2232f2208fdcSAlberto Garcia     }
2233f2208fdcSAlberto Garcia 
2234ac9d00bfSVladimir Sementsov-Ogievskiy     if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) {
2235ac9d00bfSVladimir Sementsov-Ogievskiy         /*
2236ac9d00bfSVladimir Sementsov-Ogievskiy          * Aligning zero request is nonsense. Even if driver has special meaning
2237ac9d00bfSVladimir Sementsov-Ogievskiy          * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass
2238ac9d00bfSVladimir Sementsov-Ogievskiy          * it to driver due to request_alignment.
2239ac9d00bfSVladimir Sementsov-Ogievskiy          *
2240ac9d00bfSVladimir Sementsov-Ogievskiy          * Still, no reason to return an error if someone do unaligned
2241ac9d00bfSVladimir Sementsov-Ogievskiy          * zero-length write occasionally.
2242ac9d00bfSVladimir Sementsov-Ogievskiy          */
2243ac9d00bfSVladimir Sementsov-Ogievskiy         return 0;
2244ac9d00bfSVladimir Sementsov-Ogievskiy     }
2245ac9d00bfSVladimir Sementsov-Ogievskiy 
2246f0deecffSVladimir Sementsov-Ogievskiy     if (!(flags & BDRV_REQ_ZERO_WRITE)) {
224761007b31SStefan Hajnoczi         /*
2248f0deecffSVladimir Sementsov-Ogievskiy          * Pad request for following read-modify-write cycle.
2249f0deecffSVladimir Sementsov-Ogievskiy          * bdrv_co_do_zero_pwritev() does aligning by itself, so, we do
2250f0deecffSVladimir Sementsov-Ogievskiy          * alignment only if there is no ZERO flag.
225161007b31SStefan Hajnoczi          */
225298ca4549SVladimir Sementsov-Ogievskiy         ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad,
225398ca4549SVladimir Sementsov-Ogievskiy                                &padded);
225498ca4549SVladimir Sementsov-Ogievskiy         if (ret < 0) {
225598ca4549SVladimir Sementsov-Ogievskiy             return ret;
225698ca4549SVladimir Sementsov-Ogievskiy         }
2257f0deecffSVladimir Sementsov-Ogievskiy     }
2258f0deecffSVladimir Sementsov-Ogievskiy 
2259f0deecffSVladimir Sementsov-Ogievskiy     bdrv_inc_in_flight(bs);
2260ebde595cSFam Zheng     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
226161007b31SStefan Hajnoczi 
226218a59f03SAnton Nefedov     if (flags & BDRV_REQ_ZERO_WRITE) {
2263f0deecffSVladimir Sementsov-Ogievskiy         assert(!padded);
226485c97ca7SKevin Wolf         ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req);
22659eeb6dd1SFam Zheng         goto out;
22669eeb6dd1SFam Zheng     }
22679eeb6dd1SFam Zheng 
2268f0deecffSVladimir Sementsov-Ogievskiy     if (padded) {
2269f0deecffSVladimir Sementsov-Ogievskiy         /*
2270f0deecffSVladimir Sementsov-Ogievskiy          * Request was unaligned to request_alignment and therefore
2271f0deecffSVladimir Sementsov-Ogievskiy          * padded.  We are going to do read-modify-write, and must
2272f0deecffSVladimir Sementsov-Ogievskiy          * serialize the request to prevent interactions of the
2273f0deecffSVladimir Sementsov-Ogievskiy          * widened region with other transactions.
2274f0deecffSVladimir Sementsov-Ogievskiy          */
227545e62b46SVladimir Sementsov-Ogievskiy         assert(!(flags & BDRV_REQ_NO_WAIT));
22768ac5aab2SVladimir Sementsov-Ogievskiy         bdrv_make_request_serialising(&req, align);
22777a3f542fSVladimir Sementsov-Ogievskiy         bdrv_padding_rmw_read(child, &req, &pad, false);
227861007b31SStefan Hajnoczi     }
227961007b31SStefan Hajnoczi 
228085c97ca7SKevin Wolf     ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align,
22811acc3466SVladimir Sementsov-Ogievskiy                                qiov, qiov_offset, flags);
228261007b31SStefan Hajnoczi 
22837a3f542fSVladimir Sementsov-Ogievskiy     bdrv_padding_destroy(&pad);
228461007b31SStefan Hajnoczi 
22859eeb6dd1SFam Zheng out:
22869eeb6dd1SFam Zheng     tracked_request_end(&req);
228799723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
22887a3f542fSVladimir Sementsov-Ogievskiy 
228961007b31SStefan Hajnoczi     return ret;
229061007b31SStefan Hajnoczi }
229161007b31SStefan Hajnoczi 
2292a03ef88fSKevin Wolf int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset,
2293e9e52efdSVladimir Sementsov-Ogievskiy                                        int64_t bytes, BdrvRequestFlags flags)
229461007b31SStefan Hajnoczi {
2295384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
2296f5a5ca79SManos Pitsidianakis     trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags);
229761007b31SStefan Hajnoczi 
2298a03ef88fSKevin Wolf     if (!(child->bs->open_flags & BDRV_O_UNMAP)) {
229961007b31SStefan Hajnoczi         flags &= ~BDRV_REQ_MAY_UNMAP;
230061007b31SStefan Hajnoczi     }
230161007b31SStefan Hajnoczi 
2302f5a5ca79SManos Pitsidianakis     return bdrv_co_pwritev(child, offset, bytes, NULL,
230361007b31SStefan Hajnoczi                            BDRV_REQ_ZERO_WRITE | flags);
230461007b31SStefan Hajnoczi }
230561007b31SStefan Hajnoczi 
23064085f5c7SJohn Snow /*
23074085f5c7SJohn Snow  * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not.
23084085f5c7SJohn Snow  */
23094085f5c7SJohn Snow int bdrv_flush_all(void)
23104085f5c7SJohn Snow {
23114085f5c7SJohn Snow     BdrvNextIterator it;
23124085f5c7SJohn Snow     BlockDriverState *bs = NULL;
23134085f5c7SJohn Snow     int result = 0;
23144085f5c7SJohn Snow 
2315f791bf7fSEmanuele Giuseppe Esposito     GLOBAL_STATE_CODE();
2316f791bf7fSEmanuele Giuseppe Esposito 
2317c8aa7895SPavel Dovgalyuk     /*
2318c8aa7895SPavel Dovgalyuk      * bdrv queue is managed by record/replay,
2319c8aa7895SPavel Dovgalyuk      * creating new flush request for stopping
2320c8aa7895SPavel Dovgalyuk      * the VM may break the determinism
2321c8aa7895SPavel Dovgalyuk      */
2322c8aa7895SPavel Dovgalyuk     if (replay_events_enabled()) {
2323c8aa7895SPavel Dovgalyuk         return result;
2324c8aa7895SPavel Dovgalyuk     }
2325c8aa7895SPavel Dovgalyuk 
23264085f5c7SJohn Snow     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
23274085f5c7SJohn Snow         AioContext *aio_context = bdrv_get_aio_context(bs);
23284085f5c7SJohn Snow         int ret;
23294085f5c7SJohn Snow 
23304085f5c7SJohn Snow         aio_context_acquire(aio_context);
23314085f5c7SJohn Snow         ret = bdrv_flush(bs);
23324085f5c7SJohn Snow         if (ret < 0 && !result) {
23334085f5c7SJohn Snow             result = ret;
23344085f5c7SJohn Snow         }
23354085f5c7SJohn Snow         aio_context_release(aio_context);
23364085f5c7SJohn Snow     }
23374085f5c7SJohn Snow 
23384085f5c7SJohn Snow     return result;
23394085f5c7SJohn Snow }
23404085f5c7SJohn Snow 
234161007b31SStefan Hajnoczi /*
234261007b31SStefan Hajnoczi  * Returns the allocation status of the specified sectors.
234361007b31SStefan Hajnoczi  * Drivers not implementing the functionality are assumed to not support
234461007b31SStefan Hajnoczi  * backing files, hence all their sectors are reported as allocated.
234561007b31SStefan Hajnoczi  *
234686a3d5c6SEric Blake  * If 'want_zero' is true, the caller is querying for mapping
234786a3d5c6SEric Blake  * purposes, with a focus on valid BDRV_BLOCK_OFFSET_VALID, _DATA, and
234886a3d5c6SEric Blake  * _ZERO where possible; otherwise, the result favors larger 'pnum',
234986a3d5c6SEric Blake  * with a focus on accurate BDRV_BLOCK_ALLOCATED.
2350c9ce8c4dSEric Blake  *
23512e8bc787SEric Blake  * If 'offset' is beyond the end of the disk image the return value is
2352fb0d8654SEric Blake  * BDRV_BLOCK_EOF and 'pnum' is set to 0.
235361007b31SStefan Hajnoczi  *
23542e8bc787SEric Blake  * 'bytes' is the max value 'pnum' should be set to.  If bytes goes
2355fb0d8654SEric Blake  * beyond the end of the disk image it will be clamped; if 'pnum' is set to
2356fb0d8654SEric Blake  * the end of the image, then the returned value will include BDRV_BLOCK_EOF.
235767a0fd2aSFam Zheng  *
23582e8bc787SEric Blake  * 'pnum' is set to the number of bytes (including and immediately
23592e8bc787SEric Blake  * following the specified offset) that are easily known to be in the
23602e8bc787SEric Blake  * same allocated/unallocated state.  Note that a second call starting
23612e8bc787SEric Blake  * at the original offset plus returned pnum may have the same status.
23622e8bc787SEric Blake  * The returned value is non-zero on success except at end-of-file.
23632e8bc787SEric Blake  *
23642e8bc787SEric Blake  * Returns negative errno on failure.  Otherwise, if the
23652e8bc787SEric Blake  * BDRV_BLOCK_OFFSET_VALID bit is set, 'map' and 'file' (if non-NULL) are
23662e8bc787SEric Blake  * set to the host mapping and BDS corresponding to the guest offset.
236761007b31SStefan Hajnoczi  */
23682e8bc787SEric Blake static int coroutine_fn bdrv_co_block_status(BlockDriverState *bs,
2369c9ce8c4dSEric Blake                                              bool want_zero,
23702e8bc787SEric Blake                                              int64_t offset, int64_t bytes,
23712e8bc787SEric Blake                                              int64_t *pnum, int64_t *map,
237267a0fd2aSFam Zheng                                              BlockDriverState **file)
237361007b31SStefan Hajnoczi {
23742e8bc787SEric Blake     int64_t total_size;
23752e8bc787SEric Blake     int64_t n; /* bytes */
2376efa6e2edSEric Blake     int ret;
23772e8bc787SEric Blake     int64_t local_map = 0;
2378298a1665SEric Blake     BlockDriverState *local_file = NULL;
2379efa6e2edSEric Blake     int64_t aligned_offset, aligned_bytes;
2380efa6e2edSEric Blake     uint32_t align;
2381549ec0d9SMax Reitz     bool has_filtered_child;
238261007b31SStefan Hajnoczi 
2383298a1665SEric Blake     assert(pnum);
2384298a1665SEric Blake     *pnum = 0;
23852e8bc787SEric Blake     total_size = bdrv_getlength(bs);
23862e8bc787SEric Blake     if (total_size < 0) {
23872e8bc787SEric Blake         ret = total_size;
2388298a1665SEric Blake         goto early_out;
238961007b31SStefan Hajnoczi     }
239061007b31SStefan Hajnoczi 
23912e8bc787SEric Blake     if (offset >= total_size) {
2392298a1665SEric Blake         ret = BDRV_BLOCK_EOF;
2393298a1665SEric Blake         goto early_out;
239461007b31SStefan Hajnoczi     }
23952e8bc787SEric Blake     if (!bytes) {
2396298a1665SEric Blake         ret = 0;
2397298a1665SEric Blake         goto early_out;
23989cdcfd9fSEric Blake     }
239961007b31SStefan Hajnoczi 
24002e8bc787SEric Blake     n = total_size - offset;
24012e8bc787SEric Blake     if (n < bytes) {
24022e8bc787SEric Blake         bytes = n;
240361007b31SStefan Hajnoczi     }
240461007b31SStefan Hajnoczi 
2405d470ad42SMax Reitz     /* Must be non-NULL or bdrv_getlength() would have failed */
2406d470ad42SMax Reitz     assert(bs->drv);
2407549ec0d9SMax Reitz     has_filtered_child = bdrv_filter_child(bs);
2408549ec0d9SMax Reitz     if (!bs->drv->bdrv_co_block_status && !has_filtered_child) {
24092e8bc787SEric Blake         *pnum = bytes;
241061007b31SStefan Hajnoczi         ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
24112e8bc787SEric Blake         if (offset + bytes == total_size) {
2412fb0d8654SEric Blake             ret |= BDRV_BLOCK_EOF;
2413fb0d8654SEric Blake         }
241461007b31SStefan Hajnoczi         if (bs->drv->protocol_name) {
24152e8bc787SEric Blake             ret |= BDRV_BLOCK_OFFSET_VALID;
24162e8bc787SEric Blake             local_map = offset;
2417298a1665SEric Blake             local_file = bs;
241861007b31SStefan Hajnoczi         }
2419298a1665SEric Blake         goto early_out;
242061007b31SStefan Hajnoczi     }
242161007b31SStefan Hajnoczi 
242299723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
2423efa6e2edSEric Blake 
2424efa6e2edSEric Blake     /* Round out to request_alignment boundaries */
242586a3d5c6SEric Blake     align = bs->bl.request_alignment;
2426efa6e2edSEric Blake     aligned_offset = QEMU_ALIGN_DOWN(offset, align);
2427efa6e2edSEric Blake     aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset;
2428efa6e2edSEric Blake 
2429549ec0d9SMax Reitz     if (bs->drv->bdrv_co_block_status) {
24300bc329fbSHanna Reitz         /*
24310bc329fbSHanna Reitz          * Use the block-status cache only for protocol nodes: Format
24320bc329fbSHanna Reitz          * drivers are generally quick to inquire the status, but protocol
24330bc329fbSHanna Reitz          * drivers often need to get information from outside of qemu, so
24340bc329fbSHanna Reitz          * we do not have control over the actual implementation.  There
24350bc329fbSHanna Reitz          * have been cases where inquiring the status took an unreasonably
24360bc329fbSHanna Reitz          * long time, and we can do nothing in qemu to fix it.
24370bc329fbSHanna Reitz          * This is especially problematic for images with large data areas,
24380bc329fbSHanna Reitz          * because finding the few holes in them and giving them special
24390bc329fbSHanna Reitz          * treatment does not gain much performance.  Therefore, we try to
24400bc329fbSHanna Reitz          * cache the last-identified data region.
24410bc329fbSHanna Reitz          *
24420bc329fbSHanna Reitz          * Second, limiting ourselves to protocol nodes allows us to assume
24430bc329fbSHanna Reitz          * the block status for data regions to be DATA | OFFSET_VALID, and
24440bc329fbSHanna Reitz          * that the host offset is the same as the guest offset.
24450bc329fbSHanna Reitz          *
24460bc329fbSHanna Reitz          * Note that it is possible that external writers zero parts of
24470bc329fbSHanna Reitz          * the cached regions without the cache being invalidated, and so
24480bc329fbSHanna Reitz          * we may report zeroes as data.  This is not catastrophic,
24490bc329fbSHanna Reitz          * however, because reporting zeroes as data is fine.
24500bc329fbSHanna Reitz          */
24510bc329fbSHanna Reitz         if (QLIST_EMPTY(&bs->children) &&
24520bc329fbSHanna Reitz             bdrv_bsc_is_data(bs, aligned_offset, pnum))
24530bc329fbSHanna Reitz         {
24540bc329fbSHanna Reitz             ret = BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID;
24550bc329fbSHanna Reitz             local_file = bs;
24560bc329fbSHanna Reitz             local_map = aligned_offset;
24570bc329fbSHanna Reitz         } else {
245886a3d5c6SEric Blake             ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset,
245986a3d5c6SEric Blake                                                 aligned_bytes, pnum, &local_map,
246086a3d5c6SEric Blake                                                 &local_file);
24610bc329fbSHanna Reitz 
24620bc329fbSHanna Reitz             /*
24630bc329fbSHanna Reitz              * Note that checking QLIST_EMPTY(&bs->children) is also done when
24640bc329fbSHanna Reitz              * the cache is queried above.  Technically, we do not need to check
24650bc329fbSHanna Reitz              * it here; the worst that can happen is that we fill the cache for
24660bc329fbSHanna Reitz              * non-protocol nodes, and then it is never used.  However, filling
24670bc329fbSHanna Reitz              * the cache requires an RCU update, so double check here to avoid
24680bc329fbSHanna Reitz              * such an update if possible.
2469113b727cSHanna Reitz              *
2470113b727cSHanna Reitz              * Check want_zero, because we only want to update the cache when we
2471113b727cSHanna Reitz              * have accurate information about what is zero and what is data.
24720bc329fbSHanna Reitz              */
2473113b727cSHanna Reitz             if (want_zero &&
2474113b727cSHanna Reitz                 ret == (BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID) &&
24750bc329fbSHanna Reitz                 QLIST_EMPTY(&bs->children))
24760bc329fbSHanna Reitz             {
24770bc329fbSHanna Reitz                 /*
24780bc329fbSHanna Reitz                  * When a protocol driver reports BLOCK_OFFSET_VALID, the
24790bc329fbSHanna Reitz                  * returned local_map value must be the same as the offset we
24800bc329fbSHanna Reitz                  * have passed (aligned_offset), and local_bs must be the node
24810bc329fbSHanna Reitz                  * itself.
24820bc329fbSHanna Reitz                  * Assert this, because we follow this rule when reading from
24830bc329fbSHanna Reitz                  * the cache (see the `local_file = bs` and
24840bc329fbSHanna Reitz                  * `local_map = aligned_offset` assignments above), and the
24850bc329fbSHanna Reitz                  * result the cache delivers must be the same as the driver
24860bc329fbSHanna Reitz                  * would deliver.
24870bc329fbSHanna Reitz                  */
24880bc329fbSHanna Reitz                 assert(local_file == bs);
24890bc329fbSHanna Reitz                 assert(local_map == aligned_offset);
24900bc329fbSHanna Reitz                 bdrv_bsc_fill(bs, aligned_offset, *pnum);
24910bc329fbSHanna Reitz             }
24920bc329fbSHanna Reitz         }
2493549ec0d9SMax Reitz     } else {
2494549ec0d9SMax Reitz         /* Default code for filters */
2495549ec0d9SMax Reitz 
2496549ec0d9SMax Reitz         local_file = bdrv_filter_bs(bs);
2497549ec0d9SMax Reitz         assert(local_file);
2498549ec0d9SMax Reitz 
2499549ec0d9SMax Reitz         *pnum = aligned_bytes;
2500549ec0d9SMax Reitz         local_map = aligned_offset;
2501549ec0d9SMax Reitz         ret = BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID;
2502549ec0d9SMax Reitz     }
250386a3d5c6SEric Blake     if (ret < 0) {
250486a3d5c6SEric Blake         *pnum = 0;
250586a3d5c6SEric Blake         goto out;
250686a3d5c6SEric Blake     }
2507efa6e2edSEric Blake 
2508efa6e2edSEric Blake     /*
2509636cb512SEric Blake      * The driver's result must be a non-zero multiple of request_alignment.
2510efa6e2edSEric Blake      * Clamp pnum and adjust map to original request.
2511efa6e2edSEric Blake      */
2512636cb512SEric Blake     assert(*pnum && QEMU_IS_ALIGNED(*pnum, align) &&
2513636cb512SEric Blake            align > offset - aligned_offset);
251469f47505SVladimir Sementsov-Ogievskiy     if (ret & BDRV_BLOCK_RECURSE) {
251569f47505SVladimir Sementsov-Ogievskiy         assert(ret & BDRV_BLOCK_DATA);
251669f47505SVladimir Sementsov-Ogievskiy         assert(ret & BDRV_BLOCK_OFFSET_VALID);
251769f47505SVladimir Sementsov-Ogievskiy         assert(!(ret & BDRV_BLOCK_ZERO));
251869f47505SVladimir Sementsov-Ogievskiy     }
251969f47505SVladimir Sementsov-Ogievskiy 
2520efa6e2edSEric Blake     *pnum -= offset - aligned_offset;
2521efa6e2edSEric Blake     if (*pnum > bytes) {
2522efa6e2edSEric Blake         *pnum = bytes;
2523efa6e2edSEric Blake     }
2524efa6e2edSEric Blake     if (ret & BDRV_BLOCK_OFFSET_VALID) {
2525efa6e2edSEric Blake         local_map += offset - aligned_offset;
2526efa6e2edSEric Blake     }
252761007b31SStefan Hajnoczi 
252861007b31SStefan Hajnoczi     if (ret & BDRV_BLOCK_RAW) {
2529298a1665SEric Blake         assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file);
25302e8bc787SEric Blake         ret = bdrv_co_block_status(local_file, want_zero, local_map,
25312e8bc787SEric Blake                                    *pnum, pnum, &local_map, &local_file);
253299723548SPaolo Bonzini         goto out;
253361007b31SStefan Hajnoczi     }
253461007b31SStefan Hajnoczi 
253561007b31SStefan Hajnoczi     if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
253661007b31SStefan Hajnoczi         ret |= BDRV_BLOCK_ALLOCATED;
2537d40f4a56SAlberto Garcia     } else if (bs->drv->supports_backing) {
2538cb850315SMax Reitz         BlockDriverState *cow_bs = bdrv_cow_bs(bs);
2539cb850315SMax Reitz 
2540d40f4a56SAlberto Garcia         if (!cow_bs) {
2541d40f4a56SAlberto Garcia             ret |= BDRV_BLOCK_ZERO;
2542d40f4a56SAlberto Garcia         } else if (want_zero) {
2543cb850315SMax Reitz             int64_t size2 = bdrv_getlength(cow_bs);
2544c9ce8c4dSEric Blake 
25452e8bc787SEric Blake             if (size2 >= 0 && offset >= size2) {
254661007b31SStefan Hajnoczi                 ret |= BDRV_BLOCK_ZERO;
254761007b31SStefan Hajnoczi             }
25487b1efe99SVladimir Sementsov-Ogievskiy         }
254961007b31SStefan Hajnoczi     }
255061007b31SStefan Hajnoczi 
255169f47505SVladimir Sementsov-Ogievskiy     if (want_zero && ret & BDRV_BLOCK_RECURSE &&
255269f47505SVladimir Sementsov-Ogievskiy         local_file && local_file != bs &&
255361007b31SStefan Hajnoczi         (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
255461007b31SStefan Hajnoczi         (ret & BDRV_BLOCK_OFFSET_VALID)) {
25552e8bc787SEric Blake         int64_t file_pnum;
25562e8bc787SEric Blake         int ret2;
255761007b31SStefan Hajnoczi 
25582e8bc787SEric Blake         ret2 = bdrv_co_block_status(local_file, want_zero, local_map,
25592e8bc787SEric Blake                                     *pnum, &file_pnum, NULL, NULL);
256061007b31SStefan Hajnoczi         if (ret2 >= 0) {
256161007b31SStefan Hajnoczi             /* Ignore errors.  This is just providing extra information, it
256261007b31SStefan Hajnoczi              * is useful but not necessary.
256361007b31SStefan Hajnoczi              */
2564c61e684eSEric Blake             if (ret2 & BDRV_BLOCK_EOF &&
2565c61e684eSEric Blake                 (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) {
2566c61e684eSEric Blake                 /*
2567c61e684eSEric Blake                  * It is valid for the format block driver to read
2568c61e684eSEric Blake                  * beyond the end of the underlying file's current
2569c61e684eSEric Blake                  * size; such areas read as zero.
2570c61e684eSEric Blake                  */
257161007b31SStefan Hajnoczi                 ret |= BDRV_BLOCK_ZERO;
257261007b31SStefan Hajnoczi             } else {
257361007b31SStefan Hajnoczi                 /* Limit request to the range reported by the protocol driver */
257461007b31SStefan Hajnoczi                 *pnum = file_pnum;
257561007b31SStefan Hajnoczi                 ret |= (ret2 & BDRV_BLOCK_ZERO);
257661007b31SStefan Hajnoczi             }
257761007b31SStefan Hajnoczi         }
257861007b31SStefan Hajnoczi     }
257961007b31SStefan Hajnoczi 
258099723548SPaolo Bonzini out:
258199723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
25822e8bc787SEric Blake     if (ret >= 0 && offset + *pnum == total_size) {
2583fb0d8654SEric Blake         ret |= BDRV_BLOCK_EOF;
2584fb0d8654SEric Blake     }
2585298a1665SEric Blake early_out:
2586298a1665SEric Blake     if (file) {
2587298a1665SEric Blake         *file = local_file;
2588298a1665SEric Blake     }
25892e8bc787SEric Blake     if (map) {
25902e8bc787SEric Blake         *map = local_map;
25912e8bc787SEric Blake     }
259261007b31SStefan Hajnoczi     return ret;
259361007b31SStefan Hajnoczi }
259461007b31SStefan Hajnoczi 
259521c2283eSVladimir Sementsov-Ogievskiy int coroutine_fn
2596f9e694cbSVladimir Sementsov-Ogievskiy bdrv_co_common_block_status_above(BlockDriverState *bs,
2597ba3f0e25SFam Zheng                                   BlockDriverState *base,
25983555a432SVladimir Sementsov-Ogievskiy                                   bool include_base,
2599c9ce8c4dSEric Blake                                   bool want_zero,
26005b648c67SEric Blake                                   int64_t offset,
26015b648c67SEric Blake                                   int64_t bytes,
26025b648c67SEric Blake                                   int64_t *pnum,
26035b648c67SEric Blake                                   int64_t *map,
2604a92b1b06SEric Blake                                   BlockDriverState **file,
2605a92b1b06SEric Blake                                   int *depth)
2606ba3f0e25SFam Zheng {
260767c095c8SVladimir Sementsov-Ogievskiy     int ret;
2608ba3f0e25SFam Zheng     BlockDriverState *p;
260967c095c8SVladimir Sementsov-Ogievskiy     int64_t eof = 0;
2610a92b1b06SEric Blake     int dummy;
26111581a70dSEmanuele Giuseppe Esposito     IO_CODE();
2612ba3f0e25SFam Zheng 
26133555a432SVladimir Sementsov-Ogievskiy     assert(!include_base || base); /* Can't include NULL base */
261467c095c8SVladimir Sementsov-Ogievskiy 
2615a92b1b06SEric Blake     if (!depth) {
2616a92b1b06SEric Blake         depth = &dummy;
2617a92b1b06SEric Blake     }
2618a92b1b06SEric Blake     *depth = 0;
2619a92b1b06SEric Blake 
2620624f27bbSVladimir Sementsov-Ogievskiy     if (!include_base && bs == base) {
2621624f27bbSVladimir Sementsov-Ogievskiy         *pnum = bytes;
2622624f27bbSVladimir Sementsov-Ogievskiy         return 0;
2623624f27bbSVladimir Sementsov-Ogievskiy     }
2624624f27bbSVladimir Sementsov-Ogievskiy 
262567c095c8SVladimir Sementsov-Ogievskiy     ret = bdrv_co_block_status(bs, want_zero, offset, bytes, pnum, map, file);
2626a92b1b06SEric Blake     ++*depth;
26273555a432SVladimir Sementsov-Ogievskiy     if (ret < 0 || *pnum == 0 || ret & BDRV_BLOCK_ALLOCATED || bs == base) {
262867c095c8SVladimir Sementsov-Ogievskiy         return ret;
262967c095c8SVladimir Sementsov-Ogievskiy     }
263067c095c8SVladimir Sementsov-Ogievskiy 
263167c095c8SVladimir Sementsov-Ogievskiy     if (ret & BDRV_BLOCK_EOF) {
263267c095c8SVladimir Sementsov-Ogievskiy         eof = offset + *pnum;
263367c095c8SVladimir Sementsov-Ogievskiy     }
263467c095c8SVladimir Sementsov-Ogievskiy 
263567c095c8SVladimir Sementsov-Ogievskiy     assert(*pnum <= bytes);
263667c095c8SVladimir Sementsov-Ogievskiy     bytes = *pnum;
263767c095c8SVladimir Sementsov-Ogievskiy 
26383555a432SVladimir Sementsov-Ogievskiy     for (p = bdrv_filter_or_cow_bs(bs); include_base || p != base;
263967c095c8SVladimir Sementsov-Ogievskiy          p = bdrv_filter_or_cow_bs(p))
264067c095c8SVladimir Sementsov-Ogievskiy     {
26415b648c67SEric Blake         ret = bdrv_co_block_status(p, want_zero, offset, bytes, pnum, map,
26425b648c67SEric Blake                                    file);
2643a92b1b06SEric Blake         ++*depth;
2644c61e684eSEric Blake         if (ret < 0) {
264567c095c8SVladimir Sementsov-Ogievskiy             return ret;
2646c61e684eSEric Blake         }
264767c095c8SVladimir Sementsov-Ogievskiy         if (*pnum == 0) {
2648c61e684eSEric Blake             /*
264967c095c8SVladimir Sementsov-Ogievskiy              * The top layer deferred to this layer, and because this layer is
265067c095c8SVladimir Sementsov-Ogievskiy              * short, any zeroes that we synthesize beyond EOF behave as if they
265167c095c8SVladimir Sementsov-Ogievskiy              * were allocated at this layer.
265267c095c8SVladimir Sementsov-Ogievskiy              *
265367c095c8SVladimir Sementsov-Ogievskiy              * We don't include BDRV_BLOCK_EOF into ret, as upper layer may be
265467c095c8SVladimir Sementsov-Ogievskiy              * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see
265567c095c8SVladimir Sementsov-Ogievskiy              * below.
2656c61e684eSEric Blake              */
265767c095c8SVladimir Sementsov-Ogievskiy             assert(ret & BDRV_BLOCK_EOF);
26585b648c67SEric Blake             *pnum = bytes;
265967c095c8SVladimir Sementsov-Ogievskiy             if (file) {
266067c095c8SVladimir Sementsov-Ogievskiy                 *file = p;
2661c61e684eSEric Blake             }
266267c095c8SVladimir Sementsov-Ogievskiy             ret = BDRV_BLOCK_ZERO | BDRV_BLOCK_ALLOCATED;
2663ba3f0e25SFam Zheng             break;
2664ba3f0e25SFam Zheng         }
266567c095c8SVladimir Sementsov-Ogievskiy         if (ret & BDRV_BLOCK_ALLOCATED) {
266667c095c8SVladimir Sementsov-Ogievskiy             /*
266767c095c8SVladimir Sementsov-Ogievskiy              * We've found the node and the status, we must break.
266867c095c8SVladimir Sementsov-Ogievskiy              *
266967c095c8SVladimir Sementsov-Ogievskiy              * Drop BDRV_BLOCK_EOF, as it's not for upper layer, which may be
267067c095c8SVladimir Sementsov-Ogievskiy              * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see
267167c095c8SVladimir Sementsov-Ogievskiy              * below.
267267c095c8SVladimir Sementsov-Ogievskiy              */
267367c095c8SVladimir Sementsov-Ogievskiy             ret &= ~BDRV_BLOCK_EOF;
267467c095c8SVladimir Sementsov-Ogievskiy             break;
2675ba3f0e25SFam Zheng         }
267667c095c8SVladimir Sementsov-Ogievskiy 
26773555a432SVladimir Sementsov-Ogievskiy         if (p == base) {
26783555a432SVladimir Sementsov-Ogievskiy             assert(include_base);
26793555a432SVladimir Sementsov-Ogievskiy             break;
26803555a432SVladimir Sementsov-Ogievskiy         }
26813555a432SVladimir Sementsov-Ogievskiy 
268267c095c8SVladimir Sementsov-Ogievskiy         /*
268367c095c8SVladimir Sementsov-Ogievskiy          * OK, [offset, offset + *pnum) region is unallocated on this layer,
268467c095c8SVladimir Sementsov-Ogievskiy          * let's continue the diving.
268567c095c8SVladimir Sementsov-Ogievskiy          */
268667c095c8SVladimir Sementsov-Ogievskiy         assert(*pnum <= bytes);
268767c095c8SVladimir Sementsov-Ogievskiy         bytes = *pnum;
268867c095c8SVladimir Sementsov-Ogievskiy     }
268967c095c8SVladimir Sementsov-Ogievskiy 
269067c095c8SVladimir Sementsov-Ogievskiy     if (offset + *pnum == eof) {
269167c095c8SVladimir Sementsov-Ogievskiy         ret |= BDRV_BLOCK_EOF;
269267c095c8SVladimir Sementsov-Ogievskiy     }
269367c095c8SVladimir Sementsov-Ogievskiy 
2694ba3f0e25SFam Zheng     return ret;
2695ba3f0e25SFam Zheng }
2696ba3f0e25SFam Zheng 
269731826642SEric Blake int bdrv_block_status_above(BlockDriverState *bs, BlockDriverState *base,
269831826642SEric Blake                             int64_t offset, int64_t bytes, int64_t *pnum,
269931826642SEric Blake                             int64_t *map, BlockDriverState **file)
2700c9ce8c4dSEric Blake {
2701384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
27023555a432SVladimir Sementsov-Ogievskiy     return bdrv_common_block_status_above(bs, base, false, true, offset, bytes,
2703a92b1b06SEric Blake                                           pnum, map, file, NULL);
2704c9ce8c4dSEric Blake }
2705c9ce8c4dSEric Blake 
2706237d78f8SEric Blake int bdrv_block_status(BlockDriverState *bs, int64_t offset, int64_t bytes,
2707237d78f8SEric Blake                       int64_t *pnum, int64_t *map, BlockDriverState **file)
2708ba3f0e25SFam Zheng {
2709384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
2710cb850315SMax Reitz     return bdrv_block_status_above(bs, bdrv_filter_or_cow_bs(bs),
271131826642SEric Blake                                    offset, bytes, pnum, map, file);
2712ba3f0e25SFam Zheng }
2713ba3f0e25SFam Zheng 
271446cd1e8aSAlberto Garcia /*
271546cd1e8aSAlberto Garcia  * Check @bs (and its backing chain) to see if the range defined
271646cd1e8aSAlberto Garcia  * by @offset and @bytes is known to read as zeroes.
271746cd1e8aSAlberto Garcia  * Return 1 if that is the case, 0 otherwise and -errno on error.
271846cd1e8aSAlberto Garcia  * This test is meant to be fast rather than accurate so returning 0
271946cd1e8aSAlberto Garcia  * does not guarantee non-zero data.
272046cd1e8aSAlberto Garcia  */
272146cd1e8aSAlberto Garcia int coroutine_fn bdrv_co_is_zero_fast(BlockDriverState *bs, int64_t offset,
272246cd1e8aSAlberto Garcia                                       int64_t bytes)
272346cd1e8aSAlberto Garcia {
272446cd1e8aSAlberto Garcia     int ret;
272546cd1e8aSAlberto Garcia     int64_t pnum = bytes;
2726384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
272746cd1e8aSAlberto Garcia 
272846cd1e8aSAlberto Garcia     if (!bytes) {
272946cd1e8aSAlberto Garcia         return 1;
273046cd1e8aSAlberto Garcia     }
273146cd1e8aSAlberto Garcia 
273246cd1e8aSAlberto Garcia     ret = bdrv_common_block_status_above(bs, NULL, false, false, offset,
2733a92b1b06SEric Blake                                          bytes, &pnum, NULL, NULL, NULL);
273446cd1e8aSAlberto Garcia 
273546cd1e8aSAlberto Garcia     if (ret < 0) {
273646cd1e8aSAlberto Garcia         return ret;
273746cd1e8aSAlberto Garcia     }
273846cd1e8aSAlberto Garcia 
273946cd1e8aSAlberto Garcia     return (pnum == bytes) && (ret & BDRV_BLOCK_ZERO);
274046cd1e8aSAlberto Garcia }
274146cd1e8aSAlberto Garcia 
2742d6a644bbSEric Blake int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t offset,
2743d6a644bbSEric Blake                                    int64_t bytes, int64_t *pnum)
274461007b31SStefan Hajnoczi {
27457ddb99b9SEric Blake     int ret;
27467ddb99b9SEric Blake     int64_t dummy;
2747384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
2748d6a644bbSEric Blake 
27493555a432SVladimir Sementsov-Ogievskiy     ret = bdrv_common_block_status_above(bs, bs, true, false, offset,
27503555a432SVladimir Sementsov-Ogievskiy                                          bytes, pnum ? pnum : &dummy, NULL,
2751a92b1b06SEric Blake                                          NULL, NULL);
275261007b31SStefan Hajnoczi     if (ret < 0) {
275361007b31SStefan Hajnoczi         return ret;
275461007b31SStefan Hajnoczi     }
275561007b31SStefan Hajnoczi     return !!(ret & BDRV_BLOCK_ALLOCATED);
275661007b31SStefan Hajnoczi }
275761007b31SStefan Hajnoczi 
275861007b31SStefan Hajnoczi /*
275961007b31SStefan Hajnoczi  * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
276061007b31SStefan Hajnoczi  *
2761a92b1b06SEric Blake  * Return a positive depth if (a prefix of) the given range is allocated
2762a92b1b06SEric Blake  * in any image between BASE and TOP (BASE is only included if include_base
2763a92b1b06SEric Blake  * is set).  Depth 1 is TOP, 2 is the first backing layer, and so forth.
2764170d3bd3SAndrey Shinkevich  * BASE can be NULL to check if the given offset is allocated in any
2765170d3bd3SAndrey Shinkevich  * image of the chain.  Return 0 otherwise, or negative errno on
2766170d3bd3SAndrey Shinkevich  * failure.
276761007b31SStefan Hajnoczi  *
276851b0a488SEric Blake  * 'pnum' is set to the number of bytes (including and immediately
276951b0a488SEric Blake  * following the specified offset) that are known to be in the same
277051b0a488SEric Blake  * allocated/unallocated state.  Note that a subsequent call starting
277151b0a488SEric Blake  * at 'offset + *pnum' may return the same allocation status (in other
277251b0a488SEric Blake  * words, the result is not necessarily the maximum possible range);
277351b0a488SEric Blake  * but 'pnum' will only be 0 when end of file is reached.
277461007b31SStefan Hajnoczi  */
277561007b31SStefan Hajnoczi int bdrv_is_allocated_above(BlockDriverState *top,
277661007b31SStefan Hajnoczi                             BlockDriverState *base,
2777170d3bd3SAndrey Shinkevich                             bool include_base, int64_t offset,
2778170d3bd3SAndrey Shinkevich                             int64_t bytes, int64_t *pnum)
277961007b31SStefan Hajnoczi {
2780a92b1b06SEric Blake     int depth;
27817e7e5100SVladimir Sementsov-Ogievskiy     int ret = bdrv_common_block_status_above(top, base, include_base, false,
2782a92b1b06SEric Blake                                              offset, bytes, pnum, NULL, NULL,
2783a92b1b06SEric Blake                                              &depth);
2784384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
278561007b31SStefan Hajnoczi     if (ret < 0) {
278661007b31SStefan Hajnoczi         return ret;
2787d6a644bbSEric Blake     }
278861007b31SStefan Hajnoczi 
2789a92b1b06SEric Blake     if (ret & BDRV_BLOCK_ALLOCATED) {
2790a92b1b06SEric Blake         return depth;
2791a92b1b06SEric Blake     }
2792a92b1b06SEric Blake     return 0;
279361007b31SStefan Hajnoczi }
279461007b31SStefan Hajnoczi 
279521c2283eSVladimir Sementsov-Ogievskiy int coroutine_fn
2796b33b354fSVladimir Sementsov-Ogievskiy bdrv_co_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
27971a8ae822SKevin Wolf {
27981a8ae822SKevin Wolf     BlockDriver *drv = bs->drv;
2799c4db2e25SMax Reitz     BlockDriverState *child_bs = bdrv_primary_bs(bs);
2800b984b296SVladimir Sementsov-Ogievskiy     int ret;
28011581a70dSEmanuele Giuseppe Esposito     IO_CODE();
2802b984b296SVladimir Sementsov-Ogievskiy 
2803b984b296SVladimir Sementsov-Ogievskiy     ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL);
2804b984b296SVladimir Sementsov-Ogievskiy     if (ret < 0) {
2805b984b296SVladimir Sementsov-Ogievskiy         return ret;
2806b984b296SVladimir Sementsov-Ogievskiy     }
2807dc88a467SStefan Hajnoczi 
2808b33b354fSVladimir Sementsov-Ogievskiy     if (!drv) {
2809b33b354fSVladimir Sementsov-Ogievskiy         return -ENOMEDIUM;
2810b33b354fSVladimir Sementsov-Ogievskiy     }
2811b33b354fSVladimir Sementsov-Ogievskiy 
2812dc88a467SStefan Hajnoczi     bdrv_inc_in_flight(bs);
28131a8ae822SKevin Wolf 
2814b33b354fSVladimir Sementsov-Ogievskiy     if (drv->bdrv_load_vmstate) {
2815dc88a467SStefan Hajnoczi         ret = drv->bdrv_load_vmstate(bs, qiov, pos);
2816c4db2e25SMax Reitz     } else if (child_bs) {
2817b33b354fSVladimir Sementsov-Ogievskiy         ret = bdrv_co_readv_vmstate(child_bs, qiov, pos);
2818b984b296SVladimir Sementsov-Ogievskiy     } else {
2819b984b296SVladimir Sementsov-Ogievskiy         ret = -ENOTSUP;
28201a8ae822SKevin Wolf     }
28211a8ae822SKevin Wolf 
2822dc88a467SStefan Hajnoczi     bdrv_dec_in_flight(bs);
2823b33b354fSVladimir Sementsov-Ogievskiy 
2824b33b354fSVladimir Sementsov-Ogievskiy     return ret;
2825b33b354fSVladimir Sementsov-Ogievskiy }
2826b33b354fSVladimir Sementsov-Ogievskiy 
2827b33b354fSVladimir Sementsov-Ogievskiy int coroutine_fn
2828b33b354fSVladimir Sementsov-Ogievskiy bdrv_co_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
2829b33b354fSVladimir Sementsov-Ogievskiy {
2830b33b354fSVladimir Sementsov-Ogievskiy     BlockDriver *drv = bs->drv;
2831b33b354fSVladimir Sementsov-Ogievskiy     BlockDriverState *child_bs = bdrv_primary_bs(bs);
2832b984b296SVladimir Sementsov-Ogievskiy     int ret;
28331581a70dSEmanuele Giuseppe Esposito     IO_CODE();
2834b984b296SVladimir Sementsov-Ogievskiy 
2835b984b296SVladimir Sementsov-Ogievskiy     ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL);
2836b984b296SVladimir Sementsov-Ogievskiy     if (ret < 0) {
2837b984b296SVladimir Sementsov-Ogievskiy         return ret;
2838b984b296SVladimir Sementsov-Ogievskiy     }
2839b33b354fSVladimir Sementsov-Ogievskiy 
2840b33b354fSVladimir Sementsov-Ogievskiy     if (!drv) {
2841b33b354fSVladimir Sementsov-Ogievskiy         return -ENOMEDIUM;
2842b33b354fSVladimir Sementsov-Ogievskiy     }
2843b33b354fSVladimir Sementsov-Ogievskiy 
2844b33b354fSVladimir Sementsov-Ogievskiy     bdrv_inc_in_flight(bs);
2845b33b354fSVladimir Sementsov-Ogievskiy 
2846b33b354fSVladimir Sementsov-Ogievskiy     if (drv->bdrv_save_vmstate) {
2847b33b354fSVladimir Sementsov-Ogievskiy         ret = drv->bdrv_save_vmstate(bs, qiov, pos);
2848b33b354fSVladimir Sementsov-Ogievskiy     } else if (child_bs) {
2849b33b354fSVladimir Sementsov-Ogievskiy         ret = bdrv_co_writev_vmstate(child_bs, qiov, pos);
2850b984b296SVladimir Sementsov-Ogievskiy     } else {
2851b984b296SVladimir Sementsov-Ogievskiy         ret = -ENOTSUP;
2852b33b354fSVladimir Sementsov-Ogievskiy     }
2853b33b354fSVladimir Sementsov-Ogievskiy 
2854b33b354fSVladimir Sementsov-Ogievskiy     bdrv_dec_in_flight(bs);
2855b33b354fSVladimir Sementsov-Ogievskiy 
2856dc88a467SStefan Hajnoczi     return ret;
28571a8ae822SKevin Wolf }
28581a8ae822SKevin Wolf 
285961007b31SStefan Hajnoczi int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
286061007b31SStefan Hajnoczi                       int64_t pos, int size)
286161007b31SStefan Hajnoczi {
28620d93ed08SVladimir Sementsov-Ogievskiy     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size);
2863b33b354fSVladimir Sementsov-Ogievskiy     int ret = bdrv_writev_vmstate(bs, &qiov, pos);
2864384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
286561007b31SStefan Hajnoczi 
2866b33b354fSVladimir Sementsov-Ogievskiy     return ret < 0 ? ret : size;
286761007b31SStefan Hajnoczi }
286861007b31SStefan Hajnoczi 
286961007b31SStefan Hajnoczi int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
287061007b31SStefan Hajnoczi                       int64_t pos, int size)
287161007b31SStefan Hajnoczi {
28720d93ed08SVladimir Sementsov-Ogievskiy     QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size);
2873b33b354fSVladimir Sementsov-Ogievskiy     int ret = bdrv_readv_vmstate(bs, &qiov, pos);
2874384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
28755ddda0b8SKevin Wolf 
2876b33b354fSVladimir Sementsov-Ogievskiy     return ret < 0 ? ret : size;
287761007b31SStefan Hajnoczi }
287861007b31SStefan Hajnoczi 
287961007b31SStefan Hajnoczi /**************************************************************/
288061007b31SStefan Hajnoczi /* async I/Os */
288161007b31SStefan Hajnoczi 
288261007b31SStefan Hajnoczi void bdrv_aio_cancel(BlockAIOCB *acb)
288361007b31SStefan Hajnoczi {
2884384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
288561007b31SStefan Hajnoczi     qemu_aio_ref(acb);
288661007b31SStefan Hajnoczi     bdrv_aio_cancel_async(acb);
288761007b31SStefan Hajnoczi     while (acb->refcnt > 1) {
288861007b31SStefan Hajnoczi         if (acb->aiocb_info->get_aio_context) {
288961007b31SStefan Hajnoczi             aio_poll(acb->aiocb_info->get_aio_context(acb), true);
289061007b31SStefan Hajnoczi         } else if (acb->bs) {
28912f47da5fSPaolo Bonzini             /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so
28922f47da5fSPaolo Bonzini              * assert that we're not using an I/O thread.  Thread-safe
28932f47da5fSPaolo Bonzini              * code should use bdrv_aio_cancel_async exclusively.
28942f47da5fSPaolo Bonzini              */
28952f47da5fSPaolo Bonzini             assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context());
289661007b31SStefan Hajnoczi             aio_poll(bdrv_get_aio_context(acb->bs), true);
289761007b31SStefan Hajnoczi         } else {
289861007b31SStefan Hajnoczi             abort();
289961007b31SStefan Hajnoczi         }
290061007b31SStefan Hajnoczi     }
290161007b31SStefan Hajnoczi     qemu_aio_unref(acb);
290261007b31SStefan Hajnoczi }
290361007b31SStefan Hajnoczi 
290461007b31SStefan Hajnoczi /* Async version of aio cancel. The caller is not blocked if the acb implements
290561007b31SStefan Hajnoczi  * cancel_async, otherwise we do nothing and let the request normally complete.
290661007b31SStefan Hajnoczi  * In either case the completion callback must be called. */
290761007b31SStefan Hajnoczi void bdrv_aio_cancel_async(BlockAIOCB *acb)
290861007b31SStefan Hajnoczi {
2909384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
291061007b31SStefan Hajnoczi     if (acb->aiocb_info->cancel_async) {
291161007b31SStefan Hajnoczi         acb->aiocb_info->cancel_async(acb);
291261007b31SStefan Hajnoczi     }
291361007b31SStefan Hajnoczi }
291461007b31SStefan Hajnoczi 
291561007b31SStefan Hajnoczi /**************************************************************/
291661007b31SStefan Hajnoczi /* Coroutine block device emulation */
291761007b31SStefan Hajnoczi 
291861007b31SStefan Hajnoczi int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
291961007b31SStefan Hajnoczi {
2920883833e2SMax Reitz     BdrvChild *primary_child = bdrv_primary_child(bs);
2921883833e2SMax Reitz     BdrvChild *child;
292249ca6259SFam Zheng     int current_gen;
292349ca6259SFam Zheng     int ret = 0;
2924384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
292561007b31SStefan Hajnoczi 
292699723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
2927c32b82afSPavel Dovgalyuk 
2928e914404eSFam Zheng     if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
292949ca6259SFam Zheng         bdrv_is_sg(bs)) {
293049ca6259SFam Zheng         goto early_exit;
293149ca6259SFam Zheng     }
293249ca6259SFam Zheng 
29333783fa3dSPaolo Bonzini     qemu_co_mutex_lock(&bs->reqs_lock);
2934d73415a3SStefan Hajnoczi     current_gen = qatomic_read(&bs->write_gen);
29353ff2f67aSEvgeny Yakovlev 
29363ff2f67aSEvgeny Yakovlev     /* Wait until any previous flushes are completed */
293799723548SPaolo Bonzini     while (bs->active_flush_req) {
29383783fa3dSPaolo Bonzini         qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock);
29393ff2f67aSEvgeny Yakovlev     }
29403ff2f67aSEvgeny Yakovlev 
29413783fa3dSPaolo Bonzini     /* Flushes reach this point in nondecreasing current_gen order.  */
294299723548SPaolo Bonzini     bs->active_flush_req = true;
29433783fa3dSPaolo Bonzini     qemu_co_mutex_unlock(&bs->reqs_lock);
29443ff2f67aSEvgeny Yakovlev 
2945c32b82afSPavel Dovgalyuk     /* Write back all layers by calling one driver function */
2946c32b82afSPavel Dovgalyuk     if (bs->drv->bdrv_co_flush) {
2947c32b82afSPavel Dovgalyuk         ret = bs->drv->bdrv_co_flush(bs);
2948c32b82afSPavel Dovgalyuk         goto out;
2949c32b82afSPavel Dovgalyuk     }
2950c32b82afSPavel Dovgalyuk 
295161007b31SStefan Hajnoczi     /* Write back cached data to the OS even with cache=unsafe */
2952883833e2SMax Reitz     BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_OS);
295361007b31SStefan Hajnoczi     if (bs->drv->bdrv_co_flush_to_os) {
295461007b31SStefan Hajnoczi         ret = bs->drv->bdrv_co_flush_to_os(bs);
295561007b31SStefan Hajnoczi         if (ret < 0) {
2956cdb5e315SFam Zheng             goto out;
295761007b31SStefan Hajnoczi         }
295861007b31SStefan Hajnoczi     }
295961007b31SStefan Hajnoczi 
296061007b31SStefan Hajnoczi     /* But don't actually force it to the disk with cache=unsafe */
296161007b31SStefan Hajnoczi     if (bs->open_flags & BDRV_O_NO_FLUSH) {
2962883833e2SMax Reitz         goto flush_children;
296361007b31SStefan Hajnoczi     }
296461007b31SStefan Hajnoczi 
29653ff2f67aSEvgeny Yakovlev     /* Check if we really need to flush anything */
29663ff2f67aSEvgeny Yakovlev     if (bs->flushed_gen == current_gen) {
2967883833e2SMax Reitz         goto flush_children;
29683ff2f67aSEvgeny Yakovlev     }
29693ff2f67aSEvgeny Yakovlev 
2970883833e2SMax Reitz     BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_DISK);
2971d470ad42SMax Reitz     if (!bs->drv) {
2972d470ad42SMax Reitz         /* bs->drv->bdrv_co_flush() might have ejected the BDS
2973d470ad42SMax Reitz          * (even in case of apparent success) */
2974d470ad42SMax Reitz         ret = -ENOMEDIUM;
2975d470ad42SMax Reitz         goto out;
2976d470ad42SMax Reitz     }
297761007b31SStefan Hajnoczi     if (bs->drv->bdrv_co_flush_to_disk) {
297861007b31SStefan Hajnoczi         ret = bs->drv->bdrv_co_flush_to_disk(bs);
297961007b31SStefan Hajnoczi     } else if (bs->drv->bdrv_aio_flush) {
298061007b31SStefan Hajnoczi         BlockAIOCB *acb;
298161007b31SStefan Hajnoczi         CoroutineIOCompletion co = {
298261007b31SStefan Hajnoczi             .coroutine = qemu_coroutine_self(),
298361007b31SStefan Hajnoczi         };
298461007b31SStefan Hajnoczi 
298561007b31SStefan Hajnoczi         acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
298661007b31SStefan Hajnoczi         if (acb == NULL) {
298761007b31SStefan Hajnoczi             ret = -EIO;
298861007b31SStefan Hajnoczi         } else {
298961007b31SStefan Hajnoczi             qemu_coroutine_yield();
299061007b31SStefan Hajnoczi             ret = co.ret;
299161007b31SStefan Hajnoczi         }
299261007b31SStefan Hajnoczi     } else {
299361007b31SStefan Hajnoczi         /*
299461007b31SStefan Hajnoczi          * Some block drivers always operate in either writethrough or unsafe
299561007b31SStefan Hajnoczi          * mode and don't support bdrv_flush therefore. Usually qemu doesn't
299661007b31SStefan Hajnoczi          * know how the server works (because the behaviour is hardcoded or
299761007b31SStefan Hajnoczi          * depends on server-side configuration), so we can't ensure that
299861007b31SStefan Hajnoczi          * everything is safe on disk. Returning an error doesn't work because
299961007b31SStefan Hajnoczi          * that would break guests even if the server operates in writethrough
300061007b31SStefan Hajnoczi          * mode.
300161007b31SStefan Hajnoczi          *
300261007b31SStefan Hajnoczi          * Let's hope the user knows what he's doing.
300361007b31SStefan Hajnoczi          */
300461007b31SStefan Hajnoczi         ret = 0;
300561007b31SStefan Hajnoczi     }
30063ff2f67aSEvgeny Yakovlev 
300761007b31SStefan Hajnoczi     if (ret < 0) {
3008cdb5e315SFam Zheng         goto out;
300961007b31SStefan Hajnoczi     }
301061007b31SStefan Hajnoczi 
301161007b31SStefan Hajnoczi     /* Now flush the underlying protocol.  It will also have BDRV_O_NO_FLUSH
301261007b31SStefan Hajnoczi      * in the case of cache=unsafe, so there are no useless flushes.
301361007b31SStefan Hajnoczi      */
3014883833e2SMax Reitz flush_children:
3015883833e2SMax Reitz     ret = 0;
3016883833e2SMax Reitz     QLIST_FOREACH(child, &bs->children, next) {
3017883833e2SMax Reitz         if (child->perm & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED)) {
3018883833e2SMax Reitz             int this_child_ret = bdrv_co_flush(child->bs);
3019883833e2SMax Reitz             if (!ret) {
3020883833e2SMax Reitz                 ret = this_child_ret;
3021883833e2SMax Reitz             }
3022883833e2SMax Reitz         }
3023883833e2SMax Reitz     }
3024883833e2SMax Reitz 
3025cdb5e315SFam Zheng out:
30263ff2f67aSEvgeny Yakovlev     /* Notify any pending flushes that we have completed */
3027e6af1e08SKevin Wolf     if (ret == 0) {
30283ff2f67aSEvgeny Yakovlev         bs->flushed_gen = current_gen;
3029e6af1e08SKevin Wolf     }
30303783fa3dSPaolo Bonzini 
30313783fa3dSPaolo Bonzini     qemu_co_mutex_lock(&bs->reqs_lock);
303299723548SPaolo Bonzini     bs->active_flush_req = false;
3033156af3acSDenis V. Lunev     /* Return value is ignored - it's ok if wait queue is empty */
3034156af3acSDenis V. Lunev     qemu_co_queue_next(&bs->flush_queue);
30353783fa3dSPaolo Bonzini     qemu_co_mutex_unlock(&bs->reqs_lock);
30363ff2f67aSEvgeny Yakovlev 
303749ca6259SFam Zheng early_exit:
303899723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
3039cdb5e315SFam Zheng     return ret;
304061007b31SStefan Hajnoczi }
304161007b31SStefan Hajnoczi 
3042d93e5726SVladimir Sementsov-Ogievskiy int coroutine_fn bdrv_co_pdiscard(BdrvChild *child, int64_t offset,
3043d93e5726SVladimir Sementsov-Ogievskiy                                   int64_t bytes)
304461007b31SStefan Hajnoczi {
3045b1066c87SFam Zheng     BdrvTrackedRequest req;
304639af49c0SVladimir Sementsov-Ogievskiy     int ret;
304739af49c0SVladimir Sementsov-Ogievskiy     int64_t max_pdiscard;
30483482b9bcSEric Blake     int head, tail, align;
30490b9fd3f4SFam Zheng     BlockDriverState *bs = child->bs;
3050384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
305161007b31SStefan Hajnoczi 
3052d93e5726SVladimir Sementsov-Ogievskiy     if (!bs || !bs->drv || !bdrv_is_inserted(bs)) {
305361007b31SStefan Hajnoczi         return -ENOMEDIUM;
305461007b31SStefan Hajnoczi     }
305561007b31SStefan Hajnoczi 
3056d6883bc9SVladimir Sementsov-Ogievskiy     if (bdrv_has_readonly_bitmaps(bs)) {
3057d6883bc9SVladimir Sementsov-Ogievskiy         return -EPERM;
3058d6883bc9SVladimir Sementsov-Ogievskiy     }
3059d6883bc9SVladimir Sementsov-Ogievskiy 
306069b55e03SVladimir Sementsov-Ogievskiy     ret = bdrv_check_request(offset, bytes, NULL);
30618b117001SVladimir Sementsov-Ogievskiy     if (ret < 0) {
30628b117001SVladimir Sementsov-Ogievskiy         return ret;
306361007b31SStefan Hajnoczi     }
306461007b31SStefan Hajnoczi 
306561007b31SStefan Hajnoczi     /* Do nothing if disabled.  */
306661007b31SStefan Hajnoczi     if (!(bs->open_flags & BDRV_O_UNMAP)) {
306761007b31SStefan Hajnoczi         return 0;
306861007b31SStefan Hajnoczi     }
306961007b31SStefan Hajnoczi 
307002aefe43SEric Blake     if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) {
307161007b31SStefan Hajnoczi         return 0;
307261007b31SStefan Hajnoczi     }
307361007b31SStefan Hajnoczi 
30740bc329fbSHanna Reitz     /* Invalidate the cached block-status data range if this discard overlaps */
30750bc329fbSHanna Reitz     bdrv_bsc_invalidate_range(bs, offset, bytes);
30760bc329fbSHanna Reitz 
30773482b9bcSEric Blake     /* Discard is advisory, but some devices track and coalesce
30783482b9bcSEric Blake      * unaligned requests, so we must pass everything down rather than
30793482b9bcSEric Blake      * round here.  Still, most devices will just silently ignore
30803482b9bcSEric Blake      * unaligned requests (by returning -ENOTSUP), so we must fragment
30813482b9bcSEric Blake      * the request accordingly.  */
308202aefe43SEric Blake     align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment);
3083b8d0a980SEric Blake     assert(align % bs->bl.request_alignment == 0);
3084b8d0a980SEric Blake     head = offset % align;
3085f5a5ca79SManos Pitsidianakis     tail = (offset + bytes) % align;
30869f1963b3SEric Blake 
308799723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
3088f5a5ca79SManos Pitsidianakis     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD);
308950824995SFam Zheng 
309000695c27SFam Zheng     ret = bdrv_co_write_req_prepare(child, offset, bytes, &req, 0);
3091ec050f77SDenis V. Lunev     if (ret < 0) {
3092ec050f77SDenis V. Lunev         goto out;
3093ec050f77SDenis V. Lunev     }
3094ec050f77SDenis V. Lunev 
30956a8f3dbbSVladimir Sementsov-Ogievskiy     max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT64_MAX),
30969f1963b3SEric Blake                                    align);
30973482b9bcSEric Blake     assert(max_pdiscard >= bs->bl.request_alignment);
30989f1963b3SEric Blake 
3099f5a5ca79SManos Pitsidianakis     while (bytes > 0) {
3100d93e5726SVladimir Sementsov-Ogievskiy         int64_t num = bytes;
31013482b9bcSEric Blake 
31023482b9bcSEric Blake         if (head) {
31033482b9bcSEric Blake             /* Make small requests to get to alignment boundaries. */
3104f5a5ca79SManos Pitsidianakis             num = MIN(bytes, align - head);
31053482b9bcSEric Blake             if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) {
31063482b9bcSEric Blake                 num %= bs->bl.request_alignment;
31073482b9bcSEric Blake             }
31083482b9bcSEric Blake             head = (head + num) % align;
31093482b9bcSEric Blake             assert(num < max_pdiscard);
31103482b9bcSEric Blake         } else if (tail) {
31113482b9bcSEric Blake             if (num > align) {
31123482b9bcSEric Blake                 /* Shorten the request to the last aligned cluster.  */
31133482b9bcSEric Blake                 num -= tail;
31143482b9bcSEric Blake             } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) &&
31153482b9bcSEric Blake                        tail > bs->bl.request_alignment) {
31163482b9bcSEric Blake                 tail %= bs->bl.request_alignment;
31173482b9bcSEric Blake                 num -= tail;
31183482b9bcSEric Blake             }
31193482b9bcSEric Blake         }
31203482b9bcSEric Blake         /* limit request size */
31213482b9bcSEric Blake         if (num > max_pdiscard) {
31223482b9bcSEric Blake             num = max_pdiscard;
31233482b9bcSEric Blake         }
312461007b31SStefan Hajnoczi 
3125d470ad42SMax Reitz         if (!bs->drv) {
3126d470ad42SMax Reitz             ret = -ENOMEDIUM;
3127d470ad42SMax Reitz             goto out;
3128d470ad42SMax Reitz         }
312947a5486dSEric Blake         if (bs->drv->bdrv_co_pdiscard) {
313047a5486dSEric Blake             ret = bs->drv->bdrv_co_pdiscard(bs, offset, num);
313161007b31SStefan Hajnoczi         } else {
313261007b31SStefan Hajnoczi             BlockAIOCB *acb;
313361007b31SStefan Hajnoczi             CoroutineIOCompletion co = {
313461007b31SStefan Hajnoczi                 .coroutine = qemu_coroutine_self(),
313561007b31SStefan Hajnoczi             };
313661007b31SStefan Hajnoczi 
31374da444a0SEric Blake             acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num,
313861007b31SStefan Hajnoczi                                              bdrv_co_io_em_complete, &co);
313961007b31SStefan Hajnoczi             if (acb == NULL) {
3140b1066c87SFam Zheng                 ret = -EIO;
3141b1066c87SFam Zheng                 goto out;
314261007b31SStefan Hajnoczi             } else {
314361007b31SStefan Hajnoczi                 qemu_coroutine_yield();
314461007b31SStefan Hajnoczi                 ret = co.ret;
314561007b31SStefan Hajnoczi             }
314661007b31SStefan Hajnoczi         }
314761007b31SStefan Hajnoczi         if (ret && ret != -ENOTSUP) {
3148b1066c87SFam Zheng             goto out;
314961007b31SStefan Hajnoczi         }
315061007b31SStefan Hajnoczi 
31519f1963b3SEric Blake         offset += num;
3152f5a5ca79SManos Pitsidianakis         bytes -= num;
315361007b31SStefan Hajnoczi     }
3154b1066c87SFam Zheng     ret = 0;
3155b1066c87SFam Zheng out:
315600695c27SFam Zheng     bdrv_co_write_req_finish(child, req.offset, req.bytes, &req, ret);
3157b1066c87SFam Zheng     tracked_request_end(&req);
315899723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
3159b1066c87SFam Zheng     return ret;
316061007b31SStefan Hajnoczi }
316161007b31SStefan Hajnoczi 
3162881a4c55SPaolo Bonzini int coroutine_fn bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf)
316361007b31SStefan Hajnoczi {
316461007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
31655c5ae76aSFam Zheng     CoroutineIOCompletion co = {
31665c5ae76aSFam Zheng         .coroutine = qemu_coroutine_self(),
31675c5ae76aSFam Zheng     };
31685c5ae76aSFam Zheng     BlockAIOCB *acb;
3169384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
317061007b31SStefan Hajnoczi 
317199723548SPaolo Bonzini     bdrv_inc_in_flight(bs);
317216a389dcSKevin Wolf     if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) {
31735c5ae76aSFam Zheng         co.ret = -ENOTSUP;
31745c5ae76aSFam Zheng         goto out;
31755c5ae76aSFam Zheng     }
31765c5ae76aSFam Zheng 
317716a389dcSKevin Wolf     if (drv->bdrv_co_ioctl) {
317816a389dcSKevin Wolf         co.ret = drv->bdrv_co_ioctl(bs, req, buf);
317916a389dcSKevin Wolf     } else {
31805c5ae76aSFam Zheng         acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
31815c5ae76aSFam Zheng         if (!acb) {
3182c8a9fd80SFam Zheng             co.ret = -ENOTSUP;
3183c8a9fd80SFam Zheng             goto out;
31845c5ae76aSFam Zheng         }
31855c5ae76aSFam Zheng         qemu_coroutine_yield();
318616a389dcSKevin Wolf     }
31875c5ae76aSFam Zheng out:
318899723548SPaolo Bonzini     bdrv_dec_in_flight(bs);
31895c5ae76aSFam Zheng     return co.ret;
31905c5ae76aSFam Zheng }
31915c5ae76aSFam Zheng 
319261007b31SStefan Hajnoczi void *qemu_blockalign(BlockDriverState *bs, size_t size)
319361007b31SStefan Hajnoczi {
3194384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
319561007b31SStefan Hajnoczi     return qemu_memalign(bdrv_opt_mem_align(bs), size);
319661007b31SStefan Hajnoczi }
319761007b31SStefan Hajnoczi 
319861007b31SStefan Hajnoczi void *qemu_blockalign0(BlockDriverState *bs, size_t size)
319961007b31SStefan Hajnoczi {
3200384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
320161007b31SStefan Hajnoczi     return memset(qemu_blockalign(bs, size), 0, size);
320261007b31SStefan Hajnoczi }
320361007b31SStefan Hajnoczi 
320461007b31SStefan Hajnoczi void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
320561007b31SStefan Hajnoczi {
320661007b31SStefan Hajnoczi     size_t align = bdrv_opt_mem_align(bs);
3207384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
320861007b31SStefan Hajnoczi 
320961007b31SStefan Hajnoczi     /* Ensure that NULL is never returned on success */
321061007b31SStefan Hajnoczi     assert(align > 0);
321161007b31SStefan Hajnoczi     if (size == 0) {
321261007b31SStefan Hajnoczi         size = align;
321361007b31SStefan Hajnoczi     }
321461007b31SStefan Hajnoczi 
321561007b31SStefan Hajnoczi     return qemu_try_memalign(align, size);
321661007b31SStefan Hajnoczi }
321761007b31SStefan Hajnoczi 
321861007b31SStefan Hajnoczi void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
321961007b31SStefan Hajnoczi {
322061007b31SStefan Hajnoczi     void *mem = qemu_try_blockalign(bs, size);
3221384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
322261007b31SStefan Hajnoczi 
322361007b31SStefan Hajnoczi     if (mem) {
322461007b31SStefan Hajnoczi         memset(mem, 0, size);
322561007b31SStefan Hajnoczi     }
322661007b31SStefan Hajnoczi 
322761007b31SStefan Hajnoczi     return mem;
322861007b31SStefan Hajnoczi }
322961007b31SStefan Hajnoczi 
323061007b31SStefan Hajnoczi void bdrv_io_plug(BlockDriverState *bs)
323161007b31SStefan Hajnoczi {
32326b98bd64SPaolo Bonzini     BdrvChild *child;
3233384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
32346b98bd64SPaolo Bonzini 
32356b98bd64SPaolo Bonzini     QLIST_FOREACH(child, &bs->children, next) {
32366b98bd64SPaolo Bonzini         bdrv_io_plug(child->bs);
32376b98bd64SPaolo Bonzini     }
32386b98bd64SPaolo Bonzini 
3239d73415a3SStefan Hajnoczi     if (qatomic_fetch_inc(&bs->io_plugged) == 0) {
324061007b31SStefan Hajnoczi         BlockDriver *drv = bs->drv;
324161007b31SStefan Hajnoczi         if (drv && drv->bdrv_io_plug) {
324261007b31SStefan Hajnoczi             drv->bdrv_io_plug(bs);
32436b98bd64SPaolo Bonzini         }
324461007b31SStefan Hajnoczi     }
324561007b31SStefan Hajnoczi }
324661007b31SStefan Hajnoczi 
324761007b31SStefan Hajnoczi void bdrv_io_unplug(BlockDriverState *bs)
324861007b31SStefan Hajnoczi {
32496b98bd64SPaolo Bonzini     BdrvChild *child;
3250384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
32516b98bd64SPaolo Bonzini 
32526b98bd64SPaolo Bonzini     assert(bs->io_plugged);
3253d73415a3SStefan Hajnoczi     if (qatomic_fetch_dec(&bs->io_plugged) == 1) {
325461007b31SStefan Hajnoczi         BlockDriver *drv = bs->drv;
325561007b31SStefan Hajnoczi         if (drv && drv->bdrv_io_unplug) {
325661007b31SStefan Hajnoczi             drv->bdrv_io_unplug(bs);
325761007b31SStefan Hajnoczi         }
325861007b31SStefan Hajnoczi     }
325961007b31SStefan Hajnoczi 
32606b98bd64SPaolo Bonzini     QLIST_FOREACH(child, &bs->children, next) {
32616b98bd64SPaolo Bonzini         bdrv_io_unplug(child->bs);
32626b98bd64SPaolo Bonzini     }
32636b98bd64SPaolo Bonzini }
326423d0ba93SFam Zheng 
326523d0ba93SFam Zheng void bdrv_register_buf(BlockDriverState *bs, void *host, size_t size)
326623d0ba93SFam Zheng {
326723d0ba93SFam Zheng     BdrvChild *child;
326823d0ba93SFam Zheng 
3269f791bf7fSEmanuele Giuseppe Esposito     GLOBAL_STATE_CODE();
327023d0ba93SFam Zheng     if (bs->drv && bs->drv->bdrv_register_buf) {
327123d0ba93SFam Zheng         bs->drv->bdrv_register_buf(bs, host, size);
327223d0ba93SFam Zheng     }
327323d0ba93SFam Zheng     QLIST_FOREACH(child, &bs->children, next) {
327423d0ba93SFam Zheng         bdrv_register_buf(child->bs, host, size);
327523d0ba93SFam Zheng     }
327623d0ba93SFam Zheng }
327723d0ba93SFam Zheng 
3278*4f384011SStefan Hajnoczi void bdrv_unregister_buf(BlockDriverState *bs, void *host, size_t size)
327923d0ba93SFam Zheng {
328023d0ba93SFam Zheng     BdrvChild *child;
328123d0ba93SFam Zheng 
3282f791bf7fSEmanuele Giuseppe Esposito     GLOBAL_STATE_CODE();
328323d0ba93SFam Zheng     if (bs->drv && bs->drv->bdrv_unregister_buf) {
3284*4f384011SStefan Hajnoczi         bs->drv->bdrv_unregister_buf(bs, host, size);
328523d0ba93SFam Zheng     }
328623d0ba93SFam Zheng     QLIST_FOREACH(child, &bs->children, next) {
3287*4f384011SStefan Hajnoczi         bdrv_unregister_buf(child->bs, host, size);
328823d0ba93SFam Zheng     }
328923d0ba93SFam Zheng }
3290fcc67678SFam Zheng 
329167b51fb9SVladimir Sementsov-Ogievskiy static int coroutine_fn bdrv_co_copy_range_internal(
3292a5215b8fSVladimir Sementsov-Ogievskiy         BdrvChild *src, int64_t src_offset, BdrvChild *dst,
3293a5215b8fSVladimir Sementsov-Ogievskiy         int64_t dst_offset, int64_t bytes,
329467b51fb9SVladimir Sementsov-Ogievskiy         BdrvRequestFlags read_flags, BdrvRequestFlags write_flags,
3295fcc67678SFam Zheng         bool recurse_src)
3296fcc67678SFam Zheng {
3297999658a0SVladimir Sementsov-Ogievskiy     BdrvTrackedRequest req;
3298fcc67678SFam Zheng     int ret;
3299fcc67678SFam Zheng 
3300fe0480d6SKevin Wolf     /* TODO We can support BDRV_REQ_NO_FALLBACK here */
3301fe0480d6SKevin Wolf     assert(!(read_flags & BDRV_REQ_NO_FALLBACK));
3302fe0480d6SKevin Wolf     assert(!(write_flags & BDRV_REQ_NO_FALLBACK));
330345e62b46SVladimir Sementsov-Ogievskiy     assert(!(read_flags & BDRV_REQ_NO_WAIT));
330445e62b46SVladimir Sementsov-Ogievskiy     assert(!(write_flags & BDRV_REQ_NO_WAIT));
3305fe0480d6SKevin Wolf 
3306f4dad307SVladimir Sementsov-Ogievskiy     if (!dst || !dst->bs || !bdrv_is_inserted(dst->bs)) {
3307fcc67678SFam Zheng         return -ENOMEDIUM;
3308fcc67678SFam Zheng     }
330963f4ad11SVladimir Sementsov-Ogievskiy     ret = bdrv_check_request32(dst_offset, bytes, NULL, 0);
3310fcc67678SFam Zheng     if (ret) {
3311fcc67678SFam Zheng         return ret;
3312fcc67678SFam Zheng     }
331367b51fb9SVladimir Sementsov-Ogievskiy     if (write_flags & BDRV_REQ_ZERO_WRITE) {
331467b51fb9SVladimir Sementsov-Ogievskiy         return bdrv_co_pwrite_zeroes(dst, dst_offset, bytes, write_flags);
3315fcc67678SFam Zheng     }
3316fcc67678SFam Zheng 
3317f4dad307SVladimir Sementsov-Ogievskiy     if (!src || !src->bs || !bdrv_is_inserted(src->bs)) {
3318d4d3e5a0SFam Zheng         return -ENOMEDIUM;
3319d4d3e5a0SFam Zheng     }
332063f4ad11SVladimir Sementsov-Ogievskiy     ret = bdrv_check_request32(src_offset, bytes, NULL, 0);
3321d4d3e5a0SFam Zheng     if (ret) {
3322d4d3e5a0SFam Zheng         return ret;
3323d4d3e5a0SFam Zheng     }
3324d4d3e5a0SFam Zheng 
3325fcc67678SFam Zheng     if (!src->bs->drv->bdrv_co_copy_range_from
3326fcc67678SFam Zheng         || !dst->bs->drv->bdrv_co_copy_range_to
3327fcc67678SFam Zheng         || src->bs->encrypted || dst->bs->encrypted) {
3328fcc67678SFam Zheng         return -ENOTSUP;
3329fcc67678SFam Zheng     }
3330999658a0SVladimir Sementsov-Ogievskiy 
3331999658a0SVladimir Sementsov-Ogievskiy     if (recurse_src) {
3332d4d3e5a0SFam Zheng         bdrv_inc_in_flight(src->bs);
3333999658a0SVladimir Sementsov-Ogievskiy         tracked_request_begin(&req, src->bs, src_offset, bytes,
3334999658a0SVladimir Sementsov-Ogievskiy                               BDRV_TRACKED_READ);
333537aec7d7SFam Zheng 
333609d2f948SVladimir Sementsov-Ogievskiy         /* BDRV_REQ_SERIALISING is only for write operation */
333709d2f948SVladimir Sementsov-Ogievskiy         assert(!(read_flags & BDRV_REQ_SERIALISING));
3338304d9d7fSMax Reitz         bdrv_wait_serialising_requests(&req);
3339999658a0SVladimir Sementsov-Ogievskiy 
334037aec7d7SFam Zheng         ret = src->bs->drv->bdrv_co_copy_range_from(src->bs,
3341fcc67678SFam Zheng                                                     src, src_offset,
3342fcc67678SFam Zheng                                                     dst, dst_offset,
334367b51fb9SVladimir Sementsov-Ogievskiy                                                     bytes,
334467b51fb9SVladimir Sementsov-Ogievskiy                                                     read_flags, write_flags);
3345999658a0SVladimir Sementsov-Ogievskiy 
3346999658a0SVladimir Sementsov-Ogievskiy         tracked_request_end(&req);
3347999658a0SVladimir Sementsov-Ogievskiy         bdrv_dec_in_flight(src->bs);
3348fcc67678SFam Zheng     } else {
3349999658a0SVladimir Sementsov-Ogievskiy         bdrv_inc_in_flight(dst->bs);
3350999658a0SVladimir Sementsov-Ogievskiy         tracked_request_begin(&req, dst->bs, dst_offset, bytes,
3351999658a0SVladimir Sementsov-Ogievskiy                               BDRV_TRACKED_WRITE);
33520eb1e891SFam Zheng         ret = bdrv_co_write_req_prepare(dst, dst_offset, bytes, &req,
33530eb1e891SFam Zheng                                         write_flags);
33540eb1e891SFam Zheng         if (!ret) {
335537aec7d7SFam Zheng             ret = dst->bs->drv->bdrv_co_copy_range_to(dst->bs,
3356fcc67678SFam Zheng                                                       src, src_offset,
3357fcc67678SFam Zheng                                                       dst, dst_offset,
335867b51fb9SVladimir Sementsov-Ogievskiy                                                       bytes,
335967b51fb9SVladimir Sementsov-Ogievskiy                                                       read_flags, write_flags);
33600eb1e891SFam Zheng         }
33610eb1e891SFam Zheng         bdrv_co_write_req_finish(dst, dst_offset, bytes, &req, ret);
3362999658a0SVladimir Sementsov-Ogievskiy         tracked_request_end(&req);
3363d4d3e5a0SFam Zheng         bdrv_dec_in_flight(dst->bs);
3364999658a0SVladimir Sementsov-Ogievskiy     }
3365999658a0SVladimir Sementsov-Ogievskiy 
336637aec7d7SFam Zheng     return ret;
3367fcc67678SFam Zheng }
3368fcc67678SFam Zheng 
3369fcc67678SFam Zheng /* Copy range from @src to @dst.
3370fcc67678SFam Zheng  *
3371fcc67678SFam Zheng  * See the comment of bdrv_co_copy_range for the parameter and return value
3372fcc67678SFam Zheng  * semantics. */
3373a5215b8fSVladimir Sementsov-Ogievskiy int coroutine_fn bdrv_co_copy_range_from(BdrvChild *src, int64_t src_offset,
3374a5215b8fSVladimir Sementsov-Ogievskiy                                          BdrvChild *dst, int64_t dst_offset,
3375a5215b8fSVladimir Sementsov-Ogievskiy                                          int64_t bytes,
337667b51fb9SVladimir Sementsov-Ogievskiy                                          BdrvRequestFlags read_flags,
337767b51fb9SVladimir Sementsov-Ogievskiy                                          BdrvRequestFlags write_flags)
3378fcc67678SFam Zheng {
3379967d7905SEmanuele Giuseppe Esposito     IO_CODE();
3380ecc983a5SFam Zheng     trace_bdrv_co_copy_range_from(src, src_offset, dst, dst_offset, bytes,
3381ecc983a5SFam Zheng                                   read_flags, write_flags);
3382fcc67678SFam Zheng     return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset,
338367b51fb9SVladimir Sementsov-Ogievskiy                                        bytes, read_flags, write_flags, true);
3384fcc67678SFam Zheng }
3385fcc67678SFam Zheng 
3386fcc67678SFam Zheng /* Copy range from @src to @dst.
3387fcc67678SFam Zheng  *
3388fcc67678SFam Zheng  * See the comment of bdrv_co_copy_range for the parameter and return value
3389fcc67678SFam Zheng  * semantics. */
3390a5215b8fSVladimir Sementsov-Ogievskiy int coroutine_fn bdrv_co_copy_range_to(BdrvChild *src, int64_t src_offset,
3391a5215b8fSVladimir Sementsov-Ogievskiy                                        BdrvChild *dst, int64_t dst_offset,
3392a5215b8fSVladimir Sementsov-Ogievskiy                                        int64_t bytes,
339367b51fb9SVladimir Sementsov-Ogievskiy                                        BdrvRequestFlags read_flags,
339467b51fb9SVladimir Sementsov-Ogievskiy                                        BdrvRequestFlags write_flags)
3395fcc67678SFam Zheng {
3396967d7905SEmanuele Giuseppe Esposito     IO_CODE();
3397ecc983a5SFam Zheng     trace_bdrv_co_copy_range_to(src, src_offset, dst, dst_offset, bytes,
3398ecc983a5SFam Zheng                                 read_flags, write_flags);
3399fcc67678SFam Zheng     return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset,
340067b51fb9SVladimir Sementsov-Ogievskiy                                        bytes, read_flags, write_flags, false);
3401fcc67678SFam Zheng }
3402fcc67678SFam Zheng 
3403a5215b8fSVladimir Sementsov-Ogievskiy int coroutine_fn bdrv_co_copy_range(BdrvChild *src, int64_t src_offset,
3404a5215b8fSVladimir Sementsov-Ogievskiy                                     BdrvChild *dst, int64_t dst_offset,
3405a5215b8fSVladimir Sementsov-Ogievskiy                                     int64_t bytes, BdrvRequestFlags read_flags,
340667b51fb9SVladimir Sementsov-Ogievskiy                                     BdrvRequestFlags write_flags)
3407fcc67678SFam Zheng {
3408384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
340937aec7d7SFam Zheng     return bdrv_co_copy_range_from(src, src_offset,
3410fcc67678SFam Zheng                                    dst, dst_offset,
341167b51fb9SVladimir Sementsov-Ogievskiy                                    bytes, read_flags, write_flags);
3412fcc67678SFam Zheng }
34133d9f2d2aSKevin Wolf 
34143d9f2d2aSKevin Wolf static void bdrv_parent_cb_resize(BlockDriverState *bs)
34153d9f2d2aSKevin Wolf {
34163d9f2d2aSKevin Wolf     BdrvChild *c;
34173d9f2d2aSKevin Wolf     QLIST_FOREACH(c, &bs->parents, next_parent) {
3418bd86fb99SMax Reitz         if (c->klass->resize) {
3419bd86fb99SMax Reitz             c->klass->resize(c);
34203d9f2d2aSKevin Wolf         }
34213d9f2d2aSKevin Wolf     }
34223d9f2d2aSKevin Wolf }
34233d9f2d2aSKevin Wolf 
34243d9f2d2aSKevin Wolf /**
34253d9f2d2aSKevin Wolf  * Truncate file to 'offset' bytes (needed only for file protocols)
3426c80d8b06SMax Reitz  *
3427c80d8b06SMax Reitz  * If 'exact' is true, the file must be resized to exactly the given
3428c80d8b06SMax Reitz  * 'offset'.  Otherwise, it is sufficient for the node to be at least
3429c80d8b06SMax Reitz  * 'offset' bytes in length.
34303d9f2d2aSKevin Wolf  */
3431c80d8b06SMax Reitz int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset, bool exact,
34327b8e4857SKevin Wolf                                   PreallocMode prealloc, BdrvRequestFlags flags,
34337b8e4857SKevin Wolf                                   Error **errp)
34343d9f2d2aSKevin Wolf {
34353d9f2d2aSKevin Wolf     BlockDriverState *bs = child->bs;
343623b93525SMax Reitz     BdrvChild *filtered, *backing;
34373d9f2d2aSKevin Wolf     BlockDriver *drv = bs->drv;
34381bc5f09fSKevin Wolf     BdrvTrackedRequest req;
34391bc5f09fSKevin Wolf     int64_t old_size, new_bytes;
34403d9f2d2aSKevin Wolf     int ret;
3441384a48fbSEmanuele Giuseppe Esposito     IO_CODE();
34423d9f2d2aSKevin Wolf 
34433d9f2d2aSKevin Wolf     /* if bs->drv == NULL, bs is closed, so there's nothing to do here */
34443d9f2d2aSKevin Wolf     if (!drv) {
34453d9f2d2aSKevin Wolf         error_setg(errp, "No medium inserted");
34463d9f2d2aSKevin Wolf         return -ENOMEDIUM;
34473d9f2d2aSKevin Wolf     }
34483d9f2d2aSKevin Wolf     if (offset < 0) {
34493d9f2d2aSKevin Wolf         error_setg(errp, "Image size cannot be negative");
34503d9f2d2aSKevin Wolf         return -EINVAL;
34513d9f2d2aSKevin Wolf     }
34523d9f2d2aSKevin Wolf 
345369b55e03SVladimir Sementsov-Ogievskiy     ret = bdrv_check_request(offset, 0, errp);
34548b117001SVladimir Sementsov-Ogievskiy     if (ret < 0) {
34558b117001SVladimir Sementsov-Ogievskiy         return ret;
34568b117001SVladimir Sementsov-Ogievskiy     }
34578b117001SVladimir Sementsov-Ogievskiy 
34581bc5f09fSKevin Wolf     old_size = bdrv_getlength(bs);
34591bc5f09fSKevin Wolf     if (old_size < 0) {
34601bc5f09fSKevin Wolf         error_setg_errno(errp, -old_size, "Failed to get old image size");
34611bc5f09fSKevin Wolf         return old_size;
34621bc5f09fSKevin Wolf     }
34631bc5f09fSKevin Wolf 
346497efa869SEric Blake     if (bdrv_is_read_only(bs)) {
346597efa869SEric Blake         error_setg(errp, "Image is read-only");
346697efa869SEric Blake         return -EACCES;
346797efa869SEric Blake     }
346897efa869SEric Blake 
34691bc5f09fSKevin Wolf     if (offset > old_size) {
34701bc5f09fSKevin Wolf         new_bytes = offset - old_size;
34711bc5f09fSKevin Wolf     } else {
34721bc5f09fSKevin Wolf         new_bytes = 0;
34731bc5f09fSKevin Wolf     }
34741bc5f09fSKevin Wolf 
34753d9f2d2aSKevin Wolf     bdrv_inc_in_flight(bs);
34765416a11eSFam Zheng     tracked_request_begin(&req, bs, offset - new_bytes, new_bytes,
34775416a11eSFam Zheng                           BDRV_TRACKED_TRUNCATE);
34781bc5f09fSKevin Wolf 
34791bc5f09fSKevin Wolf     /* If we are growing the image and potentially using preallocation for the
34801bc5f09fSKevin Wolf      * new area, we need to make sure that no write requests are made to it
34811bc5f09fSKevin Wolf      * concurrently or they might be overwritten by preallocation. */
34821bc5f09fSKevin Wolf     if (new_bytes) {
34838ac5aab2SVladimir Sementsov-Ogievskiy         bdrv_make_request_serialising(&req, 1);
3484cd47d792SFam Zheng     }
3485cd47d792SFam Zheng     ret = bdrv_co_write_req_prepare(child, offset - new_bytes, new_bytes, &req,
3486cd47d792SFam Zheng                                     0);
3487cd47d792SFam Zheng     if (ret < 0) {
3488cd47d792SFam Zheng         error_setg_errno(errp, -ret,
3489cd47d792SFam Zheng                          "Failed to prepare request for truncation");
3490cd47d792SFam Zheng         goto out;
34911bc5f09fSKevin Wolf     }
34923d9f2d2aSKevin Wolf 
349393393e69SMax Reitz     filtered = bdrv_filter_child(bs);
349423b93525SMax Reitz     backing = bdrv_cow_child(bs);
349593393e69SMax Reitz 
3496955c7d66SKevin Wolf     /*
3497955c7d66SKevin Wolf      * If the image has a backing file that is large enough that it would
3498955c7d66SKevin Wolf      * provide data for the new area, we cannot leave it unallocated because
3499955c7d66SKevin Wolf      * then the backing file content would become visible. Instead, zero-fill
3500955c7d66SKevin Wolf      * the new area.
3501955c7d66SKevin Wolf      *
3502955c7d66SKevin Wolf      * Note that if the image has a backing file, but was opened without the
3503955c7d66SKevin Wolf      * backing file, taking care of keeping things consistent with that backing
3504955c7d66SKevin Wolf      * file is the user's responsibility.
3505955c7d66SKevin Wolf      */
350623b93525SMax Reitz     if (new_bytes && backing) {
3507955c7d66SKevin Wolf         int64_t backing_len;
3508955c7d66SKevin Wolf 
350923b93525SMax Reitz         backing_len = bdrv_getlength(backing->bs);
3510955c7d66SKevin Wolf         if (backing_len < 0) {
3511955c7d66SKevin Wolf             ret = backing_len;
3512955c7d66SKevin Wolf             error_setg_errno(errp, -ret, "Could not get backing file size");
3513955c7d66SKevin Wolf             goto out;
3514955c7d66SKevin Wolf         }
3515955c7d66SKevin Wolf 
3516955c7d66SKevin Wolf         if (backing_len > old_size) {
3517955c7d66SKevin Wolf             flags |= BDRV_REQ_ZERO_WRITE;
3518955c7d66SKevin Wolf         }
3519955c7d66SKevin Wolf     }
3520955c7d66SKevin Wolf 
35216b7e8f8bSMax Reitz     if (drv->bdrv_co_truncate) {
352292b92799SKevin Wolf         if (flags & ~bs->supported_truncate_flags) {
352392b92799SKevin Wolf             error_setg(errp, "Block driver does not support requested flags");
352492b92799SKevin Wolf             ret = -ENOTSUP;
352592b92799SKevin Wolf             goto out;
352692b92799SKevin Wolf         }
352792b92799SKevin Wolf         ret = drv->bdrv_co_truncate(bs, offset, exact, prealloc, flags, errp);
352893393e69SMax Reitz     } else if (filtered) {
352993393e69SMax Reitz         ret = bdrv_co_truncate(filtered, offset, exact, prealloc, flags, errp);
35306b7e8f8bSMax Reitz     } else {
35313d9f2d2aSKevin Wolf         error_setg(errp, "Image format driver does not support resize");
35323d9f2d2aSKevin Wolf         ret = -ENOTSUP;
35333d9f2d2aSKevin Wolf         goto out;
35343d9f2d2aSKevin Wolf     }
35353d9f2d2aSKevin Wolf     if (ret < 0) {
35363d9f2d2aSKevin Wolf         goto out;
35373d9f2d2aSKevin Wolf     }
35386b7e8f8bSMax Reitz 
35393d9f2d2aSKevin Wolf     ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
35403d9f2d2aSKevin Wolf     if (ret < 0) {
35413d9f2d2aSKevin Wolf         error_setg_errno(errp, -ret, "Could not refresh total sector count");
35423d9f2d2aSKevin Wolf     } else {
35433d9f2d2aSKevin Wolf         offset = bs->total_sectors * BDRV_SECTOR_SIZE;
35443d9f2d2aSKevin Wolf     }
3545cd47d792SFam Zheng     /* It's possible that truncation succeeded but refresh_total_sectors
3546cd47d792SFam Zheng      * failed, but the latter doesn't affect how we should finish the request.
3547cd47d792SFam Zheng      * Pass 0 as the last parameter so that dirty bitmaps etc. are handled. */
3548cd47d792SFam Zheng     bdrv_co_write_req_finish(child, offset - new_bytes, new_bytes, &req, 0);
35493d9f2d2aSKevin Wolf 
35503d9f2d2aSKevin Wolf out:
35511bc5f09fSKevin Wolf     tracked_request_end(&req);
35523d9f2d2aSKevin Wolf     bdrv_dec_in_flight(bs);
35531bc5f09fSKevin Wolf 
35543d9f2d2aSKevin Wolf     return ret;
35553d9f2d2aSKevin Wolf }
3556bd54669aSVladimir Sementsov-Ogievskiy 
3557bd54669aSVladimir Sementsov-Ogievskiy void bdrv_cancel_in_flight(BlockDriverState *bs)
3558bd54669aSVladimir Sementsov-Ogievskiy {
3559f791bf7fSEmanuele Giuseppe Esposito     GLOBAL_STATE_CODE();
3560bd54669aSVladimir Sementsov-Ogievskiy     if (!bs || !bs->drv) {
3561bd54669aSVladimir Sementsov-Ogievskiy         return;
3562bd54669aSVladimir Sementsov-Ogievskiy     }
3563bd54669aSVladimir Sementsov-Ogievskiy 
3564bd54669aSVladimir Sementsov-Ogievskiy     if (bs->drv->bdrv_cancel_in_flight) {
3565bd54669aSVladimir Sementsov-Ogievskiy         bs->drv->bdrv_cancel_in_flight(bs);
3566bd54669aSVladimir Sementsov-Ogievskiy     }
3567bd54669aSVladimir Sementsov-Ogievskiy }
3568ce14f3b4SVladimir Sementsov-Ogievskiy 
3569ce14f3b4SVladimir Sementsov-Ogievskiy int coroutine_fn
3570ce14f3b4SVladimir Sementsov-Ogievskiy bdrv_co_preadv_snapshot(BdrvChild *child, int64_t offset, int64_t bytes,
3571ce14f3b4SVladimir Sementsov-Ogievskiy                         QEMUIOVector *qiov, size_t qiov_offset)
3572ce14f3b4SVladimir Sementsov-Ogievskiy {
3573ce14f3b4SVladimir Sementsov-Ogievskiy     BlockDriverState *bs = child->bs;
3574ce14f3b4SVladimir Sementsov-Ogievskiy     BlockDriver *drv = bs->drv;
3575ce14f3b4SVladimir Sementsov-Ogievskiy     int ret;
3576ce14f3b4SVladimir Sementsov-Ogievskiy     IO_CODE();
3577ce14f3b4SVladimir Sementsov-Ogievskiy 
3578ce14f3b4SVladimir Sementsov-Ogievskiy     if (!drv) {
3579ce14f3b4SVladimir Sementsov-Ogievskiy         return -ENOMEDIUM;
3580ce14f3b4SVladimir Sementsov-Ogievskiy     }
3581ce14f3b4SVladimir Sementsov-Ogievskiy 
3582ce14f3b4SVladimir Sementsov-Ogievskiy     if (!drv->bdrv_co_preadv_snapshot) {
3583ce14f3b4SVladimir Sementsov-Ogievskiy         return -ENOTSUP;
3584ce14f3b4SVladimir Sementsov-Ogievskiy     }
3585ce14f3b4SVladimir Sementsov-Ogievskiy 
3586ce14f3b4SVladimir Sementsov-Ogievskiy     bdrv_inc_in_flight(bs);
3587ce14f3b4SVladimir Sementsov-Ogievskiy     ret = drv->bdrv_co_preadv_snapshot(bs, offset, bytes, qiov, qiov_offset);
3588ce14f3b4SVladimir Sementsov-Ogievskiy     bdrv_dec_in_flight(bs);
3589ce14f3b4SVladimir Sementsov-Ogievskiy 
3590ce14f3b4SVladimir Sementsov-Ogievskiy     return ret;
3591ce14f3b4SVladimir Sementsov-Ogievskiy }
3592ce14f3b4SVladimir Sementsov-Ogievskiy 
3593ce14f3b4SVladimir Sementsov-Ogievskiy int coroutine_fn
3594ce14f3b4SVladimir Sementsov-Ogievskiy bdrv_co_snapshot_block_status(BlockDriverState *bs,
3595ce14f3b4SVladimir Sementsov-Ogievskiy                               bool want_zero, int64_t offset, int64_t bytes,
3596ce14f3b4SVladimir Sementsov-Ogievskiy                               int64_t *pnum, int64_t *map,
3597ce14f3b4SVladimir Sementsov-Ogievskiy                               BlockDriverState **file)
3598ce14f3b4SVladimir Sementsov-Ogievskiy {
3599ce14f3b4SVladimir Sementsov-Ogievskiy     BlockDriver *drv = bs->drv;
3600ce14f3b4SVladimir Sementsov-Ogievskiy     int ret;
3601ce14f3b4SVladimir Sementsov-Ogievskiy     IO_CODE();
3602ce14f3b4SVladimir Sementsov-Ogievskiy 
3603ce14f3b4SVladimir Sementsov-Ogievskiy     if (!drv) {
3604ce14f3b4SVladimir Sementsov-Ogievskiy         return -ENOMEDIUM;
3605ce14f3b4SVladimir Sementsov-Ogievskiy     }
3606ce14f3b4SVladimir Sementsov-Ogievskiy 
3607ce14f3b4SVladimir Sementsov-Ogievskiy     if (!drv->bdrv_co_snapshot_block_status) {
3608ce14f3b4SVladimir Sementsov-Ogievskiy         return -ENOTSUP;
3609ce14f3b4SVladimir Sementsov-Ogievskiy     }
3610ce14f3b4SVladimir Sementsov-Ogievskiy 
3611ce14f3b4SVladimir Sementsov-Ogievskiy     bdrv_inc_in_flight(bs);
3612ce14f3b4SVladimir Sementsov-Ogievskiy     ret = drv->bdrv_co_snapshot_block_status(bs, want_zero, offset, bytes,
3613ce14f3b4SVladimir Sementsov-Ogievskiy                                              pnum, map, file);
3614ce14f3b4SVladimir Sementsov-Ogievskiy     bdrv_dec_in_flight(bs);
3615ce14f3b4SVladimir Sementsov-Ogievskiy 
3616ce14f3b4SVladimir Sementsov-Ogievskiy     return ret;
3617ce14f3b4SVladimir Sementsov-Ogievskiy }
3618ce14f3b4SVladimir Sementsov-Ogievskiy 
3619ce14f3b4SVladimir Sementsov-Ogievskiy int coroutine_fn
3620ce14f3b4SVladimir Sementsov-Ogievskiy bdrv_co_pdiscard_snapshot(BlockDriverState *bs, int64_t offset, int64_t bytes)
3621ce14f3b4SVladimir Sementsov-Ogievskiy {
3622ce14f3b4SVladimir Sementsov-Ogievskiy     BlockDriver *drv = bs->drv;
3623ce14f3b4SVladimir Sementsov-Ogievskiy     int ret;
3624ce14f3b4SVladimir Sementsov-Ogievskiy     IO_CODE();
3625ce14f3b4SVladimir Sementsov-Ogievskiy 
3626ce14f3b4SVladimir Sementsov-Ogievskiy     if (!drv) {
3627ce14f3b4SVladimir Sementsov-Ogievskiy         return -ENOMEDIUM;
3628ce14f3b4SVladimir Sementsov-Ogievskiy     }
3629ce14f3b4SVladimir Sementsov-Ogievskiy 
3630ce14f3b4SVladimir Sementsov-Ogievskiy     if (!drv->bdrv_co_pdiscard_snapshot) {
3631ce14f3b4SVladimir Sementsov-Ogievskiy         return -ENOTSUP;
3632ce14f3b4SVladimir Sementsov-Ogievskiy     }
3633ce14f3b4SVladimir Sementsov-Ogievskiy 
3634ce14f3b4SVladimir Sementsov-Ogievskiy     bdrv_inc_in_flight(bs);
3635ce14f3b4SVladimir Sementsov-Ogievskiy     ret = drv->bdrv_co_pdiscard_snapshot(bs, offset, bytes);
3636ce14f3b4SVladimir Sementsov-Ogievskiy     bdrv_dec_in_flight(bs);
3637ce14f3b4SVladimir Sementsov-Ogievskiy 
3638ce14f3b4SVladimir Sementsov-Ogievskiy     return ret;
3639ce14f3b4SVladimir Sementsov-Ogievskiy }
3640