161007b31SStefan Hajnoczi /* 261007b31SStefan Hajnoczi * Block layer I/O functions 361007b31SStefan Hajnoczi * 461007b31SStefan Hajnoczi * Copyright (c) 2003 Fabrice Bellard 561007b31SStefan Hajnoczi * 661007b31SStefan Hajnoczi * Permission is hereby granted, free of charge, to any person obtaining a copy 761007b31SStefan Hajnoczi * of this software and associated documentation files (the "Software"), to deal 861007b31SStefan Hajnoczi * in the Software without restriction, including without limitation the rights 961007b31SStefan Hajnoczi * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 1061007b31SStefan Hajnoczi * copies of the Software, and to permit persons to whom the Software is 1161007b31SStefan Hajnoczi * furnished to do so, subject to the following conditions: 1261007b31SStefan Hajnoczi * 1361007b31SStefan Hajnoczi * The above copyright notice and this permission notice shall be included in 1461007b31SStefan Hajnoczi * all copies or substantial portions of the Software. 1561007b31SStefan Hajnoczi * 1661007b31SStefan Hajnoczi * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 1761007b31SStefan Hajnoczi * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 1861007b31SStefan Hajnoczi * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 1961007b31SStefan Hajnoczi * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 2061007b31SStefan Hajnoczi * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 2161007b31SStefan Hajnoczi * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 2261007b31SStefan Hajnoczi * THE SOFTWARE. 2361007b31SStefan Hajnoczi */ 2461007b31SStefan Hajnoczi 2580c71a24SPeter Maydell #include "qemu/osdep.h" 2661007b31SStefan Hajnoczi #include "trace.h" 277f0e9da6SMax Reitz #include "sysemu/block-backend.h" 287719f3c9SStefan Hajnoczi #include "block/aio-wait.h" 2961007b31SStefan Hajnoczi #include "block/blockjob.h" 30f321dcb5SPaolo Bonzini #include "block/blockjob_int.h" 3161007b31SStefan Hajnoczi #include "block/block_int.h" 3221c2283eSVladimir Sementsov-Ogievskiy #include "block/coroutines.h" 3394783301SVladimir Sementsov-Ogievskiy #include "block/write-threshold.h" 34f348b6d1SVeronia Bahaa #include "qemu/cutils.h" 355df022cfSPeter Maydell #include "qemu/memalign.h" 36da34e65cSMarkus Armbruster #include "qapi/error.h" 37d49b6836SMarkus Armbruster #include "qemu/error-report.h" 38db725815SMarkus Armbruster #include "qemu/main-loop.h" 39c8aa7895SPavel Dovgalyuk #include "sysemu/replay.h" 4061007b31SStefan Hajnoczi 41cb2e2878SEric Blake /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */ 42cb2e2878SEric Blake #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS) 43cb2e2878SEric Blake 447f8f03efSFam Zheng static void bdrv_parent_cb_resize(BlockDriverState *bs); 45d05aa8bbSEric Blake static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 465ae07b14SVladimir Sementsov-Ogievskiy int64_t offset, int64_t bytes, BdrvRequestFlags flags); 4761007b31SStefan Hajnoczi 48f4c8a43bSMax Reitz static void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore, 496cd5c9d7SKevin Wolf bool ignore_bds_parents) 5061007b31SStefan Hajnoczi { 5102d21300SKevin Wolf BdrvChild *c, *next; 5227ccdd52SKevin Wolf 5302d21300SKevin Wolf QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) { 54bd86fb99SMax Reitz if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) { 550152bf40SKevin Wolf continue; 560152bf40SKevin Wolf } 574be6a6d1SKevin Wolf bdrv_parent_drained_begin_single(c, false); 58ce0f1412SPaolo Bonzini } 59ce0f1412SPaolo Bonzini } 60ce0f1412SPaolo Bonzini 61e037c09cSMax Reitz static void bdrv_parent_drained_end_single_no_poll(BdrvChild *c, 62e037c09cSMax Reitz int *drained_end_counter) 63804db8eaSMax Reitz { 64804db8eaSMax Reitz assert(c->parent_quiesce_counter > 0); 65804db8eaSMax Reitz c->parent_quiesce_counter--; 66bd86fb99SMax Reitz if (c->klass->drained_end) { 67bd86fb99SMax Reitz c->klass->drained_end(c, drained_end_counter); 68804db8eaSMax Reitz } 69804db8eaSMax Reitz } 70804db8eaSMax Reitz 71e037c09cSMax Reitz void bdrv_parent_drained_end_single(BdrvChild *c) 72e037c09cSMax Reitz { 73e037c09cSMax Reitz int drained_end_counter = 0; 74384a48fbSEmanuele Giuseppe Esposito IO_OR_GS_CODE(); 75e037c09cSMax Reitz bdrv_parent_drained_end_single_no_poll(c, &drained_end_counter); 76d73415a3SStefan Hajnoczi BDRV_POLL_WHILE(c->bs, qatomic_read(&drained_end_counter) > 0); 77e037c09cSMax Reitz } 78e037c09cSMax Reitz 79f4c8a43bSMax Reitz static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore, 80e037c09cSMax Reitz bool ignore_bds_parents, 81e037c09cSMax Reitz int *drained_end_counter) 82ce0f1412SPaolo Bonzini { 8361ad631cSMax Reitz BdrvChild *c; 8427ccdd52SKevin Wolf 8561ad631cSMax Reitz QLIST_FOREACH(c, &bs->parents, next_parent) { 86bd86fb99SMax Reitz if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) { 870152bf40SKevin Wolf continue; 880152bf40SKevin Wolf } 89e037c09cSMax Reitz bdrv_parent_drained_end_single_no_poll(c, drained_end_counter); 90c2066af0SKevin Wolf } 9161007b31SStefan Hajnoczi } 9261007b31SStefan Hajnoczi 934be6a6d1SKevin Wolf static bool bdrv_parent_drained_poll_single(BdrvChild *c) 944be6a6d1SKevin Wolf { 95bd86fb99SMax Reitz if (c->klass->drained_poll) { 96bd86fb99SMax Reitz return c->klass->drained_poll(c); 974be6a6d1SKevin Wolf } 984be6a6d1SKevin Wolf return false; 994be6a6d1SKevin Wolf } 1004be6a6d1SKevin Wolf 1016cd5c9d7SKevin Wolf static bool bdrv_parent_drained_poll(BlockDriverState *bs, BdrvChild *ignore, 1026cd5c9d7SKevin Wolf bool ignore_bds_parents) 10389bd0305SKevin Wolf { 10489bd0305SKevin Wolf BdrvChild *c, *next; 10589bd0305SKevin Wolf bool busy = false; 10689bd0305SKevin Wolf 10789bd0305SKevin Wolf QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) { 108bd86fb99SMax Reitz if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) { 10989bd0305SKevin Wolf continue; 11089bd0305SKevin Wolf } 1114be6a6d1SKevin Wolf busy |= bdrv_parent_drained_poll_single(c); 11289bd0305SKevin Wolf } 11389bd0305SKevin Wolf 11489bd0305SKevin Wolf return busy; 11589bd0305SKevin Wolf } 11689bd0305SKevin Wolf 1174be6a6d1SKevin Wolf void bdrv_parent_drained_begin_single(BdrvChild *c, bool poll) 1184be6a6d1SKevin Wolf { 119384a48fbSEmanuele Giuseppe Esposito IO_OR_GS_CODE(); 120804db8eaSMax Reitz c->parent_quiesce_counter++; 121bd86fb99SMax Reitz if (c->klass->drained_begin) { 122bd86fb99SMax Reitz c->klass->drained_begin(c); 1234be6a6d1SKevin Wolf } 1244be6a6d1SKevin Wolf if (poll) { 1254be6a6d1SKevin Wolf BDRV_POLL_WHILE(c->bs, bdrv_parent_drained_poll_single(c)); 1264be6a6d1SKevin Wolf } 1274be6a6d1SKevin Wolf } 1284be6a6d1SKevin Wolf 129d9e0dfa2SEric Blake static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src) 130d9e0dfa2SEric Blake { 1319f460c64SAkihiko Odaki dst->pdiscard_alignment = MAX(dst->pdiscard_alignment, 1329f460c64SAkihiko Odaki src->pdiscard_alignment); 133d9e0dfa2SEric Blake dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer); 134d9e0dfa2SEric Blake dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer); 13524b36e98SPaolo Bonzini dst->max_hw_transfer = MIN_NON_ZERO(dst->max_hw_transfer, 13624b36e98SPaolo Bonzini src->max_hw_transfer); 137d9e0dfa2SEric Blake dst->opt_mem_alignment = MAX(dst->opt_mem_alignment, 138d9e0dfa2SEric Blake src->opt_mem_alignment); 139d9e0dfa2SEric Blake dst->min_mem_alignment = MAX(dst->min_mem_alignment, 140d9e0dfa2SEric Blake src->min_mem_alignment); 141d9e0dfa2SEric Blake dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov); 142cc071629SPaolo Bonzini dst->max_hw_iov = MIN_NON_ZERO(dst->max_hw_iov, src->max_hw_iov); 143d9e0dfa2SEric Blake } 144d9e0dfa2SEric Blake 1451e4c797cSVladimir Sementsov-Ogievskiy typedef struct BdrvRefreshLimitsState { 1461e4c797cSVladimir Sementsov-Ogievskiy BlockDriverState *bs; 1471e4c797cSVladimir Sementsov-Ogievskiy BlockLimits old_bl; 1481e4c797cSVladimir Sementsov-Ogievskiy } BdrvRefreshLimitsState; 1491e4c797cSVladimir Sementsov-Ogievskiy 1501e4c797cSVladimir Sementsov-Ogievskiy static void bdrv_refresh_limits_abort(void *opaque) 1511e4c797cSVladimir Sementsov-Ogievskiy { 1521e4c797cSVladimir Sementsov-Ogievskiy BdrvRefreshLimitsState *s = opaque; 1531e4c797cSVladimir Sementsov-Ogievskiy 1541e4c797cSVladimir Sementsov-Ogievskiy s->bs->bl = s->old_bl; 1551e4c797cSVladimir Sementsov-Ogievskiy } 1561e4c797cSVladimir Sementsov-Ogievskiy 1571e4c797cSVladimir Sementsov-Ogievskiy static TransactionActionDrv bdrv_refresh_limits_drv = { 1581e4c797cSVladimir Sementsov-Ogievskiy .abort = bdrv_refresh_limits_abort, 1591e4c797cSVladimir Sementsov-Ogievskiy .clean = g_free, 1601e4c797cSVladimir Sementsov-Ogievskiy }; 1611e4c797cSVladimir Sementsov-Ogievskiy 1621e4c797cSVladimir Sementsov-Ogievskiy /* @tran is allowed to be NULL, in this case no rollback is possible. */ 1631e4c797cSVladimir Sementsov-Ogievskiy void bdrv_refresh_limits(BlockDriverState *bs, Transaction *tran, Error **errp) 16461007b31SStefan Hajnoczi { 16533985614SVladimir Sementsov-Ogievskiy ERRP_GUARD(); 16661007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 16766b129acSMax Reitz BdrvChild *c; 16866b129acSMax Reitz bool have_limits; 16961007b31SStefan Hajnoczi 170f791bf7fSEmanuele Giuseppe Esposito GLOBAL_STATE_CODE(); 171f791bf7fSEmanuele Giuseppe Esposito 1721e4c797cSVladimir Sementsov-Ogievskiy if (tran) { 1731e4c797cSVladimir Sementsov-Ogievskiy BdrvRefreshLimitsState *s = g_new(BdrvRefreshLimitsState, 1); 1741e4c797cSVladimir Sementsov-Ogievskiy *s = (BdrvRefreshLimitsState) { 1751e4c797cSVladimir Sementsov-Ogievskiy .bs = bs, 1761e4c797cSVladimir Sementsov-Ogievskiy .old_bl = bs->bl, 1771e4c797cSVladimir Sementsov-Ogievskiy }; 1781e4c797cSVladimir Sementsov-Ogievskiy tran_add(tran, &bdrv_refresh_limits_drv, s); 1791e4c797cSVladimir Sementsov-Ogievskiy } 1801e4c797cSVladimir Sementsov-Ogievskiy 18161007b31SStefan Hajnoczi memset(&bs->bl, 0, sizeof(bs->bl)); 18261007b31SStefan Hajnoczi 18361007b31SStefan Hajnoczi if (!drv) { 18461007b31SStefan Hajnoczi return; 18561007b31SStefan Hajnoczi } 18661007b31SStefan Hajnoczi 18779ba8c98SEric Blake /* Default alignment based on whether driver has byte interface */ 188e31f6864SEric Blake bs->bl.request_alignment = (drv->bdrv_co_preadv || 189ac850bf0SVladimir Sementsov-Ogievskiy drv->bdrv_aio_preadv || 190ac850bf0SVladimir Sementsov-Ogievskiy drv->bdrv_co_preadv_part) ? 1 : 512; 19179ba8c98SEric Blake 19261007b31SStefan Hajnoczi /* Take some limits from the children as a default */ 19366b129acSMax Reitz have_limits = false; 19466b129acSMax Reitz QLIST_FOREACH(c, &bs->children, next) { 19566b129acSMax Reitz if (c->role & (BDRV_CHILD_DATA | BDRV_CHILD_FILTERED | BDRV_CHILD_COW)) 19666b129acSMax Reitz { 19766b129acSMax Reitz bdrv_merge_limits(&bs->bl, &c->bs->bl); 19866b129acSMax Reitz have_limits = true; 19966b129acSMax Reitz } 20066b129acSMax Reitz } 20166b129acSMax Reitz 20266b129acSMax Reitz if (!have_limits) { 2034196d2f0SDenis V. Lunev bs->bl.min_mem_alignment = 512; 2048e3b0cbbSMarc-André Lureau bs->bl.opt_mem_alignment = qemu_real_host_page_size(); 205bd44feb7SStefan Hajnoczi 206bd44feb7SStefan Hajnoczi /* Safe default since most protocols use readv()/writev()/etc */ 207bd44feb7SStefan Hajnoczi bs->bl.max_iov = IOV_MAX; 20861007b31SStefan Hajnoczi } 20961007b31SStefan Hajnoczi 21061007b31SStefan Hajnoczi /* Then let the driver override it */ 21161007b31SStefan Hajnoczi if (drv->bdrv_refresh_limits) { 21261007b31SStefan Hajnoczi drv->bdrv_refresh_limits(bs, errp); 2138b117001SVladimir Sementsov-Ogievskiy if (*errp) { 2148b117001SVladimir Sementsov-Ogievskiy return; 2158b117001SVladimir Sementsov-Ogievskiy } 2168b117001SVladimir Sementsov-Ogievskiy } 2178b117001SVladimir Sementsov-Ogievskiy 2188b117001SVladimir Sementsov-Ogievskiy if (bs->bl.request_alignment > BDRV_MAX_ALIGNMENT) { 2198b117001SVladimir Sementsov-Ogievskiy error_setg(errp, "Driver requires too large request alignment"); 22061007b31SStefan Hajnoczi } 22161007b31SStefan Hajnoczi } 22261007b31SStefan Hajnoczi 22361007b31SStefan Hajnoczi /** 22461007b31SStefan Hajnoczi * The copy-on-read flag is actually a reference count so multiple users may 22561007b31SStefan Hajnoczi * use the feature without worrying about clobbering its previous state. 22661007b31SStefan Hajnoczi * Copy-on-read stays enabled until all users have called to disable it. 22761007b31SStefan Hajnoczi */ 22861007b31SStefan Hajnoczi void bdrv_enable_copy_on_read(BlockDriverState *bs) 22961007b31SStefan Hajnoczi { 230384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 231d73415a3SStefan Hajnoczi qatomic_inc(&bs->copy_on_read); 23261007b31SStefan Hajnoczi } 23361007b31SStefan Hajnoczi 23461007b31SStefan Hajnoczi void bdrv_disable_copy_on_read(BlockDriverState *bs) 23561007b31SStefan Hajnoczi { 236d73415a3SStefan Hajnoczi int old = qatomic_fetch_dec(&bs->copy_on_read); 237384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 238d3faa13eSPaolo Bonzini assert(old >= 1); 23961007b31SStefan Hajnoczi } 24061007b31SStefan Hajnoczi 24161124f03SPaolo Bonzini typedef struct { 24261124f03SPaolo Bonzini Coroutine *co; 24361124f03SPaolo Bonzini BlockDriverState *bs; 24461124f03SPaolo Bonzini bool done; 245481cad48SManos Pitsidianakis bool begin; 246b0165585SKevin Wolf bool recursive; 247fe4f0614SKevin Wolf bool poll; 2480152bf40SKevin Wolf BdrvChild *parent; 2496cd5c9d7SKevin Wolf bool ignore_bds_parents; 2508e1da77eSMax Reitz int *drained_end_counter; 25161124f03SPaolo Bonzini } BdrvCoDrainData; 25261124f03SPaolo Bonzini 25361124f03SPaolo Bonzini static void coroutine_fn bdrv_drain_invoke_entry(void *opaque) 25461124f03SPaolo Bonzini { 25561124f03SPaolo Bonzini BdrvCoDrainData *data = opaque; 25661124f03SPaolo Bonzini BlockDriverState *bs = data->bs; 25761124f03SPaolo Bonzini 258481cad48SManos Pitsidianakis if (data->begin) { 259f8ea8dacSManos Pitsidianakis bs->drv->bdrv_co_drain_begin(bs); 260481cad48SManos Pitsidianakis } else { 261481cad48SManos Pitsidianakis bs->drv->bdrv_co_drain_end(bs); 262481cad48SManos Pitsidianakis } 26361124f03SPaolo Bonzini 26465181d63SMax Reitz /* Set data->done and decrement drained_end_counter before bdrv_wakeup() */ 265d73415a3SStefan Hajnoczi qatomic_mb_set(&data->done, true); 266e037c09cSMax Reitz if (!data->begin) { 267d73415a3SStefan Hajnoczi qatomic_dec(data->drained_end_counter); 2688e1da77eSMax Reitz } 26965181d63SMax Reitz bdrv_dec_in_flight(bs); 2708e1da77eSMax Reitz 2710109e7e6SKevin Wolf g_free(data); 2720109e7e6SKevin Wolf } 27361124f03SPaolo Bonzini 274db0289b9SKevin Wolf /* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */ 2758e1da77eSMax Reitz static void bdrv_drain_invoke(BlockDriverState *bs, bool begin, 2768e1da77eSMax Reitz int *drained_end_counter) 27761124f03SPaolo Bonzini { 2780109e7e6SKevin Wolf BdrvCoDrainData *data; 27961124f03SPaolo Bonzini 280f8ea8dacSManos Pitsidianakis if (!bs->drv || (begin && !bs->drv->bdrv_co_drain_begin) || 281481cad48SManos Pitsidianakis (!begin && !bs->drv->bdrv_co_drain_end)) { 28261124f03SPaolo Bonzini return; 28361124f03SPaolo Bonzini } 28461124f03SPaolo Bonzini 2850109e7e6SKevin Wolf data = g_new(BdrvCoDrainData, 1); 2860109e7e6SKevin Wolf *data = (BdrvCoDrainData) { 2870109e7e6SKevin Wolf .bs = bs, 2880109e7e6SKevin Wolf .done = false, 2898e1da77eSMax Reitz .begin = begin, 2908e1da77eSMax Reitz .drained_end_counter = drained_end_counter, 2910109e7e6SKevin Wolf }; 2920109e7e6SKevin Wolf 293e037c09cSMax Reitz if (!begin) { 294d73415a3SStefan Hajnoczi qatomic_inc(drained_end_counter); 2958e1da77eSMax Reitz } 2968e1da77eSMax Reitz 2970109e7e6SKevin Wolf /* Make sure the driver callback completes during the polling phase for 2980109e7e6SKevin Wolf * drain_begin. */ 2990109e7e6SKevin Wolf bdrv_inc_in_flight(bs); 3000109e7e6SKevin Wolf data->co = qemu_coroutine_create(bdrv_drain_invoke_entry, data); 3010109e7e6SKevin Wolf aio_co_schedule(bdrv_get_aio_context(bs), data->co); 30261124f03SPaolo Bonzini } 30361124f03SPaolo Bonzini 3041cc8e54aSKevin Wolf /* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */ 305fe4f0614SKevin Wolf bool bdrv_drain_poll(BlockDriverState *bs, bool recursive, 3066cd5c9d7SKevin Wolf BdrvChild *ignore_parent, bool ignore_bds_parents) 30789bd0305SKevin Wolf { 308fe4f0614SKevin Wolf BdrvChild *child, *next; 309384a48fbSEmanuele Giuseppe Esposito IO_OR_GS_CODE(); 310fe4f0614SKevin Wolf 3116cd5c9d7SKevin Wolf if (bdrv_parent_drained_poll(bs, ignore_parent, ignore_bds_parents)) { 31289bd0305SKevin Wolf return true; 31389bd0305SKevin Wolf } 31489bd0305SKevin Wolf 315d73415a3SStefan Hajnoczi if (qatomic_read(&bs->in_flight)) { 316fe4f0614SKevin Wolf return true; 31789bd0305SKevin Wolf } 31889bd0305SKevin Wolf 319fe4f0614SKevin Wolf if (recursive) { 3206cd5c9d7SKevin Wolf assert(!ignore_bds_parents); 321fe4f0614SKevin Wolf QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 3226cd5c9d7SKevin Wolf if (bdrv_drain_poll(child->bs, recursive, child, false)) { 323fe4f0614SKevin Wolf return true; 324fe4f0614SKevin Wolf } 325fe4f0614SKevin Wolf } 326fe4f0614SKevin Wolf } 327fe4f0614SKevin Wolf 328fe4f0614SKevin Wolf return false; 329fe4f0614SKevin Wolf } 330fe4f0614SKevin Wolf 331fe4f0614SKevin Wolf static bool bdrv_drain_poll_top_level(BlockDriverState *bs, bool recursive, 33289bd0305SKevin Wolf BdrvChild *ignore_parent) 3331cc8e54aSKevin Wolf { 3346cd5c9d7SKevin Wolf return bdrv_drain_poll(bs, recursive, ignore_parent, false); 3351cc8e54aSKevin Wolf } 3361cc8e54aSKevin Wolf 337b0165585SKevin Wolf static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive, 3386cd5c9d7SKevin Wolf BdrvChild *parent, bool ignore_bds_parents, 3396cd5c9d7SKevin Wolf bool poll); 340b0165585SKevin Wolf static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive, 3418e1da77eSMax Reitz BdrvChild *parent, bool ignore_bds_parents, 3428e1da77eSMax Reitz int *drained_end_counter); 3430152bf40SKevin Wolf 344a77fd4bbSFam Zheng static void bdrv_co_drain_bh_cb(void *opaque) 345a77fd4bbSFam Zheng { 346a77fd4bbSFam Zheng BdrvCoDrainData *data = opaque; 347a77fd4bbSFam Zheng Coroutine *co = data->co; 34899723548SPaolo Bonzini BlockDriverState *bs = data->bs; 349a77fd4bbSFam Zheng 350c8ca33d0SKevin Wolf if (bs) { 351aa1361d5SKevin Wolf AioContext *ctx = bdrv_get_aio_context(bs); 352aa1361d5SKevin Wolf aio_context_acquire(ctx); 35399723548SPaolo Bonzini bdrv_dec_in_flight(bs); 354481cad48SManos Pitsidianakis if (data->begin) { 355e037c09cSMax Reitz assert(!data->drained_end_counter); 3566cd5c9d7SKevin Wolf bdrv_do_drained_begin(bs, data->recursive, data->parent, 3576cd5c9d7SKevin Wolf data->ignore_bds_parents, data->poll); 358481cad48SManos Pitsidianakis } else { 359e037c09cSMax Reitz assert(!data->poll); 3606cd5c9d7SKevin Wolf bdrv_do_drained_end(bs, data->recursive, data->parent, 3618e1da77eSMax Reitz data->ignore_bds_parents, 3628e1da77eSMax Reitz data->drained_end_counter); 363481cad48SManos Pitsidianakis } 364aa1361d5SKevin Wolf aio_context_release(ctx); 365c8ca33d0SKevin Wolf } else { 366c8ca33d0SKevin Wolf assert(data->begin); 367c8ca33d0SKevin Wolf bdrv_drain_all_begin(); 368c8ca33d0SKevin Wolf } 369481cad48SManos Pitsidianakis 370a77fd4bbSFam Zheng data->done = true; 3711919631eSPaolo Bonzini aio_co_wake(co); 372a77fd4bbSFam Zheng } 373a77fd4bbSFam Zheng 374481cad48SManos Pitsidianakis static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs, 375b0165585SKevin Wolf bool begin, bool recursive, 3766cd5c9d7SKevin Wolf BdrvChild *parent, 3776cd5c9d7SKevin Wolf bool ignore_bds_parents, 3788e1da77eSMax Reitz bool poll, 3798e1da77eSMax Reitz int *drained_end_counter) 380a77fd4bbSFam Zheng { 381a77fd4bbSFam Zheng BdrvCoDrainData data; 382960d5fb3SKevin Wolf Coroutine *self = qemu_coroutine_self(); 383960d5fb3SKevin Wolf AioContext *ctx = bdrv_get_aio_context(bs); 384960d5fb3SKevin Wolf AioContext *co_ctx = qemu_coroutine_get_aio_context(self); 385a77fd4bbSFam Zheng 386a77fd4bbSFam Zheng /* Calling bdrv_drain() from a BH ensures the current coroutine yields and 387c40a2545SStefan Hajnoczi * other coroutines run if they were queued by aio_co_enter(). */ 388a77fd4bbSFam Zheng 389a77fd4bbSFam Zheng assert(qemu_in_coroutine()); 390a77fd4bbSFam Zheng data = (BdrvCoDrainData) { 391960d5fb3SKevin Wolf .co = self, 392a77fd4bbSFam Zheng .bs = bs, 393a77fd4bbSFam Zheng .done = false, 394481cad48SManos Pitsidianakis .begin = begin, 395b0165585SKevin Wolf .recursive = recursive, 3960152bf40SKevin Wolf .parent = parent, 3976cd5c9d7SKevin Wolf .ignore_bds_parents = ignore_bds_parents, 398fe4f0614SKevin Wolf .poll = poll, 3998e1da77eSMax Reitz .drained_end_counter = drained_end_counter, 400a77fd4bbSFam Zheng }; 4018e1da77eSMax Reitz 402c8ca33d0SKevin Wolf if (bs) { 40399723548SPaolo Bonzini bdrv_inc_in_flight(bs); 404c8ca33d0SKevin Wolf } 405960d5fb3SKevin Wolf 406960d5fb3SKevin Wolf /* 407960d5fb3SKevin Wolf * Temporarily drop the lock across yield or we would get deadlocks. 408960d5fb3SKevin Wolf * bdrv_co_drain_bh_cb() reaquires the lock as needed. 409960d5fb3SKevin Wolf * 410960d5fb3SKevin Wolf * When we yield below, the lock for the current context will be 411960d5fb3SKevin Wolf * released, so if this is actually the lock that protects bs, don't drop 412960d5fb3SKevin Wolf * it a second time. 413960d5fb3SKevin Wolf */ 414960d5fb3SKevin Wolf if (ctx != co_ctx) { 415960d5fb3SKevin Wolf aio_context_release(ctx); 416960d5fb3SKevin Wolf } 417960d5fb3SKevin Wolf replay_bh_schedule_oneshot_event(ctx, bdrv_co_drain_bh_cb, &data); 418a77fd4bbSFam Zheng 419a77fd4bbSFam Zheng qemu_coroutine_yield(); 420a77fd4bbSFam Zheng /* If we are resumed from some other event (such as an aio completion or a 421a77fd4bbSFam Zheng * timer callback), it is a bug in the caller that should be fixed. */ 422a77fd4bbSFam Zheng assert(data.done); 423960d5fb3SKevin Wolf 424960d5fb3SKevin Wolf /* Reaquire the AioContext of bs if we dropped it */ 425960d5fb3SKevin Wolf if (ctx != co_ctx) { 426960d5fb3SKevin Wolf aio_context_acquire(ctx); 427960d5fb3SKevin Wolf } 428a77fd4bbSFam Zheng } 429a77fd4bbSFam Zheng 430dcf94a23SKevin Wolf void bdrv_do_drained_begin_quiesce(BlockDriverState *bs, 4316cd5c9d7SKevin Wolf BdrvChild *parent, bool ignore_bds_parents) 432dcf94a23SKevin Wolf { 433384a48fbSEmanuele Giuseppe Esposito IO_OR_GS_CODE(); 434dcf94a23SKevin Wolf assert(!qemu_in_coroutine()); 435dcf94a23SKevin Wolf 436dcf94a23SKevin Wolf /* Stop things in parent-to-child order */ 437d73415a3SStefan Hajnoczi if (qatomic_fetch_inc(&bs->quiesce_counter) == 0) { 438dcf94a23SKevin Wolf aio_disable_external(bdrv_get_aio_context(bs)); 439dcf94a23SKevin Wolf } 440dcf94a23SKevin Wolf 4416cd5c9d7SKevin Wolf bdrv_parent_drained_begin(bs, parent, ignore_bds_parents); 4428e1da77eSMax Reitz bdrv_drain_invoke(bs, true, NULL); 443dcf94a23SKevin Wolf } 444dcf94a23SKevin Wolf 445dcf94a23SKevin Wolf static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive, 4466cd5c9d7SKevin Wolf BdrvChild *parent, bool ignore_bds_parents, 4476cd5c9d7SKevin Wolf bool poll) 4486820643fSKevin Wolf { 449b0165585SKevin Wolf BdrvChild *child, *next; 450b0165585SKevin Wolf 451d42cf288SPaolo Bonzini if (qemu_in_coroutine()) { 4526cd5c9d7SKevin Wolf bdrv_co_yield_to_drain(bs, true, recursive, parent, ignore_bds_parents, 4538e1da77eSMax Reitz poll, NULL); 454d42cf288SPaolo Bonzini return; 455d42cf288SPaolo Bonzini } 456d42cf288SPaolo Bonzini 4576cd5c9d7SKevin Wolf bdrv_do_drained_begin_quiesce(bs, parent, ignore_bds_parents); 458d30b8e64SKevin Wolf 459b0165585SKevin Wolf if (recursive) { 4606cd5c9d7SKevin Wolf assert(!ignore_bds_parents); 461d736f119SKevin Wolf bs->recursive_quiesce_counter++; 462b0165585SKevin Wolf QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 4636cd5c9d7SKevin Wolf bdrv_do_drained_begin(child->bs, true, child, ignore_bds_parents, 4646cd5c9d7SKevin Wolf false); 465b0165585SKevin Wolf } 466b0165585SKevin Wolf } 467fe4f0614SKevin Wolf 468fe4f0614SKevin Wolf /* 469fe4f0614SKevin Wolf * Wait for drained requests to finish. 470fe4f0614SKevin Wolf * 471fe4f0614SKevin Wolf * Calling BDRV_POLL_WHILE() only once for the top-level node is okay: The 472fe4f0614SKevin Wolf * call is needed so things in this AioContext can make progress even 473fe4f0614SKevin Wolf * though we don't return to the main AioContext loop - this automatically 474fe4f0614SKevin Wolf * includes other nodes in the same AioContext and therefore all child 475fe4f0614SKevin Wolf * nodes. 476fe4f0614SKevin Wolf */ 477fe4f0614SKevin Wolf if (poll) { 4786cd5c9d7SKevin Wolf assert(!ignore_bds_parents); 479fe4f0614SKevin Wolf BDRV_POLL_WHILE(bs, bdrv_drain_poll_top_level(bs, recursive, parent)); 480fe4f0614SKevin Wolf } 4816820643fSKevin Wolf } 4826820643fSKevin Wolf 4830152bf40SKevin Wolf void bdrv_drained_begin(BlockDriverState *bs) 4840152bf40SKevin Wolf { 485384a48fbSEmanuele Giuseppe Esposito IO_OR_GS_CODE(); 4866cd5c9d7SKevin Wolf bdrv_do_drained_begin(bs, false, NULL, false, true); 4870152bf40SKevin Wolf } 4880152bf40SKevin Wolf 489b0165585SKevin Wolf void bdrv_subtree_drained_begin(BlockDriverState *bs) 4906820643fSKevin Wolf { 491384a48fbSEmanuele Giuseppe Esposito IO_OR_GS_CODE(); 4926cd5c9d7SKevin Wolf bdrv_do_drained_begin(bs, true, NULL, false, true); 493b0165585SKevin Wolf } 494b0165585SKevin Wolf 495e037c09cSMax Reitz /** 496e037c09cSMax Reitz * This function does not poll, nor must any of its recursively called 497e037c09cSMax Reitz * functions. The *drained_end_counter pointee will be incremented 498e037c09cSMax Reitz * once for every background operation scheduled, and decremented once 499e037c09cSMax Reitz * the operation settles. Therefore, the pointer must remain valid 500e037c09cSMax Reitz * until the pointee reaches 0. That implies that whoever sets up the 501e037c09cSMax Reitz * pointee has to poll until it is 0. 502e037c09cSMax Reitz * 503e037c09cSMax Reitz * We use atomic operations to access *drained_end_counter, because 504e037c09cSMax Reitz * (1) when called from bdrv_set_aio_context_ignore(), the subgraph of 505e037c09cSMax Reitz * @bs may contain nodes in different AioContexts, 506e037c09cSMax Reitz * (2) bdrv_drain_all_end() uses the same counter for all nodes, 507e037c09cSMax Reitz * regardless of which AioContext they are in. 508e037c09cSMax Reitz */ 5096cd5c9d7SKevin Wolf static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive, 5108e1da77eSMax Reitz BdrvChild *parent, bool ignore_bds_parents, 5118e1da77eSMax Reitz int *drained_end_counter) 512b0165585SKevin Wolf { 51361ad631cSMax Reitz BdrvChild *child; 5140f115168SKevin Wolf int old_quiesce_counter; 5150f115168SKevin Wolf 516e037c09cSMax Reitz assert(drained_end_counter != NULL); 517e037c09cSMax Reitz 518481cad48SManos Pitsidianakis if (qemu_in_coroutine()) { 5196cd5c9d7SKevin Wolf bdrv_co_yield_to_drain(bs, false, recursive, parent, ignore_bds_parents, 5208e1da77eSMax Reitz false, drained_end_counter); 521481cad48SManos Pitsidianakis return; 522481cad48SManos Pitsidianakis } 5236820643fSKevin Wolf assert(bs->quiesce_counter > 0); 5246820643fSKevin Wolf 52560369b86SKevin Wolf /* Re-enable things in child-to-parent order */ 5268e1da77eSMax Reitz bdrv_drain_invoke(bs, false, drained_end_counter); 527e037c09cSMax Reitz bdrv_parent_drained_end(bs, parent, ignore_bds_parents, 528e037c09cSMax Reitz drained_end_counter); 5295cb2737eSMax Reitz 530d73415a3SStefan Hajnoczi old_quiesce_counter = qatomic_fetch_dec(&bs->quiesce_counter); 5310f115168SKevin Wolf if (old_quiesce_counter == 1) { 5326820643fSKevin Wolf aio_enable_external(bdrv_get_aio_context(bs)); 5336820643fSKevin Wolf } 534b0165585SKevin Wolf 535b0165585SKevin Wolf if (recursive) { 5366cd5c9d7SKevin Wolf assert(!ignore_bds_parents); 537d736f119SKevin Wolf bs->recursive_quiesce_counter--; 53861ad631cSMax Reitz QLIST_FOREACH(child, &bs->children, next) { 5398e1da77eSMax Reitz bdrv_do_drained_end(child->bs, true, child, ignore_bds_parents, 5408e1da77eSMax Reitz drained_end_counter); 541b0165585SKevin Wolf } 542b0165585SKevin Wolf } 5430f115168SKevin Wolf } 5446820643fSKevin Wolf 5450152bf40SKevin Wolf void bdrv_drained_end(BlockDriverState *bs) 5460152bf40SKevin Wolf { 547e037c09cSMax Reitz int drained_end_counter = 0; 548384a48fbSEmanuele Giuseppe Esposito IO_OR_GS_CODE(); 549e037c09cSMax Reitz bdrv_do_drained_end(bs, false, NULL, false, &drained_end_counter); 550d73415a3SStefan Hajnoczi BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0); 551e037c09cSMax Reitz } 552e037c09cSMax Reitz 553e037c09cSMax Reitz void bdrv_drained_end_no_poll(BlockDriverState *bs, int *drained_end_counter) 554e037c09cSMax Reitz { 555384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 556e037c09cSMax Reitz bdrv_do_drained_end(bs, false, NULL, false, drained_end_counter); 557b0165585SKevin Wolf } 558b0165585SKevin Wolf 559b0165585SKevin Wolf void bdrv_subtree_drained_end(BlockDriverState *bs) 560b0165585SKevin Wolf { 561e037c09cSMax Reitz int drained_end_counter = 0; 562384a48fbSEmanuele Giuseppe Esposito IO_OR_GS_CODE(); 563e037c09cSMax Reitz bdrv_do_drained_end(bs, true, NULL, false, &drained_end_counter); 564d73415a3SStefan Hajnoczi BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0); 5650152bf40SKevin Wolf } 5660152bf40SKevin Wolf 567d736f119SKevin Wolf void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent) 568d736f119SKevin Wolf { 569d736f119SKevin Wolf int i; 570967d7905SEmanuele Giuseppe Esposito IO_OR_GS_CODE(); 571d736f119SKevin Wolf 572d736f119SKevin Wolf for (i = 0; i < new_parent->recursive_quiesce_counter; i++) { 5736cd5c9d7SKevin Wolf bdrv_do_drained_begin(child->bs, true, child, false, true); 574d736f119SKevin Wolf } 575d736f119SKevin Wolf } 576d736f119SKevin Wolf 577d736f119SKevin Wolf void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent) 578d736f119SKevin Wolf { 579e037c09cSMax Reitz int drained_end_counter = 0; 580d736f119SKevin Wolf int i; 581967d7905SEmanuele Giuseppe Esposito IO_OR_GS_CODE(); 582d736f119SKevin Wolf 583d736f119SKevin Wolf for (i = 0; i < old_parent->recursive_quiesce_counter; i++) { 584e037c09cSMax Reitz bdrv_do_drained_end(child->bs, true, child, false, 585e037c09cSMax Reitz &drained_end_counter); 586d736f119SKevin Wolf } 587e037c09cSMax Reitz 588d73415a3SStefan Hajnoczi BDRV_POLL_WHILE(child->bs, qatomic_read(&drained_end_counter) > 0); 589d736f119SKevin Wolf } 590d736f119SKevin Wolf 59161007b31SStefan Hajnoczi void bdrv_drain(BlockDriverState *bs) 59261007b31SStefan Hajnoczi { 593384a48fbSEmanuele Giuseppe Esposito IO_OR_GS_CODE(); 5946820643fSKevin Wolf bdrv_drained_begin(bs); 5956820643fSKevin Wolf bdrv_drained_end(bs); 59661007b31SStefan Hajnoczi } 59761007b31SStefan Hajnoczi 598c13ad59fSKevin Wolf static void bdrv_drain_assert_idle(BlockDriverState *bs) 599c13ad59fSKevin Wolf { 600c13ad59fSKevin Wolf BdrvChild *child, *next; 601c13ad59fSKevin Wolf 602d73415a3SStefan Hajnoczi assert(qatomic_read(&bs->in_flight) == 0); 603c13ad59fSKevin Wolf QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 604c13ad59fSKevin Wolf bdrv_drain_assert_idle(child->bs); 605c13ad59fSKevin Wolf } 606c13ad59fSKevin Wolf } 607c13ad59fSKevin Wolf 6080f12264eSKevin Wolf unsigned int bdrv_drain_all_count = 0; 6090f12264eSKevin Wolf 6100f12264eSKevin Wolf static bool bdrv_drain_all_poll(void) 6110f12264eSKevin Wolf { 6120f12264eSKevin Wolf BlockDriverState *bs = NULL; 6130f12264eSKevin Wolf bool result = false; 614f791bf7fSEmanuele Giuseppe Esposito GLOBAL_STATE_CODE(); 6150f12264eSKevin Wolf 6160f12264eSKevin Wolf /* bdrv_drain_poll() can't make changes to the graph and we are holding the 6170f12264eSKevin Wolf * main AioContext lock, so iterating bdrv_next_all_states() is safe. */ 6180f12264eSKevin Wolf while ((bs = bdrv_next_all_states(bs))) { 6190f12264eSKevin Wolf AioContext *aio_context = bdrv_get_aio_context(bs); 6200f12264eSKevin Wolf aio_context_acquire(aio_context); 6210f12264eSKevin Wolf result |= bdrv_drain_poll(bs, false, NULL, true); 6220f12264eSKevin Wolf aio_context_release(aio_context); 6230f12264eSKevin Wolf } 6240f12264eSKevin Wolf 6250f12264eSKevin Wolf return result; 6260f12264eSKevin Wolf } 6270f12264eSKevin Wolf 62861007b31SStefan Hajnoczi /* 62961007b31SStefan Hajnoczi * Wait for pending requests to complete across all BlockDriverStates 63061007b31SStefan Hajnoczi * 63161007b31SStefan Hajnoczi * This function does not flush data to disk, use bdrv_flush_all() for that 63261007b31SStefan Hajnoczi * after calling this function. 633c0778f66SAlberto Garcia * 634c0778f66SAlberto Garcia * This pauses all block jobs and disables external clients. It must 635c0778f66SAlberto Garcia * be paired with bdrv_drain_all_end(). 636c0778f66SAlberto Garcia * 637c0778f66SAlberto Garcia * NOTE: no new block jobs or BlockDriverStates can be created between 638c0778f66SAlberto Garcia * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls. 63961007b31SStefan Hajnoczi */ 640c0778f66SAlberto Garcia void bdrv_drain_all_begin(void) 64161007b31SStefan Hajnoczi { 6420f12264eSKevin Wolf BlockDriverState *bs = NULL; 643f791bf7fSEmanuele Giuseppe Esposito GLOBAL_STATE_CODE(); 64461007b31SStefan Hajnoczi 645c8ca33d0SKevin Wolf if (qemu_in_coroutine()) { 6468e1da77eSMax Reitz bdrv_co_yield_to_drain(NULL, true, false, NULL, true, true, NULL); 647c8ca33d0SKevin Wolf return; 648c8ca33d0SKevin Wolf } 649c8ca33d0SKevin Wolf 650c8aa7895SPavel Dovgalyuk /* 651c8aa7895SPavel Dovgalyuk * bdrv queue is managed by record/replay, 652c8aa7895SPavel Dovgalyuk * waiting for finishing the I/O requests may 653c8aa7895SPavel Dovgalyuk * be infinite 654c8aa7895SPavel Dovgalyuk */ 655c8aa7895SPavel Dovgalyuk if (replay_events_enabled()) { 656c8aa7895SPavel Dovgalyuk return; 657c8aa7895SPavel Dovgalyuk } 658c8aa7895SPavel Dovgalyuk 6590f12264eSKevin Wolf /* AIO_WAIT_WHILE() with a NULL context can only be called from the main 6600f12264eSKevin Wolf * loop AioContext, so make sure we're in the main context. */ 6619a7e86c8SKevin Wolf assert(qemu_get_current_aio_context() == qemu_get_aio_context()); 6620f12264eSKevin Wolf assert(bdrv_drain_all_count < INT_MAX); 6630f12264eSKevin Wolf bdrv_drain_all_count++; 6649a7e86c8SKevin Wolf 6650f12264eSKevin Wolf /* Quiesce all nodes, without polling in-flight requests yet. The graph 6660f12264eSKevin Wolf * cannot change during this loop. */ 6670f12264eSKevin Wolf while ((bs = bdrv_next_all_states(bs))) { 66861007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 66961007b31SStefan Hajnoczi 67061007b31SStefan Hajnoczi aio_context_acquire(aio_context); 6710f12264eSKevin Wolf bdrv_do_drained_begin(bs, false, NULL, true, false); 67261007b31SStefan Hajnoczi aio_context_release(aio_context); 67361007b31SStefan Hajnoczi } 67461007b31SStefan Hajnoczi 6750f12264eSKevin Wolf /* Now poll the in-flight requests */ 676cfe29d82SKevin Wolf AIO_WAIT_WHILE(NULL, bdrv_drain_all_poll()); 6770f12264eSKevin Wolf 6780f12264eSKevin Wolf while ((bs = bdrv_next_all_states(bs))) { 679c13ad59fSKevin Wolf bdrv_drain_assert_idle(bs); 680f406c03cSAlexander Yarygin } 681f406c03cSAlexander Yarygin } 682c0778f66SAlberto Garcia 6831a6d3bd2SGreg Kurz void bdrv_drain_all_end_quiesce(BlockDriverState *bs) 6841a6d3bd2SGreg Kurz { 6851a6d3bd2SGreg Kurz int drained_end_counter = 0; 686b4ad82aaSEmanuele Giuseppe Esposito GLOBAL_STATE_CODE(); 6871a6d3bd2SGreg Kurz 6881a6d3bd2SGreg Kurz g_assert(bs->quiesce_counter > 0); 6891a6d3bd2SGreg Kurz g_assert(!bs->refcnt); 6901a6d3bd2SGreg Kurz 6911a6d3bd2SGreg Kurz while (bs->quiesce_counter) { 6921a6d3bd2SGreg Kurz bdrv_do_drained_end(bs, false, NULL, true, &drained_end_counter); 6931a6d3bd2SGreg Kurz } 6941a6d3bd2SGreg Kurz BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0); 6951a6d3bd2SGreg Kurz } 6961a6d3bd2SGreg Kurz 697c0778f66SAlberto Garcia void bdrv_drain_all_end(void) 698c0778f66SAlberto Garcia { 6990f12264eSKevin Wolf BlockDriverState *bs = NULL; 700e037c09cSMax Reitz int drained_end_counter = 0; 701f791bf7fSEmanuele Giuseppe Esposito GLOBAL_STATE_CODE(); 702c0778f66SAlberto Garcia 703c8aa7895SPavel Dovgalyuk /* 704c8aa7895SPavel Dovgalyuk * bdrv queue is managed by record/replay, 705c8aa7895SPavel Dovgalyuk * waiting for finishing the I/O requests may 706c8aa7895SPavel Dovgalyuk * be endless 707c8aa7895SPavel Dovgalyuk */ 708c8aa7895SPavel Dovgalyuk if (replay_events_enabled()) { 709c8aa7895SPavel Dovgalyuk return; 710c8aa7895SPavel Dovgalyuk } 711c8aa7895SPavel Dovgalyuk 7120f12264eSKevin Wolf while ((bs = bdrv_next_all_states(bs))) { 71361007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 71461007b31SStefan Hajnoczi 71561007b31SStefan Hajnoczi aio_context_acquire(aio_context); 716e037c09cSMax Reitz bdrv_do_drained_end(bs, false, NULL, true, &drained_end_counter); 71761007b31SStefan Hajnoczi aio_context_release(aio_context); 71861007b31SStefan Hajnoczi } 7190f12264eSKevin Wolf 720e037c09cSMax Reitz assert(qemu_get_current_aio_context() == qemu_get_aio_context()); 721d73415a3SStefan Hajnoczi AIO_WAIT_WHILE(NULL, qatomic_read(&drained_end_counter) > 0); 722e037c09cSMax Reitz 7230f12264eSKevin Wolf assert(bdrv_drain_all_count > 0); 7240f12264eSKevin Wolf bdrv_drain_all_count--; 72561007b31SStefan Hajnoczi } 72661007b31SStefan Hajnoczi 727c0778f66SAlberto Garcia void bdrv_drain_all(void) 728c0778f66SAlberto Garcia { 729f791bf7fSEmanuele Giuseppe Esposito GLOBAL_STATE_CODE(); 730c0778f66SAlberto Garcia bdrv_drain_all_begin(); 731c0778f66SAlberto Garcia bdrv_drain_all_end(); 732c0778f66SAlberto Garcia } 733c0778f66SAlberto Garcia 73461007b31SStefan Hajnoczi /** 73561007b31SStefan Hajnoczi * Remove an active request from the tracked requests list 73661007b31SStefan Hajnoczi * 73761007b31SStefan Hajnoczi * This function should be called when a tracked request is completing. 73861007b31SStefan Hajnoczi */ 739f0d43b1eSPaolo Bonzini static void coroutine_fn tracked_request_end(BdrvTrackedRequest *req) 74061007b31SStefan Hajnoczi { 74161007b31SStefan Hajnoczi if (req->serialising) { 742d73415a3SStefan Hajnoczi qatomic_dec(&req->bs->serialising_in_flight); 74361007b31SStefan Hajnoczi } 74461007b31SStefan Hajnoczi 7453783fa3dSPaolo Bonzini qemu_co_mutex_lock(&req->bs->reqs_lock); 74661007b31SStefan Hajnoczi QLIST_REMOVE(req, list); 74761007b31SStefan Hajnoczi qemu_co_queue_restart_all(&req->wait_queue); 7483783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&req->bs->reqs_lock); 74961007b31SStefan Hajnoczi } 75061007b31SStefan Hajnoczi 75161007b31SStefan Hajnoczi /** 75261007b31SStefan Hajnoczi * Add an active request to the tracked requests list 75361007b31SStefan Hajnoczi */ 754881a4c55SPaolo Bonzini static void coroutine_fn tracked_request_begin(BdrvTrackedRequest *req, 75561007b31SStefan Hajnoczi BlockDriverState *bs, 75661007b31SStefan Hajnoczi int64_t offset, 75780247264SEric Blake int64_t bytes, 758ebde595cSFam Zheng enum BdrvTrackedRequestType type) 75961007b31SStefan Hajnoczi { 76080247264SEric Blake bdrv_check_request(offset, bytes, &error_abort); 76122931a15SFam Zheng 76261007b31SStefan Hajnoczi *req = (BdrvTrackedRequest){ 76361007b31SStefan Hajnoczi .bs = bs, 76461007b31SStefan Hajnoczi .offset = offset, 76561007b31SStefan Hajnoczi .bytes = bytes, 766ebde595cSFam Zheng .type = type, 76761007b31SStefan Hajnoczi .co = qemu_coroutine_self(), 76861007b31SStefan Hajnoczi .serialising = false, 76961007b31SStefan Hajnoczi .overlap_offset = offset, 77061007b31SStefan Hajnoczi .overlap_bytes = bytes, 77161007b31SStefan Hajnoczi }; 77261007b31SStefan Hajnoczi 77361007b31SStefan Hajnoczi qemu_co_queue_init(&req->wait_queue); 77461007b31SStefan Hajnoczi 7753783fa3dSPaolo Bonzini qemu_co_mutex_lock(&bs->reqs_lock); 77661007b31SStefan Hajnoczi QLIST_INSERT_HEAD(&bs->tracked_requests, req, list); 7773783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&bs->reqs_lock); 77861007b31SStefan Hajnoczi } 77961007b31SStefan Hajnoczi 7803ba0e1a0SPaolo Bonzini static bool tracked_request_overlaps(BdrvTrackedRequest *req, 78180247264SEric Blake int64_t offset, int64_t bytes) 7823ba0e1a0SPaolo Bonzini { 78380247264SEric Blake bdrv_check_request(offset, bytes, &error_abort); 78480247264SEric Blake 7853ba0e1a0SPaolo Bonzini /* aaaa bbbb */ 7863ba0e1a0SPaolo Bonzini if (offset >= req->overlap_offset + req->overlap_bytes) { 7873ba0e1a0SPaolo Bonzini return false; 7883ba0e1a0SPaolo Bonzini } 7893ba0e1a0SPaolo Bonzini /* bbbb aaaa */ 7903ba0e1a0SPaolo Bonzini if (req->overlap_offset >= offset + bytes) { 7913ba0e1a0SPaolo Bonzini return false; 7923ba0e1a0SPaolo Bonzini } 7933ba0e1a0SPaolo Bonzini return true; 7943ba0e1a0SPaolo Bonzini } 7953ba0e1a0SPaolo Bonzini 7963183937fSVladimir Sementsov-Ogievskiy /* Called with self->bs->reqs_lock held */ 797881a4c55SPaolo Bonzini static coroutine_fn BdrvTrackedRequest * 7983183937fSVladimir Sementsov-Ogievskiy bdrv_find_conflicting_request(BdrvTrackedRequest *self) 7993ba0e1a0SPaolo Bonzini { 8003ba0e1a0SPaolo Bonzini BdrvTrackedRequest *req; 8013ba0e1a0SPaolo Bonzini 8023183937fSVladimir Sementsov-Ogievskiy QLIST_FOREACH(req, &self->bs->tracked_requests, list) { 8033ba0e1a0SPaolo Bonzini if (req == self || (!req->serialising && !self->serialising)) { 8043ba0e1a0SPaolo Bonzini continue; 8053ba0e1a0SPaolo Bonzini } 8063ba0e1a0SPaolo Bonzini if (tracked_request_overlaps(req, self->overlap_offset, 8073ba0e1a0SPaolo Bonzini self->overlap_bytes)) 8083ba0e1a0SPaolo Bonzini { 8093183937fSVladimir Sementsov-Ogievskiy /* 8103183937fSVladimir Sementsov-Ogievskiy * Hitting this means there was a reentrant request, for 8113ba0e1a0SPaolo Bonzini * example, a block driver issuing nested requests. This must 8123ba0e1a0SPaolo Bonzini * never happen since it means deadlock. 8133ba0e1a0SPaolo Bonzini */ 8143ba0e1a0SPaolo Bonzini assert(qemu_coroutine_self() != req->co); 8153ba0e1a0SPaolo Bonzini 8163183937fSVladimir Sementsov-Ogievskiy /* 8173183937fSVladimir Sementsov-Ogievskiy * If the request is already (indirectly) waiting for us, or 8183ba0e1a0SPaolo Bonzini * will wait for us as soon as it wakes up, then just go on 8193183937fSVladimir Sementsov-Ogievskiy * (instead of producing a deadlock in the former case). 8203183937fSVladimir Sementsov-Ogievskiy */ 8213ba0e1a0SPaolo Bonzini if (!req->waiting_for) { 8223183937fSVladimir Sementsov-Ogievskiy return req; 8233183937fSVladimir Sementsov-Ogievskiy } 8243183937fSVladimir Sementsov-Ogievskiy } 8253183937fSVladimir Sementsov-Ogievskiy } 8263183937fSVladimir Sementsov-Ogievskiy 8273183937fSVladimir Sementsov-Ogievskiy return NULL; 8283183937fSVladimir Sementsov-Ogievskiy } 8293183937fSVladimir Sementsov-Ogievskiy 830ec1c8868SVladimir Sementsov-Ogievskiy /* Called with self->bs->reqs_lock held */ 831131498f7SDenis V. Lunev static void coroutine_fn 832ec1c8868SVladimir Sementsov-Ogievskiy bdrv_wait_serialising_requests_locked(BdrvTrackedRequest *self) 8333183937fSVladimir Sementsov-Ogievskiy { 8343183937fSVladimir Sementsov-Ogievskiy BdrvTrackedRequest *req; 8353183937fSVladimir Sementsov-Ogievskiy 8363183937fSVladimir Sementsov-Ogievskiy while ((req = bdrv_find_conflicting_request(self))) { 8373ba0e1a0SPaolo Bonzini self->waiting_for = req; 838ec1c8868SVladimir Sementsov-Ogievskiy qemu_co_queue_wait(&req->wait_queue, &self->bs->reqs_lock); 8393ba0e1a0SPaolo Bonzini self->waiting_for = NULL; 8403ba0e1a0SPaolo Bonzini } 8413ba0e1a0SPaolo Bonzini } 8423ba0e1a0SPaolo Bonzini 8438ac5aab2SVladimir Sementsov-Ogievskiy /* Called with req->bs->reqs_lock held */ 8448ac5aab2SVladimir Sementsov-Ogievskiy static void tracked_request_set_serialising(BdrvTrackedRequest *req, 8458ac5aab2SVladimir Sementsov-Ogievskiy uint64_t align) 84661007b31SStefan Hajnoczi { 84761007b31SStefan Hajnoczi int64_t overlap_offset = req->offset & ~(align - 1); 84880247264SEric Blake int64_t overlap_bytes = 84980247264SEric Blake ROUND_UP(req->offset + req->bytes, align) - overlap_offset; 85080247264SEric Blake 85180247264SEric Blake bdrv_check_request(req->offset, req->bytes, &error_abort); 85261007b31SStefan Hajnoczi 85361007b31SStefan Hajnoczi if (!req->serialising) { 854d73415a3SStefan Hajnoczi qatomic_inc(&req->bs->serialising_in_flight); 85561007b31SStefan Hajnoczi req->serialising = true; 85661007b31SStefan Hajnoczi } 85761007b31SStefan Hajnoczi 85861007b31SStefan Hajnoczi req->overlap_offset = MIN(req->overlap_offset, overlap_offset); 85961007b31SStefan Hajnoczi req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes); 86009d2f948SVladimir Sementsov-Ogievskiy } 86109d2f948SVladimir Sementsov-Ogievskiy 86261007b31SStefan Hajnoczi /** 863c28107e9SMax Reitz * Return the tracked request on @bs for the current coroutine, or 864c28107e9SMax Reitz * NULL if there is none. 865c28107e9SMax Reitz */ 866c28107e9SMax Reitz BdrvTrackedRequest *coroutine_fn bdrv_co_get_self_request(BlockDriverState *bs) 867c28107e9SMax Reitz { 868c28107e9SMax Reitz BdrvTrackedRequest *req; 869c28107e9SMax Reitz Coroutine *self = qemu_coroutine_self(); 870967d7905SEmanuele Giuseppe Esposito IO_CODE(); 871c28107e9SMax Reitz 872c28107e9SMax Reitz QLIST_FOREACH(req, &bs->tracked_requests, list) { 873c28107e9SMax Reitz if (req->co == self) { 874c28107e9SMax Reitz return req; 875c28107e9SMax Reitz } 876c28107e9SMax Reitz } 877c28107e9SMax Reitz 878c28107e9SMax Reitz return NULL; 879c28107e9SMax Reitz } 880c28107e9SMax Reitz 881c28107e9SMax Reitz /** 882244483e6SKevin Wolf * Round a region to cluster boundaries 883244483e6SKevin Wolf */ 884244483e6SKevin Wolf void bdrv_round_to_clusters(BlockDriverState *bs, 8857cfd5275SEric Blake int64_t offset, int64_t bytes, 886244483e6SKevin Wolf int64_t *cluster_offset, 8877cfd5275SEric Blake int64_t *cluster_bytes) 888244483e6SKevin Wolf { 889244483e6SKevin Wolf BlockDriverInfo bdi; 890384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 891244483e6SKevin Wolf if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 892244483e6SKevin Wolf *cluster_offset = offset; 893244483e6SKevin Wolf *cluster_bytes = bytes; 894244483e6SKevin Wolf } else { 895244483e6SKevin Wolf int64_t c = bdi.cluster_size; 896244483e6SKevin Wolf *cluster_offset = QEMU_ALIGN_DOWN(offset, c); 897244483e6SKevin Wolf *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c); 898244483e6SKevin Wolf } 899244483e6SKevin Wolf } 900244483e6SKevin Wolf 90161007b31SStefan Hajnoczi static int bdrv_get_cluster_size(BlockDriverState *bs) 90261007b31SStefan Hajnoczi { 90361007b31SStefan Hajnoczi BlockDriverInfo bdi; 90461007b31SStefan Hajnoczi int ret; 90561007b31SStefan Hajnoczi 90661007b31SStefan Hajnoczi ret = bdrv_get_info(bs, &bdi); 90761007b31SStefan Hajnoczi if (ret < 0 || bdi.cluster_size == 0) { 908a5b8dd2cSEric Blake return bs->bl.request_alignment; 90961007b31SStefan Hajnoczi } else { 91061007b31SStefan Hajnoczi return bdi.cluster_size; 91161007b31SStefan Hajnoczi } 91261007b31SStefan Hajnoczi } 91361007b31SStefan Hajnoczi 91499723548SPaolo Bonzini void bdrv_inc_in_flight(BlockDriverState *bs) 91599723548SPaolo Bonzini { 916967d7905SEmanuele Giuseppe Esposito IO_CODE(); 917d73415a3SStefan Hajnoczi qatomic_inc(&bs->in_flight); 91899723548SPaolo Bonzini } 91999723548SPaolo Bonzini 920c9d1a561SPaolo Bonzini void bdrv_wakeup(BlockDriverState *bs) 921c9d1a561SPaolo Bonzini { 922967d7905SEmanuele Giuseppe Esposito IO_CODE(); 923cfe29d82SKevin Wolf aio_wait_kick(); 924c9d1a561SPaolo Bonzini } 925c9d1a561SPaolo Bonzini 92699723548SPaolo Bonzini void bdrv_dec_in_flight(BlockDriverState *bs) 92799723548SPaolo Bonzini { 928967d7905SEmanuele Giuseppe Esposito IO_CODE(); 929d73415a3SStefan Hajnoczi qatomic_dec(&bs->in_flight); 930c9d1a561SPaolo Bonzini bdrv_wakeup(bs); 93199723548SPaolo Bonzini } 93299723548SPaolo Bonzini 933131498f7SDenis V. Lunev static void coroutine_fn 934131498f7SDenis V. Lunev bdrv_wait_serialising_requests(BdrvTrackedRequest *self) 93561007b31SStefan Hajnoczi { 93661007b31SStefan Hajnoczi BlockDriverState *bs = self->bs; 93761007b31SStefan Hajnoczi 938d73415a3SStefan Hajnoczi if (!qatomic_read(&bs->serialising_in_flight)) { 939131498f7SDenis V. Lunev return; 94061007b31SStefan Hajnoczi } 94161007b31SStefan Hajnoczi 9423783fa3dSPaolo Bonzini qemu_co_mutex_lock(&bs->reqs_lock); 943131498f7SDenis V. Lunev bdrv_wait_serialising_requests_locked(self); 9443783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&bs->reqs_lock); 94561007b31SStefan Hajnoczi } 94661007b31SStefan Hajnoczi 947131498f7SDenis V. Lunev void coroutine_fn bdrv_make_request_serialising(BdrvTrackedRequest *req, 9488ac5aab2SVladimir Sementsov-Ogievskiy uint64_t align) 9498ac5aab2SVladimir Sementsov-Ogievskiy { 950967d7905SEmanuele Giuseppe Esposito IO_CODE(); 9518ac5aab2SVladimir Sementsov-Ogievskiy 9528ac5aab2SVladimir Sementsov-Ogievskiy qemu_co_mutex_lock(&req->bs->reqs_lock); 9538ac5aab2SVladimir Sementsov-Ogievskiy 9548ac5aab2SVladimir Sementsov-Ogievskiy tracked_request_set_serialising(req, align); 955131498f7SDenis V. Lunev bdrv_wait_serialising_requests_locked(req); 9568ac5aab2SVladimir Sementsov-Ogievskiy 9578ac5aab2SVladimir Sementsov-Ogievskiy qemu_co_mutex_unlock(&req->bs->reqs_lock); 9588ac5aab2SVladimir Sementsov-Ogievskiy } 9598ac5aab2SVladimir Sementsov-Ogievskiy 960558902ccSVladimir Sementsov-Ogievskiy int bdrv_check_qiov_request(int64_t offset, int64_t bytes, 96163f4ad11SVladimir Sementsov-Ogievskiy QEMUIOVector *qiov, size_t qiov_offset, 96263f4ad11SVladimir Sementsov-Ogievskiy Error **errp) 96361007b31SStefan Hajnoczi { 96463f4ad11SVladimir Sementsov-Ogievskiy /* 96563f4ad11SVladimir Sementsov-Ogievskiy * Check generic offset/bytes correctness 96663f4ad11SVladimir Sementsov-Ogievskiy */ 96763f4ad11SVladimir Sementsov-Ogievskiy 96869b55e03SVladimir Sementsov-Ogievskiy if (offset < 0) { 96969b55e03SVladimir Sementsov-Ogievskiy error_setg(errp, "offset is negative: %" PRIi64, offset); 97069b55e03SVladimir Sementsov-Ogievskiy return -EIO; 97169b55e03SVladimir Sementsov-Ogievskiy } 97269b55e03SVladimir Sementsov-Ogievskiy 97369b55e03SVladimir Sementsov-Ogievskiy if (bytes < 0) { 97469b55e03SVladimir Sementsov-Ogievskiy error_setg(errp, "bytes is negative: %" PRIi64, bytes); 97561007b31SStefan Hajnoczi return -EIO; 97661007b31SStefan Hajnoczi } 97761007b31SStefan Hajnoczi 9788b117001SVladimir Sementsov-Ogievskiy if (bytes > BDRV_MAX_LENGTH) { 97969b55e03SVladimir Sementsov-Ogievskiy error_setg(errp, "bytes(%" PRIi64 ") exceeds maximum(%" PRIi64 ")", 98069b55e03SVladimir Sementsov-Ogievskiy bytes, BDRV_MAX_LENGTH); 98169b55e03SVladimir Sementsov-Ogievskiy return -EIO; 98269b55e03SVladimir Sementsov-Ogievskiy } 98369b55e03SVladimir Sementsov-Ogievskiy 98469b55e03SVladimir Sementsov-Ogievskiy if (offset > BDRV_MAX_LENGTH) { 98569b55e03SVladimir Sementsov-Ogievskiy error_setg(errp, "offset(%" PRIi64 ") exceeds maximum(%" PRIi64 ")", 98669b55e03SVladimir Sementsov-Ogievskiy offset, BDRV_MAX_LENGTH); 9878b117001SVladimir Sementsov-Ogievskiy return -EIO; 9888b117001SVladimir Sementsov-Ogievskiy } 9898b117001SVladimir Sementsov-Ogievskiy 9908b117001SVladimir Sementsov-Ogievskiy if (offset > BDRV_MAX_LENGTH - bytes) { 99169b55e03SVladimir Sementsov-Ogievskiy error_setg(errp, "sum of offset(%" PRIi64 ") and bytes(%" PRIi64 ") " 99269b55e03SVladimir Sementsov-Ogievskiy "exceeds maximum(%" PRIi64 ")", offset, bytes, 99369b55e03SVladimir Sementsov-Ogievskiy BDRV_MAX_LENGTH); 9948b117001SVladimir Sementsov-Ogievskiy return -EIO; 9958b117001SVladimir Sementsov-Ogievskiy } 9968b117001SVladimir Sementsov-Ogievskiy 99763f4ad11SVladimir Sementsov-Ogievskiy if (!qiov) { 9988b117001SVladimir Sementsov-Ogievskiy return 0; 9998b117001SVladimir Sementsov-Ogievskiy } 10008b117001SVladimir Sementsov-Ogievskiy 100163f4ad11SVladimir Sementsov-Ogievskiy /* 100263f4ad11SVladimir Sementsov-Ogievskiy * Check qiov and qiov_offset 100363f4ad11SVladimir Sementsov-Ogievskiy */ 100463f4ad11SVladimir Sementsov-Ogievskiy 100563f4ad11SVladimir Sementsov-Ogievskiy if (qiov_offset > qiov->size) { 100663f4ad11SVladimir Sementsov-Ogievskiy error_setg(errp, "qiov_offset(%zu) overflow io vector size(%zu)", 100763f4ad11SVladimir Sementsov-Ogievskiy qiov_offset, qiov->size); 100863f4ad11SVladimir Sementsov-Ogievskiy return -EIO; 100963f4ad11SVladimir Sementsov-Ogievskiy } 101063f4ad11SVladimir Sementsov-Ogievskiy 101163f4ad11SVladimir Sementsov-Ogievskiy if (bytes > qiov->size - qiov_offset) { 101263f4ad11SVladimir Sementsov-Ogievskiy error_setg(errp, "bytes(%" PRIi64 ") + qiov_offset(%zu) overflow io " 101363f4ad11SVladimir Sementsov-Ogievskiy "vector size(%zu)", bytes, qiov_offset, qiov->size); 101463f4ad11SVladimir Sementsov-Ogievskiy return -EIO; 101563f4ad11SVladimir Sementsov-Ogievskiy } 101663f4ad11SVladimir Sementsov-Ogievskiy 101763f4ad11SVladimir Sementsov-Ogievskiy return 0; 101863f4ad11SVladimir Sementsov-Ogievskiy } 101963f4ad11SVladimir Sementsov-Ogievskiy 102063f4ad11SVladimir Sementsov-Ogievskiy int bdrv_check_request(int64_t offset, int64_t bytes, Error **errp) 10218b117001SVladimir Sementsov-Ogievskiy { 102263f4ad11SVladimir Sementsov-Ogievskiy return bdrv_check_qiov_request(offset, bytes, NULL, 0, errp); 102363f4ad11SVladimir Sementsov-Ogievskiy } 102463f4ad11SVladimir Sementsov-Ogievskiy 102563f4ad11SVladimir Sementsov-Ogievskiy static int bdrv_check_request32(int64_t offset, int64_t bytes, 102663f4ad11SVladimir Sementsov-Ogievskiy QEMUIOVector *qiov, size_t qiov_offset) 102763f4ad11SVladimir Sementsov-Ogievskiy { 102863f4ad11SVladimir Sementsov-Ogievskiy int ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL); 10298b117001SVladimir Sementsov-Ogievskiy if (ret < 0) { 10308b117001SVladimir Sementsov-Ogievskiy return ret; 10318b117001SVladimir Sementsov-Ogievskiy } 10328b117001SVladimir Sementsov-Ogievskiy 10338b117001SVladimir Sementsov-Ogievskiy if (bytes > BDRV_REQUEST_MAX_BYTES) { 103461007b31SStefan Hajnoczi return -EIO; 103561007b31SStefan Hajnoczi } 103661007b31SStefan Hajnoczi 103761007b31SStefan Hajnoczi return 0; 103861007b31SStefan Hajnoczi } 103961007b31SStefan Hajnoczi 104061007b31SStefan Hajnoczi /* 104174021bc4SEric Blake * Completely zero out a block device with the help of bdrv_pwrite_zeroes. 104261007b31SStefan Hajnoczi * The operation is sped up by checking the block status and only writing 104361007b31SStefan Hajnoczi * zeroes to the device if they currently do not return zeroes. Optional 104474021bc4SEric Blake * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP, 1045465fe887SEric Blake * BDRV_REQ_FUA). 104661007b31SStefan Hajnoczi * 1047f4649069SEric Blake * Returns < 0 on error, 0 on success. For error codes see bdrv_pwrite(). 104861007b31SStefan Hajnoczi */ 1049720ff280SKevin Wolf int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags) 105061007b31SStefan Hajnoczi { 1051237d78f8SEric Blake int ret; 1052237d78f8SEric Blake int64_t target_size, bytes, offset = 0; 1053720ff280SKevin Wolf BlockDriverState *bs = child->bs; 1054384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 105561007b31SStefan Hajnoczi 10567286d610SEric Blake target_size = bdrv_getlength(bs); 10577286d610SEric Blake if (target_size < 0) { 10587286d610SEric Blake return target_size; 105961007b31SStefan Hajnoczi } 106061007b31SStefan Hajnoczi 106161007b31SStefan Hajnoczi for (;;) { 10627286d610SEric Blake bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES); 10637286d610SEric Blake if (bytes <= 0) { 106461007b31SStefan Hajnoczi return 0; 106561007b31SStefan Hajnoczi } 1066237d78f8SEric Blake ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL); 106761007b31SStefan Hajnoczi if (ret < 0) { 106861007b31SStefan Hajnoczi return ret; 106961007b31SStefan Hajnoczi } 107061007b31SStefan Hajnoczi if (ret & BDRV_BLOCK_ZERO) { 1071237d78f8SEric Blake offset += bytes; 107261007b31SStefan Hajnoczi continue; 107361007b31SStefan Hajnoczi } 1074237d78f8SEric Blake ret = bdrv_pwrite_zeroes(child, offset, bytes, flags); 107561007b31SStefan Hajnoczi if (ret < 0) { 107661007b31SStefan Hajnoczi return ret; 107761007b31SStefan Hajnoczi } 1078237d78f8SEric Blake offset += bytes; 107961007b31SStefan Hajnoczi } 108061007b31SStefan Hajnoczi } 108161007b31SStefan Hajnoczi 108261007b31SStefan Hajnoczi /* 108361007b31SStefan Hajnoczi * Writes to the file and ensures that no writes are reordered across this 108461007b31SStefan Hajnoczi * request (acts as a barrier) 108561007b31SStefan Hajnoczi * 108661007b31SStefan Hajnoczi * Returns 0 on success, -errno in error cases. 108761007b31SStefan Hajnoczi */ 1088e97190a4SAlberto Faria int coroutine_fn bdrv_co_pwrite_sync(BdrvChild *child, int64_t offset, 1089e97190a4SAlberto Faria int64_t bytes, const void *buf, 1090e97190a4SAlberto Faria BdrvRequestFlags flags) 109161007b31SStefan Hajnoczi { 109261007b31SStefan Hajnoczi int ret; 1093384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 109461007b31SStefan Hajnoczi 1095e97190a4SAlberto Faria ret = bdrv_co_pwrite(child, offset, bytes, buf, flags); 109661007b31SStefan Hajnoczi if (ret < 0) { 109761007b31SStefan Hajnoczi return ret; 109861007b31SStefan Hajnoczi } 109961007b31SStefan Hajnoczi 1100e97190a4SAlberto Faria ret = bdrv_co_flush(child->bs); 1101855a6a93SKevin Wolf if (ret < 0) { 1102855a6a93SKevin Wolf return ret; 110361007b31SStefan Hajnoczi } 110461007b31SStefan Hajnoczi 110561007b31SStefan Hajnoczi return 0; 110661007b31SStefan Hajnoczi } 110761007b31SStefan Hajnoczi 110808844473SKevin Wolf typedef struct CoroutineIOCompletion { 110908844473SKevin Wolf Coroutine *coroutine; 111008844473SKevin Wolf int ret; 111108844473SKevin Wolf } CoroutineIOCompletion; 111208844473SKevin Wolf 111308844473SKevin Wolf static void bdrv_co_io_em_complete(void *opaque, int ret) 111408844473SKevin Wolf { 111508844473SKevin Wolf CoroutineIOCompletion *co = opaque; 111608844473SKevin Wolf 111708844473SKevin Wolf co->ret = ret; 1118b9e413ddSPaolo Bonzini aio_co_wake(co->coroutine); 111908844473SKevin Wolf } 112008844473SKevin Wolf 1121166fe960SKevin Wolf static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs, 112217abcbeeSVladimir Sementsov-Ogievskiy int64_t offset, int64_t bytes, 1123ac850bf0SVladimir Sementsov-Ogievskiy QEMUIOVector *qiov, 1124ac850bf0SVladimir Sementsov-Ogievskiy size_t qiov_offset, int flags) 1125166fe960SKevin Wolf { 1126166fe960SKevin Wolf BlockDriver *drv = bs->drv; 11273fb06697SKevin Wolf int64_t sector_num; 11283fb06697SKevin Wolf unsigned int nb_sectors; 1129ac850bf0SVladimir Sementsov-Ogievskiy QEMUIOVector local_qiov; 1130ac850bf0SVladimir Sementsov-Ogievskiy int ret; 11313fb06697SKevin Wolf 113217abcbeeSVladimir Sementsov-Ogievskiy bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1133fa166538SEric Blake assert(!(flags & ~BDRV_REQ_MASK)); 1134fe0480d6SKevin Wolf assert(!(flags & BDRV_REQ_NO_FALLBACK)); 1135fa166538SEric Blake 1136d470ad42SMax Reitz if (!drv) { 1137d470ad42SMax Reitz return -ENOMEDIUM; 1138d470ad42SMax Reitz } 1139d470ad42SMax Reitz 1140ac850bf0SVladimir Sementsov-Ogievskiy if (drv->bdrv_co_preadv_part) { 1141ac850bf0SVladimir Sementsov-Ogievskiy return drv->bdrv_co_preadv_part(bs, offset, bytes, qiov, qiov_offset, 1142ac850bf0SVladimir Sementsov-Ogievskiy flags); 1143ac850bf0SVladimir Sementsov-Ogievskiy } 1144ac850bf0SVladimir Sementsov-Ogievskiy 1145ac850bf0SVladimir Sementsov-Ogievskiy if (qiov_offset > 0 || bytes != qiov->size) { 1146ac850bf0SVladimir Sementsov-Ogievskiy qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1147ac850bf0SVladimir Sementsov-Ogievskiy qiov = &local_qiov; 1148ac850bf0SVladimir Sementsov-Ogievskiy } 1149ac850bf0SVladimir Sementsov-Ogievskiy 11503fb06697SKevin Wolf if (drv->bdrv_co_preadv) { 1151ac850bf0SVladimir Sementsov-Ogievskiy ret = drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags); 1152ac850bf0SVladimir Sementsov-Ogievskiy goto out; 11533fb06697SKevin Wolf } 11543fb06697SKevin Wolf 1155edfab6a0SEric Blake if (drv->bdrv_aio_preadv) { 115608844473SKevin Wolf BlockAIOCB *acb; 115708844473SKevin Wolf CoroutineIOCompletion co = { 115808844473SKevin Wolf .coroutine = qemu_coroutine_self(), 115908844473SKevin Wolf }; 116008844473SKevin Wolf 1161e31f6864SEric Blake acb = drv->bdrv_aio_preadv(bs, offset, bytes, qiov, flags, 116208844473SKevin Wolf bdrv_co_io_em_complete, &co); 116308844473SKevin Wolf if (acb == NULL) { 1164ac850bf0SVladimir Sementsov-Ogievskiy ret = -EIO; 1165ac850bf0SVladimir Sementsov-Ogievskiy goto out; 116608844473SKevin Wolf } else { 116708844473SKevin Wolf qemu_coroutine_yield(); 1168ac850bf0SVladimir Sementsov-Ogievskiy ret = co.ret; 1169ac850bf0SVladimir Sementsov-Ogievskiy goto out; 117008844473SKevin Wolf } 117108844473SKevin Wolf } 1172edfab6a0SEric Blake 1173edfab6a0SEric Blake sector_num = offset >> BDRV_SECTOR_BITS; 1174edfab6a0SEric Blake nb_sectors = bytes >> BDRV_SECTOR_BITS; 1175edfab6a0SEric Blake 11761bbbf32dSNir Soffer assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE)); 11771bbbf32dSNir Soffer assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE)); 117841ae31e3SAlberto Garcia assert(bytes <= BDRV_REQUEST_MAX_BYTES); 1179edfab6a0SEric Blake assert(drv->bdrv_co_readv); 1180edfab6a0SEric Blake 1181ac850bf0SVladimir Sementsov-Ogievskiy ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov); 1182ac850bf0SVladimir Sementsov-Ogievskiy 1183ac850bf0SVladimir Sementsov-Ogievskiy out: 1184ac850bf0SVladimir Sementsov-Ogievskiy if (qiov == &local_qiov) { 1185ac850bf0SVladimir Sementsov-Ogievskiy qemu_iovec_destroy(&local_qiov); 1186ac850bf0SVladimir Sementsov-Ogievskiy } 1187ac850bf0SVladimir Sementsov-Ogievskiy 1188ac850bf0SVladimir Sementsov-Ogievskiy return ret; 1189166fe960SKevin Wolf } 1190166fe960SKevin Wolf 119178a07294SKevin Wolf static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs, 119217abcbeeSVladimir Sementsov-Ogievskiy int64_t offset, int64_t bytes, 1193ac850bf0SVladimir Sementsov-Ogievskiy QEMUIOVector *qiov, 1194e75abedaSVladimir Sementsov-Ogievskiy size_t qiov_offset, 1195e75abedaSVladimir Sementsov-Ogievskiy BdrvRequestFlags flags) 119678a07294SKevin Wolf { 119778a07294SKevin Wolf BlockDriver *drv = bs->drv; 11983fb06697SKevin Wolf int64_t sector_num; 11993fb06697SKevin Wolf unsigned int nb_sectors; 1200ac850bf0SVladimir Sementsov-Ogievskiy QEMUIOVector local_qiov; 120178a07294SKevin Wolf int ret; 120278a07294SKevin Wolf 120317abcbeeSVladimir Sementsov-Ogievskiy bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1204fa166538SEric Blake assert(!(flags & ~BDRV_REQ_MASK)); 1205fe0480d6SKevin Wolf assert(!(flags & BDRV_REQ_NO_FALLBACK)); 1206fa166538SEric Blake 1207d470ad42SMax Reitz if (!drv) { 1208d470ad42SMax Reitz return -ENOMEDIUM; 1209d470ad42SMax Reitz } 1210d470ad42SMax Reitz 1211ac850bf0SVladimir Sementsov-Ogievskiy if (drv->bdrv_co_pwritev_part) { 1212ac850bf0SVladimir Sementsov-Ogievskiy ret = drv->bdrv_co_pwritev_part(bs, offset, bytes, qiov, qiov_offset, 1213ac850bf0SVladimir Sementsov-Ogievskiy flags & bs->supported_write_flags); 1214ac850bf0SVladimir Sementsov-Ogievskiy flags &= ~bs->supported_write_flags; 1215ac850bf0SVladimir Sementsov-Ogievskiy goto emulate_flags; 1216ac850bf0SVladimir Sementsov-Ogievskiy } 1217ac850bf0SVladimir Sementsov-Ogievskiy 1218ac850bf0SVladimir Sementsov-Ogievskiy if (qiov_offset > 0 || bytes != qiov->size) { 1219ac850bf0SVladimir Sementsov-Ogievskiy qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1220ac850bf0SVladimir Sementsov-Ogievskiy qiov = &local_qiov; 1221ac850bf0SVladimir Sementsov-Ogievskiy } 1222ac850bf0SVladimir Sementsov-Ogievskiy 12233fb06697SKevin Wolf if (drv->bdrv_co_pwritev) { 1224515c2f43SKevin Wolf ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, 1225515c2f43SKevin Wolf flags & bs->supported_write_flags); 1226515c2f43SKevin Wolf flags &= ~bs->supported_write_flags; 12273fb06697SKevin Wolf goto emulate_flags; 12283fb06697SKevin Wolf } 12293fb06697SKevin Wolf 1230edfab6a0SEric Blake if (drv->bdrv_aio_pwritev) { 123108844473SKevin Wolf BlockAIOCB *acb; 123208844473SKevin Wolf CoroutineIOCompletion co = { 123308844473SKevin Wolf .coroutine = qemu_coroutine_self(), 123408844473SKevin Wolf }; 123508844473SKevin Wolf 1236e31f6864SEric Blake acb = drv->bdrv_aio_pwritev(bs, offset, bytes, qiov, 1237e31f6864SEric Blake flags & bs->supported_write_flags, 123808844473SKevin Wolf bdrv_co_io_em_complete, &co); 1239e31f6864SEric Blake flags &= ~bs->supported_write_flags; 124008844473SKevin Wolf if (acb == NULL) { 12413fb06697SKevin Wolf ret = -EIO; 124208844473SKevin Wolf } else { 124308844473SKevin Wolf qemu_coroutine_yield(); 12443fb06697SKevin Wolf ret = co.ret; 124508844473SKevin Wolf } 1246edfab6a0SEric Blake goto emulate_flags; 1247edfab6a0SEric Blake } 1248edfab6a0SEric Blake 1249edfab6a0SEric Blake sector_num = offset >> BDRV_SECTOR_BITS; 1250edfab6a0SEric Blake nb_sectors = bytes >> BDRV_SECTOR_BITS; 1251edfab6a0SEric Blake 12521bbbf32dSNir Soffer assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE)); 12531bbbf32dSNir Soffer assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE)); 125441ae31e3SAlberto Garcia assert(bytes <= BDRV_REQUEST_MAX_BYTES); 1255edfab6a0SEric Blake 1256e18a58b4SEric Blake assert(drv->bdrv_co_writev); 1257e18a58b4SEric Blake ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov, 1258edfab6a0SEric Blake flags & bs->supported_write_flags); 1259edfab6a0SEric Blake flags &= ~bs->supported_write_flags; 126078a07294SKevin Wolf 12613fb06697SKevin Wolf emulate_flags: 12624df863f3SEric Blake if (ret == 0 && (flags & BDRV_REQ_FUA)) { 126378a07294SKevin Wolf ret = bdrv_co_flush(bs); 126478a07294SKevin Wolf } 126578a07294SKevin Wolf 1266ac850bf0SVladimir Sementsov-Ogievskiy if (qiov == &local_qiov) { 1267ac850bf0SVladimir Sementsov-Ogievskiy qemu_iovec_destroy(&local_qiov); 1268ac850bf0SVladimir Sementsov-Ogievskiy } 1269ac850bf0SVladimir Sementsov-Ogievskiy 127078a07294SKevin Wolf return ret; 127178a07294SKevin Wolf } 127278a07294SKevin Wolf 127329a298afSPavel Butsykin static int coroutine_fn 127417abcbeeSVladimir Sementsov-Ogievskiy bdrv_driver_pwritev_compressed(BlockDriverState *bs, int64_t offset, 127517abcbeeSVladimir Sementsov-Ogievskiy int64_t bytes, QEMUIOVector *qiov, 1276ac850bf0SVladimir Sementsov-Ogievskiy size_t qiov_offset) 127729a298afSPavel Butsykin { 127829a298afSPavel Butsykin BlockDriver *drv = bs->drv; 1279ac850bf0SVladimir Sementsov-Ogievskiy QEMUIOVector local_qiov; 1280ac850bf0SVladimir Sementsov-Ogievskiy int ret; 128129a298afSPavel Butsykin 128217abcbeeSVladimir Sementsov-Ogievskiy bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 128317abcbeeSVladimir Sementsov-Ogievskiy 1284d470ad42SMax Reitz if (!drv) { 1285d470ad42SMax Reitz return -ENOMEDIUM; 1286d470ad42SMax Reitz } 1287d470ad42SMax Reitz 1288ac850bf0SVladimir Sementsov-Ogievskiy if (!block_driver_can_compress(drv)) { 128929a298afSPavel Butsykin return -ENOTSUP; 129029a298afSPavel Butsykin } 129129a298afSPavel Butsykin 1292ac850bf0SVladimir Sementsov-Ogievskiy if (drv->bdrv_co_pwritev_compressed_part) { 1293ac850bf0SVladimir Sementsov-Ogievskiy return drv->bdrv_co_pwritev_compressed_part(bs, offset, bytes, 1294ac850bf0SVladimir Sementsov-Ogievskiy qiov, qiov_offset); 1295ac850bf0SVladimir Sementsov-Ogievskiy } 1296ac850bf0SVladimir Sementsov-Ogievskiy 1297ac850bf0SVladimir Sementsov-Ogievskiy if (qiov_offset == 0) { 129829a298afSPavel Butsykin return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov); 129929a298afSPavel Butsykin } 130029a298afSPavel Butsykin 1301ac850bf0SVladimir Sementsov-Ogievskiy qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1302ac850bf0SVladimir Sementsov-Ogievskiy ret = drv->bdrv_co_pwritev_compressed(bs, offset, bytes, &local_qiov); 1303ac850bf0SVladimir Sementsov-Ogievskiy qemu_iovec_destroy(&local_qiov); 1304ac850bf0SVladimir Sementsov-Ogievskiy 1305ac850bf0SVladimir Sementsov-Ogievskiy return ret; 1306ac850bf0SVladimir Sementsov-Ogievskiy } 1307ac850bf0SVladimir Sementsov-Ogievskiy 130885c97ca7SKevin Wolf static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child, 13099df5afbdSVladimir Sementsov-Ogievskiy int64_t offset, int64_t bytes, QEMUIOVector *qiov, 13101143ec5eSVladimir Sementsov-Ogievskiy size_t qiov_offset, int flags) 131161007b31SStefan Hajnoczi { 131285c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 131385c97ca7SKevin Wolf 131461007b31SStefan Hajnoczi /* Perform I/O through a temporary buffer so that users who scribble over 131561007b31SStefan Hajnoczi * their read buffer while the operation is in progress do not end up 131661007b31SStefan Hajnoczi * modifying the image file. This is critical for zero-copy guest I/O 131761007b31SStefan Hajnoczi * where anything might happen inside guest memory. 131861007b31SStefan Hajnoczi */ 13192275cc90SVladimir Sementsov-Ogievskiy void *bounce_buffer = NULL; 132061007b31SStefan Hajnoczi 132161007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 1322244483e6SKevin Wolf int64_t cluster_offset; 13237cfd5275SEric Blake int64_t cluster_bytes; 13249df5afbdSVladimir Sementsov-Ogievskiy int64_t skip_bytes; 132561007b31SStefan Hajnoczi int ret; 1326cb2e2878SEric Blake int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, 1327cb2e2878SEric Blake BDRV_REQUEST_MAX_BYTES); 13289df5afbdSVladimir Sementsov-Ogievskiy int64_t progress = 0; 13298644476eSMax Reitz bool skip_write; 133061007b31SStefan Hajnoczi 13319df5afbdSVladimir Sementsov-Ogievskiy bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 13329df5afbdSVladimir Sementsov-Ogievskiy 1333d470ad42SMax Reitz if (!drv) { 1334d470ad42SMax Reitz return -ENOMEDIUM; 1335d470ad42SMax Reitz } 1336d470ad42SMax Reitz 13378644476eSMax Reitz /* 13388644476eSMax Reitz * Do not write anything when the BDS is inactive. That is not 13398644476eSMax Reitz * allowed, and it would not help. 13408644476eSMax Reitz */ 13418644476eSMax Reitz skip_write = (bs->open_flags & BDRV_O_INACTIVE); 13428644476eSMax Reitz 13431bf03e66SKevin Wolf /* FIXME We cannot require callers to have write permissions when all they 13441bf03e66SKevin Wolf * are doing is a read request. If we did things right, write permissions 13451bf03e66SKevin Wolf * would be obtained anyway, but internally by the copy-on-read code. As 1346765d9df9SEric Blake * long as it is implemented here rather than in a separate filter driver, 13471bf03e66SKevin Wolf * the copy-on-read code doesn't have its own BdrvChild, however, for which 13481bf03e66SKevin Wolf * it could request permissions. Therefore we have to bypass the permission 13491bf03e66SKevin Wolf * system for the moment. */ 13501bf03e66SKevin Wolf // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE)); 1351afa4b293SKevin Wolf 135261007b31SStefan Hajnoczi /* Cover entire cluster so no additional backing file I/O is required when 1353cb2e2878SEric Blake * allocating cluster in the image file. Note that this value may exceed 1354cb2e2878SEric Blake * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which 1355cb2e2878SEric Blake * is one reason we loop rather than doing it all at once. 135661007b31SStefan Hajnoczi */ 1357244483e6SKevin Wolf bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes); 1358cb2e2878SEric Blake skip_bytes = offset - cluster_offset; 135961007b31SStefan Hajnoczi 1360244483e6SKevin Wolf trace_bdrv_co_do_copy_on_readv(bs, offset, bytes, 1361244483e6SKevin Wolf cluster_offset, cluster_bytes); 136261007b31SStefan Hajnoczi 1363cb2e2878SEric Blake while (cluster_bytes) { 1364cb2e2878SEric Blake int64_t pnum; 136561007b31SStefan Hajnoczi 13668644476eSMax Reitz if (skip_write) { 13678644476eSMax Reitz ret = 1; /* "already allocated", so nothing will be copied */ 13688644476eSMax Reitz pnum = MIN(cluster_bytes, max_transfer); 13698644476eSMax Reitz } else { 1370cb2e2878SEric Blake ret = bdrv_is_allocated(bs, cluster_offset, 1371cb2e2878SEric Blake MIN(cluster_bytes, max_transfer), &pnum); 1372cb2e2878SEric Blake if (ret < 0) { 13738644476eSMax Reitz /* 13748644476eSMax Reitz * Safe to treat errors in querying allocation as if 1375cb2e2878SEric Blake * unallocated; we'll probably fail again soon on the 1376cb2e2878SEric Blake * read, but at least that will set a decent errno. 1377cb2e2878SEric Blake */ 1378cb2e2878SEric Blake pnum = MIN(cluster_bytes, max_transfer); 1379cb2e2878SEric Blake } 1380cb2e2878SEric Blake 1381b0ddcbbbSKevin Wolf /* Stop at EOF if the image ends in the middle of the cluster */ 1382b0ddcbbbSKevin Wolf if (ret == 0 && pnum == 0) { 1383b0ddcbbbSKevin Wolf assert(progress >= bytes); 1384b0ddcbbbSKevin Wolf break; 1385b0ddcbbbSKevin Wolf } 1386b0ddcbbbSKevin Wolf 1387cb2e2878SEric Blake assert(skip_bytes < pnum); 13888644476eSMax Reitz } 1389cb2e2878SEric Blake 1390cb2e2878SEric Blake if (ret <= 0) { 13911143ec5eSVladimir Sementsov-Ogievskiy QEMUIOVector local_qiov; 13921143ec5eSVladimir Sementsov-Ogievskiy 1393cb2e2878SEric Blake /* Must copy-on-read; use the bounce buffer */ 13940d93ed08SVladimir Sementsov-Ogievskiy pnum = MIN(pnum, MAX_BOUNCE_BUFFER); 13952275cc90SVladimir Sementsov-Ogievskiy if (!bounce_buffer) { 13962275cc90SVladimir Sementsov-Ogievskiy int64_t max_we_need = MAX(pnum, cluster_bytes - pnum); 13972275cc90SVladimir Sementsov-Ogievskiy int64_t max_allowed = MIN(max_transfer, MAX_BOUNCE_BUFFER); 13982275cc90SVladimir Sementsov-Ogievskiy int64_t bounce_buffer_len = MIN(max_we_need, max_allowed); 13992275cc90SVladimir Sementsov-Ogievskiy 14002275cc90SVladimir Sementsov-Ogievskiy bounce_buffer = qemu_try_blockalign(bs, bounce_buffer_len); 14012275cc90SVladimir Sementsov-Ogievskiy if (!bounce_buffer) { 14022275cc90SVladimir Sementsov-Ogievskiy ret = -ENOMEM; 14032275cc90SVladimir Sementsov-Ogievskiy goto err; 14042275cc90SVladimir Sementsov-Ogievskiy } 14052275cc90SVladimir Sementsov-Ogievskiy } 14060d93ed08SVladimir Sementsov-Ogievskiy qemu_iovec_init_buf(&local_qiov, bounce_buffer, pnum); 1407cb2e2878SEric Blake 1408cb2e2878SEric Blake ret = bdrv_driver_preadv(bs, cluster_offset, pnum, 1409ac850bf0SVladimir Sementsov-Ogievskiy &local_qiov, 0, 0); 141061007b31SStefan Hajnoczi if (ret < 0) { 141161007b31SStefan Hajnoczi goto err; 141261007b31SStefan Hajnoczi } 141361007b31SStefan Hajnoczi 1414d855ebcdSEric Blake bdrv_debug_event(bs, BLKDBG_COR_WRITE); 1415c1499a5eSEric Blake if (drv->bdrv_co_pwrite_zeroes && 1416cb2e2878SEric Blake buffer_is_zero(bounce_buffer, pnum)) { 1417a604fa2bSEric Blake /* FIXME: Should we (perhaps conditionally) be setting 1418a604fa2bSEric Blake * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy 1419a604fa2bSEric Blake * that still correctly reads as zero? */ 14207adcf59fSMax Reitz ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, pnum, 14217adcf59fSMax Reitz BDRV_REQ_WRITE_UNCHANGED); 142261007b31SStefan Hajnoczi } else { 1423cb2e2878SEric Blake /* This does not change the data on the disk, it is not 1424cb2e2878SEric Blake * necessary to flush even in cache=writethrough mode. 142561007b31SStefan Hajnoczi */ 1426cb2e2878SEric Blake ret = bdrv_driver_pwritev(bs, cluster_offset, pnum, 1427ac850bf0SVladimir Sementsov-Ogievskiy &local_qiov, 0, 14287adcf59fSMax Reitz BDRV_REQ_WRITE_UNCHANGED); 142961007b31SStefan Hajnoczi } 143061007b31SStefan Hajnoczi 143161007b31SStefan Hajnoczi if (ret < 0) { 1432cb2e2878SEric Blake /* It might be okay to ignore write errors for guest 1433cb2e2878SEric Blake * requests. If this is a deliberate copy-on-read 1434cb2e2878SEric Blake * then we don't want to ignore the error. Simply 1435cb2e2878SEric Blake * report it in all cases. 143661007b31SStefan Hajnoczi */ 143761007b31SStefan Hajnoczi goto err; 143861007b31SStefan Hajnoczi } 143961007b31SStefan Hajnoczi 14403299e5ecSVladimir Sementsov-Ogievskiy if (!(flags & BDRV_REQ_PREFETCH)) { 14411143ec5eSVladimir Sementsov-Ogievskiy qemu_iovec_from_buf(qiov, qiov_offset + progress, 14421143ec5eSVladimir Sementsov-Ogievskiy bounce_buffer + skip_bytes, 14434ab78b19SVladimir Sementsov-Ogievskiy MIN(pnum - skip_bytes, bytes - progress)); 14443299e5ecSVladimir Sementsov-Ogievskiy } 14453299e5ecSVladimir Sementsov-Ogievskiy } else if (!(flags & BDRV_REQ_PREFETCH)) { 1446cb2e2878SEric Blake /* Read directly into the destination */ 14471143ec5eSVladimir Sementsov-Ogievskiy ret = bdrv_driver_preadv(bs, offset + progress, 14481143ec5eSVladimir Sementsov-Ogievskiy MIN(pnum - skip_bytes, bytes - progress), 14491143ec5eSVladimir Sementsov-Ogievskiy qiov, qiov_offset + progress, 0); 1450cb2e2878SEric Blake if (ret < 0) { 1451cb2e2878SEric Blake goto err; 1452cb2e2878SEric Blake } 1453cb2e2878SEric Blake } 1454cb2e2878SEric Blake 1455cb2e2878SEric Blake cluster_offset += pnum; 1456cb2e2878SEric Blake cluster_bytes -= pnum; 1457cb2e2878SEric Blake progress += pnum - skip_bytes; 1458cb2e2878SEric Blake skip_bytes = 0; 1459cb2e2878SEric Blake } 1460cb2e2878SEric Blake ret = 0; 146161007b31SStefan Hajnoczi 146261007b31SStefan Hajnoczi err: 146361007b31SStefan Hajnoczi qemu_vfree(bounce_buffer); 146461007b31SStefan Hajnoczi return ret; 146561007b31SStefan Hajnoczi } 146661007b31SStefan Hajnoczi 146761007b31SStefan Hajnoczi /* 146861007b31SStefan Hajnoczi * Forwards an already correctly aligned request to the BlockDriver. This 14691a62d0acSEric Blake * handles copy on read, zeroing after EOF, and fragmentation of large 14701a62d0acSEric Blake * reads; any other features must be implemented by the caller. 147161007b31SStefan Hajnoczi */ 147285c97ca7SKevin Wolf static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child, 14738b0c5d76SVladimir Sementsov-Ogievskiy BdrvTrackedRequest *req, int64_t offset, int64_t bytes, 147465cd4424SVladimir Sementsov-Ogievskiy int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags) 147561007b31SStefan Hajnoczi { 147685c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 1477c9d20029SKevin Wolf int64_t total_bytes, max_bytes; 14781a62d0acSEric Blake int ret = 0; 14798b0c5d76SVladimir Sementsov-Ogievskiy int64_t bytes_remaining = bytes; 14801a62d0acSEric Blake int max_transfer; 148161007b31SStefan Hajnoczi 14828b0c5d76SVladimir Sementsov-Ogievskiy bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 148349c07526SKevin Wolf assert(is_power_of_2(align)); 148449c07526SKevin Wolf assert((offset & (align - 1)) == 0); 148549c07526SKevin Wolf assert((bytes & (align - 1)) == 0); 1486abb06c5aSDaniel P. Berrange assert((bs->open_flags & BDRV_O_NO_IO) == 0); 14871a62d0acSEric Blake max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 14881a62d0acSEric Blake align); 1489a604fa2bSEric Blake 1490a604fa2bSEric Blake /* TODO: We would need a per-BDS .supported_read_flags and 1491a604fa2bSEric Blake * potential fallback support, if we ever implement any read flags 1492a604fa2bSEric Blake * to pass through to drivers. For now, there aren't any 1493a604fa2bSEric Blake * passthrough flags. */ 1494c53cb427SPaolo Bonzini assert(!(flags & ~(BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH))); 149561007b31SStefan Hajnoczi 149661007b31SStefan Hajnoczi /* Handle Copy on Read and associated serialisation */ 149761007b31SStefan Hajnoczi if (flags & BDRV_REQ_COPY_ON_READ) { 149861007b31SStefan Hajnoczi /* If we touch the same cluster it counts as an overlap. This 149961007b31SStefan Hajnoczi * guarantees that allocating writes will be serialized and not race 150061007b31SStefan Hajnoczi * with each other for the same cluster. For example, in copy-on-read 150161007b31SStefan Hajnoczi * it ensures that the CoR read and write operations are atomic and 150261007b31SStefan Hajnoczi * guest writes cannot interleave between them. */ 15038ac5aab2SVladimir Sementsov-Ogievskiy bdrv_make_request_serialising(req, bdrv_get_cluster_size(bs)); 150418fbd0deSPaolo Bonzini } else { 1505304d9d7fSMax Reitz bdrv_wait_serialising_requests(req); 150618fbd0deSPaolo Bonzini } 150761007b31SStefan Hajnoczi 150861007b31SStefan Hajnoczi if (flags & BDRV_REQ_COPY_ON_READ) { 1509d6a644bbSEric Blake int64_t pnum; 151061007b31SStefan Hajnoczi 1511897dd0ecSAndrey Shinkevich /* The flag BDRV_REQ_COPY_ON_READ has reached its addressee */ 1512897dd0ecSAndrey Shinkevich flags &= ~BDRV_REQ_COPY_ON_READ; 1513897dd0ecSAndrey Shinkevich 151488e63df2SEric Blake ret = bdrv_is_allocated(bs, offset, bytes, &pnum); 151561007b31SStefan Hajnoczi if (ret < 0) { 151661007b31SStefan Hajnoczi goto out; 151761007b31SStefan Hajnoczi } 151861007b31SStefan Hajnoczi 151988e63df2SEric Blake if (!ret || pnum != bytes) { 152065cd4424SVladimir Sementsov-Ogievskiy ret = bdrv_co_do_copy_on_readv(child, offset, bytes, 152165cd4424SVladimir Sementsov-Ogievskiy qiov, qiov_offset, flags); 15223299e5ecSVladimir Sementsov-Ogievskiy goto out; 15233299e5ecSVladimir Sementsov-Ogievskiy } else if (flags & BDRV_REQ_PREFETCH) { 152461007b31SStefan Hajnoczi goto out; 152561007b31SStefan Hajnoczi } 152661007b31SStefan Hajnoczi } 152761007b31SStefan Hajnoczi 15281a62d0acSEric Blake /* Forward the request to the BlockDriver, possibly fragmenting it */ 152949c07526SKevin Wolf total_bytes = bdrv_getlength(bs); 153049c07526SKevin Wolf if (total_bytes < 0) { 153149c07526SKevin Wolf ret = total_bytes; 153261007b31SStefan Hajnoczi goto out; 153361007b31SStefan Hajnoczi } 153461007b31SStefan Hajnoczi 1535897dd0ecSAndrey Shinkevich assert(!(flags & ~bs->supported_read_flags)); 1536897dd0ecSAndrey Shinkevich 153749c07526SKevin Wolf max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align); 15381a62d0acSEric Blake if (bytes <= max_bytes && bytes <= max_transfer) { 1539897dd0ecSAndrey Shinkevich ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, flags); 15401a62d0acSEric Blake goto out; 154161007b31SStefan Hajnoczi } 154261007b31SStefan Hajnoczi 15431a62d0acSEric Blake while (bytes_remaining) { 15448b0c5d76SVladimir Sementsov-Ogievskiy int64_t num; 15451a62d0acSEric Blake 15461a62d0acSEric Blake if (max_bytes) { 15471a62d0acSEric Blake num = MIN(bytes_remaining, MIN(max_bytes, max_transfer)); 15481a62d0acSEric Blake assert(num); 15491a62d0acSEric Blake 15501a62d0acSEric Blake ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining, 1551134b7decSMax Reitz num, qiov, 1552897dd0ecSAndrey Shinkevich qiov_offset + bytes - bytes_remaining, 1553897dd0ecSAndrey Shinkevich flags); 15541a62d0acSEric Blake max_bytes -= num; 15551a62d0acSEric Blake } else { 15561a62d0acSEric Blake num = bytes_remaining; 1557134b7decSMax Reitz ret = qemu_iovec_memset(qiov, qiov_offset + bytes - bytes_remaining, 1558134b7decSMax Reitz 0, bytes_remaining); 15591a62d0acSEric Blake } 15601a62d0acSEric Blake if (ret < 0) { 15611a62d0acSEric Blake goto out; 15621a62d0acSEric Blake } 15631a62d0acSEric Blake bytes_remaining -= num; 156461007b31SStefan Hajnoczi } 156561007b31SStefan Hajnoczi 156661007b31SStefan Hajnoczi out: 15671a62d0acSEric Blake return ret < 0 ? ret : 0; 156861007b31SStefan Hajnoczi } 156961007b31SStefan Hajnoczi 157061007b31SStefan Hajnoczi /* 15717a3f542fSVladimir Sementsov-Ogievskiy * Request padding 15727a3f542fSVladimir Sementsov-Ogievskiy * 15737a3f542fSVladimir Sementsov-Ogievskiy * |<---- align ----->| |<----- align ---->| 15747a3f542fSVladimir Sementsov-Ogievskiy * |<- head ->|<------------- bytes ------------->|<-- tail -->| 15757a3f542fSVladimir Sementsov-Ogievskiy * | | | | | | 15767a3f542fSVladimir Sementsov-Ogievskiy * -*----------$-------*-------- ... --------*-----$------------*--- 15777a3f542fSVladimir Sementsov-Ogievskiy * | | | | | | 15787a3f542fSVladimir Sementsov-Ogievskiy * | offset | | end | 15797a3f542fSVladimir Sementsov-Ogievskiy * ALIGN_DOWN(offset) ALIGN_UP(offset) ALIGN_DOWN(end) ALIGN_UP(end) 15807a3f542fSVladimir Sementsov-Ogievskiy * [buf ... ) [tail_buf ) 15817a3f542fSVladimir Sementsov-Ogievskiy * 15827a3f542fSVladimir Sementsov-Ogievskiy * @buf is an aligned allocation needed to store @head and @tail paddings. @head 15837a3f542fSVladimir Sementsov-Ogievskiy * is placed at the beginning of @buf and @tail at the @end. 15847a3f542fSVladimir Sementsov-Ogievskiy * 15857a3f542fSVladimir Sementsov-Ogievskiy * @tail_buf is a pointer to sub-buffer, corresponding to align-sized chunk 15867a3f542fSVladimir Sementsov-Ogievskiy * around tail, if tail exists. 15877a3f542fSVladimir Sementsov-Ogievskiy * 15887a3f542fSVladimir Sementsov-Ogievskiy * @merge_reads is true for small requests, 15897a3f542fSVladimir Sementsov-Ogievskiy * if @buf_len == @head + bytes + @tail. In this case it is possible that both 15907a3f542fSVladimir Sementsov-Ogievskiy * head and tail exist but @buf_len == align and @tail_buf == @buf. 159161007b31SStefan Hajnoczi */ 15927a3f542fSVladimir Sementsov-Ogievskiy typedef struct BdrvRequestPadding { 15937a3f542fSVladimir Sementsov-Ogievskiy uint8_t *buf; 15947a3f542fSVladimir Sementsov-Ogievskiy size_t buf_len; 15957a3f542fSVladimir Sementsov-Ogievskiy uint8_t *tail_buf; 15967a3f542fSVladimir Sementsov-Ogievskiy size_t head; 15977a3f542fSVladimir Sementsov-Ogievskiy size_t tail; 15987a3f542fSVladimir Sementsov-Ogievskiy bool merge_reads; 15997a3f542fSVladimir Sementsov-Ogievskiy QEMUIOVector local_qiov; 16007a3f542fSVladimir Sementsov-Ogievskiy } BdrvRequestPadding; 16017a3f542fSVladimir Sementsov-Ogievskiy 16027a3f542fSVladimir Sementsov-Ogievskiy static bool bdrv_init_padding(BlockDriverState *bs, 16037a3f542fSVladimir Sementsov-Ogievskiy int64_t offset, int64_t bytes, 16047a3f542fSVladimir Sementsov-Ogievskiy BdrvRequestPadding *pad) 16057a3f542fSVladimir Sementsov-Ogievskiy { 1606a56ed80cSVladimir Sementsov-Ogievskiy int64_t align = bs->bl.request_alignment; 1607a56ed80cSVladimir Sementsov-Ogievskiy int64_t sum; 1608a56ed80cSVladimir Sementsov-Ogievskiy 1609a56ed80cSVladimir Sementsov-Ogievskiy bdrv_check_request(offset, bytes, &error_abort); 1610a56ed80cSVladimir Sementsov-Ogievskiy assert(align <= INT_MAX); /* documented in block/block_int.h */ 1611a56ed80cSVladimir Sementsov-Ogievskiy assert(align <= SIZE_MAX / 2); /* so we can allocate the buffer */ 16127a3f542fSVladimir Sementsov-Ogievskiy 16137a3f542fSVladimir Sementsov-Ogievskiy memset(pad, 0, sizeof(*pad)); 16147a3f542fSVladimir Sementsov-Ogievskiy 16157a3f542fSVladimir Sementsov-Ogievskiy pad->head = offset & (align - 1); 16167a3f542fSVladimir Sementsov-Ogievskiy pad->tail = ((offset + bytes) & (align - 1)); 16177a3f542fSVladimir Sementsov-Ogievskiy if (pad->tail) { 16187a3f542fSVladimir Sementsov-Ogievskiy pad->tail = align - pad->tail; 16197a3f542fSVladimir Sementsov-Ogievskiy } 16207a3f542fSVladimir Sementsov-Ogievskiy 1621ac9d00bfSVladimir Sementsov-Ogievskiy if (!pad->head && !pad->tail) { 16227a3f542fSVladimir Sementsov-Ogievskiy return false; 16237a3f542fSVladimir Sementsov-Ogievskiy } 16247a3f542fSVladimir Sementsov-Ogievskiy 1625ac9d00bfSVladimir Sementsov-Ogievskiy assert(bytes); /* Nothing good in aligning zero-length requests */ 1626ac9d00bfSVladimir Sementsov-Ogievskiy 16277a3f542fSVladimir Sementsov-Ogievskiy sum = pad->head + bytes + pad->tail; 16287a3f542fSVladimir Sementsov-Ogievskiy pad->buf_len = (sum > align && pad->head && pad->tail) ? 2 * align : align; 16297a3f542fSVladimir Sementsov-Ogievskiy pad->buf = qemu_blockalign(bs, pad->buf_len); 16307a3f542fSVladimir Sementsov-Ogievskiy pad->merge_reads = sum == pad->buf_len; 16317a3f542fSVladimir Sementsov-Ogievskiy if (pad->tail) { 16327a3f542fSVladimir Sementsov-Ogievskiy pad->tail_buf = pad->buf + pad->buf_len - align; 16337a3f542fSVladimir Sementsov-Ogievskiy } 16347a3f542fSVladimir Sementsov-Ogievskiy 16357a3f542fSVladimir Sementsov-Ogievskiy return true; 16367a3f542fSVladimir Sementsov-Ogievskiy } 16377a3f542fSVladimir Sementsov-Ogievskiy 1638881a4c55SPaolo Bonzini static coroutine_fn int bdrv_padding_rmw_read(BdrvChild *child, 16397a3f542fSVladimir Sementsov-Ogievskiy BdrvTrackedRequest *req, 16407a3f542fSVladimir Sementsov-Ogievskiy BdrvRequestPadding *pad, 16417a3f542fSVladimir Sementsov-Ogievskiy bool zero_middle) 16427a3f542fSVladimir Sementsov-Ogievskiy { 16437a3f542fSVladimir Sementsov-Ogievskiy QEMUIOVector local_qiov; 16447a3f542fSVladimir Sementsov-Ogievskiy BlockDriverState *bs = child->bs; 16457a3f542fSVladimir Sementsov-Ogievskiy uint64_t align = bs->bl.request_alignment; 16467a3f542fSVladimir Sementsov-Ogievskiy int ret; 16477a3f542fSVladimir Sementsov-Ogievskiy 16487a3f542fSVladimir Sementsov-Ogievskiy assert(req->serialising && pad->buf); 16497a3f542fSVladimir Sementsov-Ogievskiy 16507a3f542fSVladimir Sementsov-Ogievskiy if (pad->head || pad->merge_reads) { 16518b0c5d76SVladimir Sementsov-Ogievskiy int64_t bytes = pad->merge_reads ? pad->buf_len : align; 16527a3f542fSVladimir Sementsov-Ogievskiy 16537a3f542fSVladimir Sementsov-Ogievskiy qemu_iovec_init_buf(&local_qiov, pad->buf, bytes); 16547a3f542fSVladimir Sementsov-Ogievskiy 16557a3f542fSVladimir Sementsov-Ogievskiy if (pad->head) { 16567a3f542fSVladimir Sementsov-Ogievskiy bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 16577a3f542fSVladimir Sementsov-Ogievskiy } 16587a3f542fSVladimir Sementsov-Ogievskiy if (pad->merge_reads && pad->tail) { 16597a3f542fSVladimir Sementsov-Ogievskiy bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 16607a3f542fSVladimir Sementsov-Ogievskiy } 16617a3f542fSVladimir Sementsov-Ogievskiy ret = bdrv_aligned_preadv(child, req, req->overlap_offset, bytes, 166265cd4424SVladimir Sementsov-Ogievskiy align, &local_qiov, 0, 0); 16637a3f542fSVladimir Sementsov-Ogievskiy if (ret < 0) { 16647a3f542fSVladimir Sementsov-Ogievskiy return ret; 16657a3f542fSVladimir Sementsov-Ogievskiy } 16667a3f542fSVladimir Sementsov-Ogievskiy if (pad->head) { 16677a3f542fSVladimir Sementsov-Ogievskiy bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 16687a3f542fSVladimir Sementsov-Ogievskiy } 16697a3f542fSVladimir Sementsov-Ogievskiy if (pad->merge_reads && pad->tail) { 16707a3f542fSVladimir Sementsov-Ogievskiy bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 16717a3f542fSVladimir Sementsov-Ogievskiy } 16727a3f542fSVladimir Sementsov-Ogievskiy 16737a3f542fSVladimir Sementsov-Ogievskiy if (pad->merge_reads) { 16747a3f542fSVladimir Sementsov-Ogievskiy goto zero_mem; 16757a3f542fSVladimir Sementsov-Ogievskiy } 16767a3f542fSVladimir Sementsov-Ogievskiy } 16777a3f542fSVladimir Sementsov-Ogievskiy 16787a3f542fSVladimir Sementsov-Ogievskiy if (pad->tail) { 16797a3f542fSVladimir Sementsov-Ogievskiy qemu_iovec_init_buf(&local_qiov, pad->tail_buf, align); 16807a3f542fSVladimir Sementsov-Ogievskiy 16817a3f542fSVladimir Sementsov-Ogievskiy bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 16827a3f542fSVladimir Sementsov-Ogievskiy ret = bdrv_aligned_preadv( 16837a3f542fSVladimir Sementsov-Ogievskiy child, req, 16847a3f542fSVladimir Sementsov-Ogievskiy req->overlap_offset + req->overlap_bytes - align, 168565cd4424SVladimir Sementsov-Ogievskiy align, align, &local_qiov, 0, 0); 16867a3f542fSVladimir Sementsov-Ogievskiy if (ret < 0) { 16877a3f542fSVladimir Sementsov-Ogievskiy return ret; 16887a3f542fSVladimir Sementsov-Ogievskiy } 16897a3f542fSVladimir Sementsov-Ogievskiy bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 16907a3f542fSVladimir Sementsov-Ogievskiy } 16917a3f542fSVladimir Sementsov-Ogievskiy 16927a3f542fSVladimir Sementsov-Ogievskiy zero_mem: 16937a3f542fSVladimir Sementsov-Ogievskiy if (zero_middle) { 16947a3f542fSVladimir Sementsov-Ogievskiy memset(pad->buf + pad->head, 0, pad->buf_len - pad->head - pad->tail); 16957a3f542fSVladimir Sementsov-Ogievskiy } 16967a3f542fSVladimir Sementsov-Ogievskiy 16977a3f542fSVladimir Sementsov-Ogievskiy return 0; 16987a3f542fSVladimir Sementsov-Ogievskiy } 16997a3f542fSVladimir Sementsov-Ogievskiy 17007a3f542fSVladimir Sementsov-Ogievskiy static void bdrv_padding_destroy(BdrvRequestPadding *pad) 17017a3f542fSVladimir Sementsov-Ogievskiy { 17027a3f542fSVladimir Sementsov-Ogievskiy if (pad->buf) { 17037a3f542fSVladimir Sementsov-Ogievskiy qemu_vfree(pad->buf); 17047a3f542fSVladimir Sementsov-Ogievskiy qemu_iovec_destroy(&pad->local_qiov); 17057a3f542fSVladimir Sementsov-Ogievskiy } 170698ca4549SVladimir Sementsov-Ogievskiy memset(pad, 0, sizeof(*pad)); 17077a3f542fSVladimir Sementsov-Ogievskiy } 17087a3f542fSVladimir Sementsov-Ogievskiy 17097a3f542fSVladimir Sementsov-Ogievskiy /* 17107a3f542fSVladimir Sementsov-Ogievskiy * bdrv_pad_request 17117a3f542fSVladimir Sementsov-Ogievskiy * 17127a3f542fSVladimir Sementsov-Ogievskiy * Exchange request parameters with padded request if needed. Don't include RMW 17137a3f542fSVladimir Sementsov-Ogievskiy * read of padding, bdrv_padding_rmw_read() should be called separately if 17147a3f542fSVladimir Sementsov-Ogievskiy * needed. 17157a3f542fSVladimir Sementsov-Ogievskiy * 171698ca4549SVladimir Sementsov-Ogievskiy * Request parameters (@qiov, &qiov_offset, &offset, &bytes) are in-out: 171798ca4549SVladimir Sementsov-Ogievskiy * - on function start they represent original request 171898ca4549SVladimir Sementsov-Ogievskiy * - on failure or when padding is not needed they are unchanged 171998ca4549SVladimir Sementsov-Ogievskiy * - on success when padding is needed they represent padded request 17207a3f542fSVladimir Sementsov-Ogievskiy */ 172198ca4549SVladimir Sementsov-Ogievskiy static int bdrv_pad_request(BlockDriverState *bs, 17221acc3466SVladimir Sementsov-Ogievskiy QEMUIOVector **qiov, size_t *qiov_offset, 172337e9403eSVladimir Sementsov-Ogievskiy int64_t *offset, int64_t *bytes, 172498ca4549SVladimir Sementsov-Ogievskiy BdrvRequestPadding *pad, bool *padded) 17257a3f542fSVladimir Sementsov-Ogievskiy { 17264c002cefSVladimir Sementsov-Ogievskiy int ret; 17274c002cefSVladimir Sementsov-Ogievskiy 172837e9403eSVladimir Sementsov-Ogievskiy bdrv_check_qiov_request(*offset, *bytes, *qiov, *qiov_offset, &error_abort); 172937e9403eSVladimir Sementsov-Ogievskiy 17307a3f542fSVladimir Sementsov-Ogievskiy if (!bdrv_init_padding(bs, *offset, *bytes, pad)) { 173198ca4549SVladimir Sementsov-Ogievskiy if (padded) { 173298ca4549SVladimir Sementsov-Ogievskiy *padded = false; 173398ca4549SVladimir Sementsov-Ogievskiy } 173498ca4549SVladimir Sementsov-Ogievskiy return 0; 17357a3f542fSVladimir Sementsov-Ogievskiy } 17367a3f542fSVladimir Sementsov-Ogievskiy 17374c002cefSVladimir Sementsov-Ogievskiy ret = qemu_iovec_init_extended(&pad->local_qiov, pad->buf, pad->head, 17381acc3466SVladimir Sementsov-Ogievskiy *qiov, *qiov_offset, *bytes, 17394c002cefSVladimir Sementsov-Ogievskiy pad->buf + pad->buf_len - pad->tail, 17404c002cefSVladimir Sementsov-Ogievskiy pad->tail); 174198ca4549SVladimir Sementsov-Ogievskiy if (ret < 0) { 174298ca4549SVladimir Sementsov-Ogievskiy bdrv_padding_destroy(pad); 174398ca4549SVladimir Sementsov-Ogievskiy return ret; 174498ca4549SVladimir Sementsov-Ogievskiy } 17457a3f542fSVladimir Sementsov-Ogievskiy *bytes += pad->head + pad->tail; 17467a3f542fSVladimir Sementsov-Ogievskiy *offset -= pad->head; 17477a3f542fSVladimir Sementsov-Ogievskiy *qiov = &pad->local_qiov; 17481acc3466SVladimir Sementsov-Ogievskiy *qiov_offset = 0; 174998ca4549SVladimir Sementsov-Ogievskiy if (padded) { 175098ca4549SVladimir Sementsov-Ogievskiy *padded = true; 175198ca4549SVladimir Sementsov-Ogievskiy } 17527a3f542fSVladimir Sementsov-Ogievskiy 175398ca4549SVladimir Sementsov-Ogievskiy return 0; 17547a3f542fSVladimir Sementsov-Ogievskiy } 17557a3f542fSVladimir Sementsov-Ogievskiy 1756a03ef88fSKevin Wolf int coroutine_fn bdrv_co_preadv(BdrvChild *child, 1757e9e52efdSVladimir Sementsov-Ogievskiy int64_t offset, int64_t bytes, QEMUIOVector *qiov, 175861007b31SStefan Hajnoczi BdrvRequestFlags flags) 175961007b31SStefan Hajnoczi { 1760967d7905SEmanuele Giuseppe Esposito IO_CODE(); 17611acc3466SVladimir Sementsov-Ogievskiy return bdrv_co_preadv_part(child, offset, bytes, qiov, 0, flags); 17621acc3466SVladimir Sementsov-Ogievskiy } 17631acc3466SVladimir Sementsov-Ogievskiy 17641acc3466SVladimir Sementsov-Ogievskiy int coroutine_fn bdrv_co_preadv_part(BdrvChild *child, 176537e9403eSVladimir Sementsov-Ogievskiy int64_t offset, int64_t bytes, 17661acc3466SVladimir Sementsov-Ogievskiy QEMUIOVector *qiov, size_t qiov_offset, 17671acc3466SVladimir Sementsov-Ogievskiy BdrvRequestFlags flags) 17681acc3466SVladimir Sementsov-Ogievskiy { 1769a03ef88fSKevin Wolf BlockDriverState *bs = child->bs; 177061007b31SStefan Hajnoczi BdrvTrackedRequest req; 17717a3f542fSVladimir Sementsov-Ogievskiy BdrvRequestPadding pad; 177261007b31SStefan Hajnoczi int ret; 1773967d7905SEmanuele Giuseppe Esposito IO_CODE(); 177461007b31SStefan Hajnoczi 177537e9403eSVladimir Sementsov-Ogievskiy trace_bdrv_co_preadv_part(bs, offset, bytes, flags); 177661007b31SStefan Hajnoczi 1777f4dad307SVladimir Sementsov-Ogievskiy if (!bdrv_is_inserted(bs)) { 1778f4dad307SVladimir Sementsov-Ogievskiy return -ENOMEDIUM; 1779f4dad307SVladimir Sementsov-Ogievskiy } 1780f4dad307SVladimir Sementsov-Ogievskiy 178163f4ad11SVladimir Sementsov-Ogievskiy ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset); 178261007b31SStefan Hajnoczi if (ret < 0) { 178361007b31SStefan Hajnoczi return ret; 178461007b31SStefan Hajnoczi } 178561007b31SStefan Hajnoczi 1786ac9d00bfSVladimir Sementsov-Ogievskiy if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) { 1787ac9d00bfSVladimir Sementsov-Ogievskiy /* 1788ac9d00bfSVladimir Sementsov-Ogievskiy * Aligning zero request is nonsense. Even if driver has special meaning 1789ac9d00bfSVladimir Sementsov-Ogievskiy * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass 1790ac9d00bfSVladimir Sementsov-Ogievskiy * it to driver due to request_alignment. 1791ac9d00bfSVladimir Sementsov-Ogievskiy * 1792ac9d00bfSVladimir Sementsov-Ogievskiy * Still, no reason to return an error if someone do unaligned 1793ac9d00bfSVladimir Sementsov-Ogievskiy * zero-length read occasionally. 1794ac9d00bfSVladimir Sementsov-Ogievskiy */ 1795ac9d00bfSVladimir Sementsov-Ogievskiy return 0; 1796ac9d00bfSVladimir Sementsov-Ogievskiy } 1797ac9d00bfSVladimir Sementsov-Ogievskiy 179899723548SPaolo Bonzini bdrv_inc_in_flight(bs); 179999723548SPaolo Bonzini 18009568b511SWen Congyang /* Don't do copy-on-read if we read data before write operation */ 1801d73415a3SStefan Hajnoczi if (qatomic_read(&bs->copy_on_read)) { 180261007b31SStefan Hajnoczi flags |= BDRV_REQ_COPY_ON_READ; 180361007b31SStefan Hajnoczi } 180461007b31SStefan Hajnoczi 180598ca4549SVladimir Sementsov-Ogievskiy ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad, 180698ca4549SVladimir Sementsov-Ogievskiy NULL); 180798ca4549SVladimir Sementsov-Ogievskiy if (ret < 0) { 180887ab8802SKevin Wolf goto fail; 180998ca4549SVladimir Sementsov-Ogievskiy } 181061007b31SStefan Hajnoczi 1811ebde595cSFam Zheng tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ); 18127a3f542fSVladimir Sementsov-Ogievskiy ret = bdrv_aligned_preadv(child, &req, offset, bytes, 18137a3f542fSVladimir Sementsov-Ogievskiy bs->bl.request_alignment, 18141acc3466SVladimir Sementsov-Ogievskiy qiov, qiov_offset, flags); 181561007b31SStefan Hajnoczi tracked_request_end(&req); 18167a3f542fSVladimir Sementsov-Ogievskiy bdrv_padding_destroy(&pad); 181761007b31SStefan Hajnoczi 181887ab8802SKevin Wolf fail: 181987ab8802SKevin Wolf bdrv_dec_in_flight(bs); 182087ab8802SKevin Wolf 182161007b31SStefan Hajnoczi return ret; 182261007b31SStefan Hajnoczi } 182361007b31SStefan Hajnoczi 1824d05aa8bbSEric Blake static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 18255ae07b14SVladimir Sementsov-Ogievskiy int64_t offset, int64_t bytes, BdrvRequestFlags flags) 182661007b31SStefan Hajnoczi { 182761007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 182861007b31SStefan Hajnoczi QEMUIOVector qiov; 18290d93ed08SVladimir Sementsov-Ogievskiy void *buf = NULL; 183061007b31SStefan Hajnoczi int ret = 0; 1831465fe887SEric Blake bool need_flush = false; 1832443668caSDenis V. Lunev int head = 0; 1833443668caSDenis V. Lunev int tail = 0; 183461007b31SStefan Hajnoczi 18352aaa3f9bSVladimir Sementsov-Ogievskiy int64_t max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, 18362aaa3f9bSVladimir Sementsov-Ogievskiy INT64_MAX); 1837a5b8dd2cSEric Blake int alignment = MAX(bs->bl.pwrite_zeroes_alignment, 1838a5b8dd2cSEric Blake bs->bl.request_alignment); 1839cb2e2878SEric Blake int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER); 1840cf081fcaSEric Blake 18415ae07b14SVladimir Sementsov-Ogievskiy bdrv_check_request(offset, bytes, &error_abort); 18425ae07b14SVladimir Sementsov-Ogievskiy 1843d470ad42SMax Reitz if (!drv) { 1844d470ad42SMax Reitz return -ENOMEDIUM; 1845d470ad42SMax Reitz } 1846d470ad42SMax Reitz 1847fe0480d6SKevin Wolf if ((flags & ~bs->supported_zero_flags) & BDRV_REQ_NO_FALLBACK) { 1848fe0480d6SKevin Wolf return -ENOTSUP; 1849fe0480d6SKevin Wolf } 1850fe0480d6SKevin Wolf 18510bc329fbSHanna Reitz /* Invalidate the cached block-status data range if this write overlaps */ 18520bc329fbSHanna Reitz bdrv_bsc_invalidate_range(bs, offset, bytes); 18530bc329fbSHanna Reitz 1854b8d0a980SEric Blake assert(alignment % bs->bl.request_alignment == 0); 1855b8d0a980SEric Blake head = offset % alignment; 1856f5a5ca79SManos Pitsidianakis tail = (offset + bytes) % alignment; 1857b8d0a980SEric Blake max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment); 1858b8d0a980SEric Blake assert(max_write_zeroes >= bs->bl.request_alignment); 185961007b31SStefan Hajnoczi 1860f5a5ca79SManos Pitsidianakis while (bytes > 0 && !ret) { 18615ae07b14SVladimir Sementsov-Ogievskiy int64_t num = bytes; 186261007b31SStefan Hajnoczi 186361007b31SStefan Hajnoczi /* Align request. Block drivers can expect the "bulk" of the request 1864443668caSDenis V. Lunev * to be aligned, and that unaligned requests do not cross cluster 1865443668caSDenis V. Lunev * boundaries. 186661007b31SStefan Hajnoczi */ 1867443668caSDenis V. Lunev if (head) { 1868b2f95feeSEric Blake /* Make a small request up to the first aligned sector. For 1869b2f95feeSEric Blake * convenience, limit this request to max_transfer even if 1870b2f95feeSEric Blake * we don't need to fall back to writes. */ 1871f5a5ca79SManos Pitsidianakis num = MIN(MIN(bytes, max_transfer), alignment - head); 1872b2f95feeSEric Blake head = (head + num) % alignment; 1873b2f95feeSEric Blake assert(num < max_write_zeroes); 1874d05aa8bbSEric Blake } else if (tail && num > alignment) { 1875443668caSDenis V. Lunev /* Shorten the request to the last aligned sector. */ 1876443668caSDenis V. Lunev num -= tail; 187761007b31SStefan Hajnoczi } 187861007b31SStefan Hajnoczi 187961007b31SStefan Hajnoczi /* limit request size */ 188061007b31SStefan Hajnoczi if (num > max_write_zeroes) { 188161007b31SStefan Hajnoczi num = max_write_zeroes; 188261007b31SStefan Hajnoczi } 188361007b31SStefan Hajnoczi 188461007b31SStefan Hajnoczi ret = -ENOTSUP; 188561007b31SStefan Hajnoczi /* First try the efficient write zeroes operation */ 1886d05aa8bbSEric Blake if (drv->bdrv_co_pwrite_zeroes) { 1887d05aa8bbSEric Blake ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num, 1888d05aa8bbSEric Blake flags & bs->supported_zero_flags); 1889d05aa8bbSEric Blake if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) && 1890d05aa8bbSEric Blake !(bs->supported_zero_flags & BDRV_REQ_FUA)) { 1891d05aa8bbSEric Blake need_flush = true; 1892d05aa8bbSEric Blake } 1893465fe887SEric Blake } else { 1894465fe887SEric Blake assert(!bs->supported_zero_flags); 189561007b31SStefan Hajnoczi } 189661007b31SStefan Hajnoczi 1897294682ccSAndrey Shinkevich if (ret == -ENOTSUP && !(flags & BDRV_REQ_NO_FALLBACK)) { 189861007b31SStefan Hajnoczi /* Fall back to bounce buffer if write zeroes is unsupported */ 1899465fe887SEric Blake BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE; 1900465fe887SEric Blake 1901465fe887SEric Blake if ((flags & BDRV_REQ_FUA) && 1902465fe887SEric Blake !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1903465fe887SEric Blake /* No need for bdrv_driver_pwrite() to do a fallback 1904465fe887SEric Blake * flush on each chunk; use just one at the end */ 1905465fe887SEric Blake write_flags &= ~BDRV_REQ_FUA; 1906465fe887SEric Blake need_flush = true; 1907465fe887SEric Blake } 19085def6b80SEric Blake num = MIN(num, max_transfer); 19090d93ed08SVladimir Sementsov-Ogievskiy if (buf == NULL) { 19100d93ed08SVladimir Sementsov-Ogievskiy buf = qemu_try_blockalign0(bs, num); 19110d93ed08SVladimir Sementsov-Ogievskiy if (buf == NULL) { 191261007b31SStefan Hajnoczi ret = -ENOMEM; 191361007b31SStefan Hajnoczi goto fail; 191461007b31SStefan Hajnoczi } 191561007b31SStefan Hajnoczi } 19160d93ed08SVladimir Sementsov-Ogievskiy qemu_iovec_init_buf(&qiov, buf, num); 191761007b31SStefan Hajnoczi 1918ac850bf0SVladimir Sementsov-Ogievskiy ret = bdrv_driver_pwritev(bs, offset, num, &qiov, 0, write_flags); 191961007b31SStefan Hajnoczi 192061007b31SStefan Hajnoczi /* Keep bounce buffer around if it is big enough for all 192161007b31SStefan Hajnoczi * all future requests. 192261007b31SStefan Hajnoczi */ 19235def6b80SEric Blake if (num < max_transfer) { 19240d93ed08SVladimir Sementsov-Ogievskiy qemu_vfree(buf); 19250d93ed08SVladimir Sementsov-Ogievskiy buf = NULL; 192661007b31SStefan Hajnoczi } 192761007b31SStefan Hajnoczi } 192861007b31SStefan Hajnoczi 1929d05aa8bbSEric Blake offset += num; 1930f5a5ca79SManos Pitsidianakis bytes -= num; 193161007b31SStefan Hajnoczi } 193261007b31SStefan Hajnoczi 193361007b31SStefan Hajnoczi fail: 1934465fe887SEric Blake if (ret == 0 && need_flush) { 1935465fe887SEric Blake ret = bdrv_co_flush(bs); 1936465fe887SEric Blake } 19370d93ed08SVladimir Sementsov-Ogievskiy qemu_vfree(buf); 193861007b31SStefan Hajnoczi return ret; 193961007b31SStefan Hajnoczi } 194061007b31SStefan Hajnoczi 194185fe2479SFam Zheng static inline int coroutine_fn 1942fcfd9adeSVladimir Sementsov-Ogievskiy bdrv_co_write_req_prepare(BdrvChild *child, int64_t offset, int64_t bytes, 194385fe2479SFam Zheng BdrvTrackedRequest *req, int flags) 194485fe2479SFam Zheng { 194585fe2479SFam Zheng BlockDriverState *bs = child->bs; 1946fcfd9adeSVladimir Sementsov-Ogievskiy 1947fcfd9adeSVladimir Sementsov-Ogievskiy bdrv_check_request(offset, bytes, &error_abort); 194885fe2479SFam Zheng 1949307261b2SVladimir Sementsov-Ogievskiy if (bdrv_is_read_only(bs)) { 195085fe2479SFam Zheng return -EPERM; 195185fe2479SFam Zheng } 195285fe2479SFam Zheng 195385fe2479SFam Zheng assert(!(bs->open_flags & BDRV_O_INACTIVE)); 195485fe2479SFam Zheng assert((bs->open_flags & BDRV_O_NO_IO) == 0); 195585fe2479SFam Zheng assert(!(flags & ~BDRV_REQ_MASK)); 1956d1a764d1SVladimir Sementsov-Ogievskiy assert(!((flags & BDRV_REQ_NO_WAIT) && !(flags & BDRV_REQ_SERIALISING))); 195785fe2479SFam Zheng 195885fe2479SFam Zheng if (flags & BDRV_REQ_SERIALISING) { 1959d1a764d1SVladimir Sementsov-Ogievskiy QEMU_LOCK_GUARD(&bs->reqs_lock); 1960d1a764d1SVladimir Sementsov-Ogievskiy 1961d1a764d1SVladimir Sementsov-Ogievskiy tracked_request_set_serialising(req, bdrv_get_cluster_size(bs)); 1962d1a764d1SVladimir Sementsov-Ogievskiy 1963d1a764d1SVladimir Sementsov-Ogievskiy if ((flags & BDRV_REQ_NO_WAIT) && bdrv_find_conflicting_request(req)) { 1964d1a764d1SVladimir Sementsov-Ogievskiy return -EBUSY; 1965d1a764d1SVladimir Sementsov-Ogievskiy } 1966d1a764d1SVladimir Sementsov-Ogievskiy 1967d1a764d1SVladimir Sementsov-Ogievskiy bdrv_wait_serialising_requests_locked(req); 196818fbd0deSPaolo Bonzini } else { 196918fbd0deSPaolo Bonzini bdrv_wait_serialising_requests(req); 197085fe2479SFam Zheng } 197185fe2479SFam Zheng 197285fe2479SFam Zheng assert(req->overlap_offset <= offset); 197385fe2479SFam Zheng assert(offset + bytes <= req->overlap_offset + req->overlap_bytes); 1974fcfd9adeSVladimir Sementsov-Ogievskiy assert(offset + bytes <= bs->total_sectors * BDRV_SECTOR_SIZE || 1975fcfd9adeSVladimir Sementsov-Ogievskiy child->perm & BLK_PERM_RESIZE); 197685fe2479SFam Zheng 1977cd47d792SFam Zheng switch (req->type) { 1978cd47d792SFam Zheng case BDRV_TRACKED_WRITE: 1979cd47d792SFam Zheng case BDRV_TRACKED_DISCARD: 198085fe2479SFam Zheng if (flags & BDRV_REQ_WRITE_UNCHANGED) { 198185fe2479SFam Zheng assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE)); 198285fe2479SFam Zheng } else { 198385fe2479SFam Zheng assert(child->perm & BLK_PERM_WRITE); 198485fe2479SFam Zheng } 198594783301SVladimir Sementsov-Ogievskiy bdrv_write_threshold_check_write(bs, offset, bytes); 198694783301SVladimir Sementsov-Ogievskiy return 0; 1987cd47d792SFam Zheng case BDRV_TRACKED_TRUNCATE: 1988cd47d792SFam Zheng assert(child->perm & BLK_PERM_RESIZE); 1989cd47d792SFam Zheng return 0; 1990cd47d792SFam Zheng default: 1991cd47d792SFam Zheng abort(); 1992cd47d792SFam Zheng } 199385fe2479SFam Zheng } 199485fe2479SFam Zheng 199585fe2479SFam Zheng static inline void coroutine_fn 1996fcfd9adeSVladimir Sementsov-Ogievskiy bdrv_co_write_req_finish(BdrvChild *child, int64_t offset, int64_t bytes, 199785fe2479SFam Zheng BdrvTrackedRequest *req, int ret) 199885fe2479SFam Zheng { 199985fe2479SFam Zheng int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 200085fe2479SFam Zheng BlockDriverState *bs = child->bs; 200185fe2479SFam Zheng 2002fcfd9adeSVladimir Sementsov-Ogievskiy bdrv_check_request(offset, bytes, &error_abort); 2003fcfd9adeSVladimir Sementsov-Ogievskiy 2004d73415a3SStefan Hajnoczi qatomic_inc(&bs->write_gen); 200585fe2479SFam Zheng 200600695c27SFam Zheng /* 200700695c27SFam Zheng * Discard cannot extend the image, but in error handling cases, such as 200800695c27SFam Zheng * when reverting a qcow2 cluster allocation, the discarded range can pass 200900695c27SFam Zheng * the end of image file, so we cannot assert about BDRV_TRACKED_DISCARD 201000695c27SFam Zheng * here. Instead, just skip it, since semantically a discard request 201100695c27SFam Zheng * beyond EOF cannot expand the image anyway. 201200695c27SFam Zheng */ 20137f8f03efSFam Zheng if (ret == 0 && 2014cd47d792SFam Zheng (req->type == BDRV_TRACKED_TRUNCATE || 2015cd47d792SFam Zheng end_sector > bs->total_sectors) && 201600695c27SFam Zheng req->type != BDRV_TRACKED_DISCARD) { 20177f8f03efSFam Zheng bs->total_sectors = end_sector; 20187f8f03efSFam Zheng bdrv_parent_cb_resize(bs); 20197f8f03efSFam Zheng bdrv_dirty_bitmap_truncate(bs, end_sector << BDRV_SECTOR_BITS); 202085fe2479SFam Zheng } 202100695c27SFam Zheng if (req->bytes) { 202200695c27SFam Zheng switch (req->type) { 202300695c27SFam Zheng case BDRV_TRACKED_WRITE: 202400695c27SFam Zheng stat64_max(&bs->wr_highest_offset, offset + bytes); 202500695c27SFam Zheng /* fall through, to set dirty bits */ 202600695c27SFam Zheng case BDRV_TRACKED_DISCARD: 20277f8f03efSFam Zheng bdrv_set_dirty(bs, offset, bytes); 202800695c27SFam Zheng break; 202900695c27SFam Zheng default: 203000695c27SFam Zheng break; 203100695c27SFam Zheng } 203200695c27SFam Zheng } 203385fe2479SFam Zheng } 203485fe2479SFam Zheng 203561007b31SStefan Hajnoczi /* 203604ed95f4SEric Blake * Forwards an already correctly aligned write request to the BlockDriver, 203704ed95f4SEric Blake * after possibly fragmenting it. 203861007b31SStefan Hajnoczi */ 203985c97ca7SKevin Wolf static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child, 2040fcfd9adeSVladimir Sementsov-Ogievskiy BdrvTrackedRequest *req, int64_t offset, int64_t bytes, 2041e75abedaSVladimir Sementsov-Ogievskiy int64_t align, QEMUIOVector *qiov, size_t qiov_offset, 2042e75abedaSVladimir Sementsov-Ogievskiy BdrvRequestFlags flags) 204361007b31SStefan Hajnoczi { 204485c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 204561007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 204661007b31SStefan Hajnoczi int ret; 204761007b31SStefan Hajnoczi 2048fcfd9adeSVladimir Sementsov-Ogievskiy int64_t bytes_remaining = bytes; 204904ed95f4SEric Blake int max_transfer; 205061007b31SStefan Hajnoczi 2051fcfd9adeSVladimir Sementsov-Ogievskiy bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 2052fcfd9adeSVladimir Sementsov-Ogievskiy 2053d470ad42SMax Reitz if (!drv) { 2054d470ad42SMax Reitz return -ENOMEDIUM; 2055d470ad42SMax Reitz } 2056d470ad42SMax Reitz 2057d6883bc9SVladimir Sementsov-Ogievskiy if (bdrv_has_readonly_bitmaps(bs)) { 2058d6883bc9SVladimir Sementsov-Ogievskiy return -EPERM; 2059d6883bc9SVladimir Sementsov-Ogievskiy } 2060d6883bc9SVladimir Sementsov-Ogievskiy 2061cff86b38SEric Blake assert(is_power_of_2(align)); 2062cff86b38SEric Blake assert((offset & (align - 1)) == 0); 2063cff86b38SEric Blake assert((bytes & (align - 1)) == 0); 206404ed95f4SEric Blake max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 206504ed95f4SEric Blake align); 206661007b31SStefan Hajnoczi 206785fe2479SFam Zheng ret = bdrv_co_write_req_prepare(child, offset, bytes, req, flags); 206861007b31SStefan Hajnoczi 206961007b31SStefan Hajnoczi if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF && 2070c1499a5eSEric Blake !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes && 207128c4da28SVladimir Sementsov-Ogievskiy qemu_iovec_is_zero(qiov, qiov_offset, bytes)) { 207261007b31SStefan Hajnoczi flags |= BDRV_REQ_ZERO_WRITE; 207361007b31SStefan Hajnoczi if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) { 207461007b31SStefan Hajnoczi flags |= BDRV_REQ_MAY_UNMAP; 207561007b31SStefan Hajnoczi } 207661007b31SStefan Hajnoczi } 207761007b31SStefan Hajnoczi 207861007b31SStefan Hajnoczi if (ret < 0) { 207961007b31SStefan Hajnoczi /* Do nothing, write notifier decided to fail this request */ 208061007b31SStefan Hajnoczi } else if (flags & BDRV_REQ_ZERO_WRITE) { 20819a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO); 20829896c876SKevin Wolf ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags); 20833ea1a091SPavel Butsykin } else if (flags & BDRV_REQ_WRITE_COMPRESSED) { 208428c4da28SVladimir Sementsov-Ogievskiy ret = bdrv_driver_pwritev_compressed(bs, offset, bytes, 208528c4da28SVladimir Sementsov-Ogievskiy qiov, qiov_offset); 208604ed95f4SEric Blake } else if (bytes <= max_transfer) { 20879a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV); 208828c4da28SVladimir Sementsov-Ogievskiy ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, qiov_offset, flags); 208904ed95f4SEric Blake } else { 209004ed95f4SEric Blake bdrv_debug_event(bs, BLKDBG_PWRITEV); 209104ed95f4SEric Blake while (bytes_remaining) { 209204ed95f4SEric Blake int num = MIN(bytes_remaining, max_transfer); 209304ed95f4SEric Blake int local_flags = flags; 209404ed95f4SEric Blake 209504ed95f4SEric Blake assert(num); 209604ed95f4SEric Blake if (num < bytes_remaining && (flags & BDRV_REQ_FUA) && 209704ed95f4SEric Blake !(bs->supported_write_flags & BDRV_REQ_FUA)) { 209804ed95f4SEric Blake /* If FUA is going to be emulated by flush, we only 209904ed95f4SEric Blake * need to flush on the last iteration */ 210004ed95f4SEric Blake local_flags &= ~BDRV_REQ_FUA; 210104ed95f4SEric Blake } 210204ed95f4SEric Blake 210304ed95f4SEric Blake ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining, 2104134b7decSMax Reitz num, qiov, 2105134b7decSMax Reitz qiov_offset + bytes - bytes_remaining, 210628c4da28SVladimir Sementsov-Ogievskiy local_flags); 210704ed95f4SEric Blake if (ret < 0) { 210804ed95f4SEric Blake break; 210904ed95f4SEric Blake } 211004ed95f4SEric Blake bytes_remaining -= num; 211104ed95f4SEric Blake } 211261007b31SStefan Hajnoczi } 21139a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE); 211461007b31SStefan Hajnoczi 211561007b31SStefan Hajnoczi if (ret >= 0) { 211604ed95f4SEric Blake ret = 0; 211761007b31SStefan Hajnoczi } 211885fe2479SFam Zheng bdrv_co_write_req_finish(child, offset, bytes, req, ret); 211961007b31SStefan Hajnoczi 212061007b31SStefan Hajnoczi return ret; 212161007b31SStefan Hajnoczi } 212261007b31SStefan Hajnoczi 212385c97ca7SKevin Wolf static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child, 21249eeb6dd1SFam Zheng int64_t offset, 212537e9403eSVladimir Sementsov-Ogievskiy int64_t bytes, 21269eeb6dd1SFam Zheng BdrvRequestFlags flags, 21279eeb6dd1SFam Zheng BdrvTrackedRequest *req) 21289eeb6dd1SFam Zheng { 212985c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 21309eeb6dd1SFam Zheng QEMUIOVector local_qiov; 2131a5b8dd2cSEric Blake uint64_t align = bs->bl.request_alignment; 21329eeb6dd1SFam Zheng int ret = 0; 21337a3f542fSVladimir Sementsov-Ogievskiy bool padding; 21347a3f542fSVladimir Sementsov-Ogievskiy BdrvRequestPadding pad; 21359eeb6dd1SFam Zheng 21367a3f542fSVladimir Sementsov-Ogievskiy padding = bdrv_init_padding(bs, offset, bytes, &pad); 21377a3f542fSVladimir Sementsov-Ogievskiy if (padding) { 213845e62b46SVladimir Sementsov-Ogievskiy assert(!(flags & BDRV_REQ_NO_WAIT)); 21398ac5aab2SVladimir Sementsov-Ogievskiy bdrv_make_request_serialising(req, align); 21409eeb6dd1SFam Zheng 21417a3f542fSVladimir Sementsov-Ogievskiy bdrv_padding_rmw_read(child, req, &pad, true); 21427a3f542fSVladimir Sementsov-Ogievskiy 21437a3f542fSVladimir Sementsov-Ogievskiy if (pad.head || pad.merge_reads) { 21447a3f542fSVladimir Sementsov-Ogievskiy int64_t aligned_offset = offset & ~(align - 1); 21457a3f542fSVladimir Sementsov-Ogievskiy int64_t write_bytes = pad.merge_reads ? pad.buf_len : align; 21467a3f542fSVladimir Sementsov-Ogievskiy 21477a3f542fSVladimir Sementsov-Ogievskiy qemu_iovec_init_buf(&local_qiov, pad.buf, write_bytes); 21487a3f542fSVladimir Sementsov-Ogievskiy ret = bdrv_aligned_pwritev(child, req, aligned_offset, write_bytes, 214928c4da28SVladimir Sementsov-Ogievskiy align, &local_qiov, 0, 21509eeb6dd1SFam Zheng flags & ~BDRV_REQ_ZERO_WRITE); 21517a3f542fSVladimir Sementsov-Ogievskiy if (ret < 0 || pad.merge_reads) { 21527a3f542fSVladimir Sementsov-Ogievskiy /* Error or all work is done */ 21537a3f542fSVladimir Sementsov-Ogievskiy goto out; 21549eeb6dd1SFam Zheng } 21557a3f542fSVladimir Sementsov-Ogievskiy offset += write_bytes - pad.head; 21567a3f542fSVladimir Sementsov-Ogievskiy bytes -= write_bytes - pad.head; 21577a3f542fSVladimir Sementsov-Ogievskiy } 21589eeb6dd1SFam Zheng } 21599eeb6dd1SFam Zheng 21609eeb6dd1SFam Zheng assert(!bytes || (offset & (align - 1)) == 0); 21619eeb6dd1SFam Zheng if (bytes >= align) { 21629eeb6dd1SFam Zheng /* Write the aligned part in the middle. */ 2163fcfd9adeSVladimir Sementsov-Ogievskiy int64_t aligned_bytes = bytes & ~(align - 1); 216485c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align, 216528c4da28SVladimir Sementsov-Ogievskiy NULL, 0, flags); 21669eeb6dd1SFam Zheng if (ret < 0) { 21677a3f542fSVladimir Sementsov-Ogievskiy goto out; 21689eeb6dd1SFam Zheng } 21699eeb6dd1SFam Zheng bytes -= aligned_bytes; 21709eeb6dd1SFam Zheng offset += aligned_bytes; 21719eeb6dd1SFam Zheng } 21729eeb6dd1SFam Zheng 21739eeb6dd1SFam Zheng assert(!bytes || (offset & (align - 1)) == 0); 21749eeb6dd1SFam Zheng if (bytes) { 21757a3f542fSVladimir Sementsov-Ogievskiy assert(align == pad.tail + bytes); 21769eeb6dd1SFam Zheng 21777a3f542fSVladimir Sementsov-Ogievskiy qemu_iovec_init_buf(&local_qiov, pad.tail_buf, align); 217885c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, req, offset, align, align, 217928c4da28SVladimir Sementsov-Ogievskiy &local_qiov, 0, 218028c4da28SVladimir Sementsov-Ogievskiy flags & ~BDRV_REQ_ZERO_WRITE); 21819eeb6dd1SFam Zheng } 21829eeb6dd1SFam Zheng 21837a3f542fSVladimir Sementsov-Ogievskiy out: 21847a3f542fSVladimir Sementsov-Ogievskiy bdrv_padding_destroy(&pad); 21857a3f542fSVladimir Sementsov-Ogievskiy 21867a3f542fSVladimir Sementsov-Ogievskiy return ret; 21879eeb6dd1SFam Zheng } 21889eeb6dd1SFam Zheng 218961007b31SStefan Hajnoczi /* 219061007b31SStefan Hajnoczi * Handle a write request in coroutine context 219161007b31SStefan Hajnoczi */ 2192a03ef88fSKevin Wolf int coroutine_fn bdrv_co_pwritev(BdrvChild *child, 2193e9e52efdSVladimir Sementsov-Ogievskiy int64_t offset, int64_t bytes, QEMUIOVector *qiov, 219461007b31SStefan Hajnoczi BdrvRequestFlags flags) 219561007b31SStefan Hajnoczi { 2196967d7905SEmanuele Giuseppe Esposito IO_CODE(); 21971acc3466SVladimir Sementsov-Ogievskiy return bdrv_co_pwritev_part(child, offset, bytes, qiov, 0, flags); 21981acc3466SVladimir Sementsov-Ogievskiy } 21991acc3466SVladimir Sementsov-Ogievskiy 22001acc3466SVladimir Sementsov-Ogievskiy int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child, 220137e9403eSVladimir Sementsov-Ogievskiy int64_t offset, int64_t bytes, QEMUIOVector *qiov, size_t qiov_offset, 22021acc3466SVladimir Sementsov-Ogievskiy BdrvRequestFlags flags) 22031acc3466SVladimir Sementsov-Ogievskiy { 2204a03ef88fSKevin Wolf BlockDriverState *bs = child->bs; 220561007b31SStefan Hajnoczi BdrvTrackedRequest req; 2206a5b8dd2cSEric Blake uint64_t align = bs->bl.request_alignment; 22077a3f542fSVladimir Sementsov-Ogievskiy BdrvRequestPadding pad; 220861007b31SStefan Hajnoczi int ret; 2209f0deecffSVladimir Sementsov-Ogievskiy bool padded = false; 2210967d7905SEmanuele Giuseppe Esposito IO_CODE(); 221161007b31SStefan Hajnoczi 221237e9403eSVladimir Sementsov-Ogievskiy trace_bdrv_co_pwritev_part(child->bs, offset, bytes, flags); 2213f42cf447SDaniel P. Berrange 2214f4dad307SVladimir Sementsov-Ogievskiy if (!bdrv_is_inserted(bs)) { 221561007b31SStefan Hajnoczi return -ENOMEDIUM; 221661007b31SStefan Hajnoczi } 221761007b31SStefan Hajnoczi 22182aaa3f9bSVladimir Sementsov-Ogievskiy if (flags & BDRV_REQ_ZERO_WRITE) { 22192aaa3f9bSVladimir Sementsov-Ogievskiy ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL); 22202aaa3f9bSVladimir Sementsov-Ogievskiy } else { 222163f4ad11SVladimir Sementsov-Ogievskiy ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset); 22222aaa3f9bSVladimir Sementsov-Ogievskiy } 222361007b31SStefan Hajnoczi if (ret < 0) { 222461007b31SStefan Hajnoczi return ret; 222561007b31SStefan Hajnoczi } 222661007b31SStefan Hajnoczi 2227f2208fdcSAlberto Garcia /* If the request is misaligned then we can't make it efficient */ 2228f2208fdcSAlberto Garcia if ((flags & BDRV_REQ_NO_FALLBACK) && 2229f2208fdcSAlberto Garcia !QEMU_IS_ALIGNED(offset | bytes, align)) 2230f2208fdcSAlberto Garcia { 2231f2208fdcSAlberto Garcia return -ENOTSUP; 2232f2208fdcSAlberto Garcia } 2233f2208fdcSAlberto Garcia 2234ac9d00bfSVladimir Sementsov-Ogievskiy if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) { 2235ac9d00bfSVladimir Sementsov-Ogievskiy /* 2236ac9d00bfSVladimir Sementsov-Ogievskiy * Aligning zero request is nonsense. Even if driver has special meaning 2237ac9d00bfSVladimir Sementsov-Ogievskiy * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass 2238ac9d00bfSVladimir Sementsov-Ogievskiy * it to driver due to request_alignment. 2239ac9d00bfSVladimir Sementsov-Ogievskiy * 2240ac9d00bfSVladimir Sementsov-Ogievskiy * Still, no reason to return an error if someone do unaligned 2241ac9d00bfSVladimir Sementsov-Ogievskiy * zero-length write occasionally. 2242ac9d00bfSVladimir Sementsov-Ogievskiy */ 2243ac9d00bfSVladimir Sementsov-Ogievskiy return 0; 2244ac9d00bfSVladimir Sementsov-Ogievskiy } 2245ac9d00bfSVladimir Sementsov-Ogievskiy 2246f0deecffSVladimir Sementsov-Ogievskiy if (!(flags & BDRV_REQ_ZERO_WRITE)) { 224761007b31SStefan Hajnoczi /* 2248f0deecffSVladimir Sementsov-Ogievskiy * Pad request for following read-modify-write cycle. 2249f0deecffSVladimir Sementsov-Ogievskiy * bdrv_co_do_zero_pwritev() does aligning by itself, so, we do 2250f0deecffSVladimir Sementsov-Ogievskiy * alignment only if there is no ZERO flag. 225161007b31SStefan Hajnoczi */ 225298ca4549SVladimir Sementsov-Ogievskiy ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad, 225398ca4549SVladimir Sementsov-Ogievskiy &padded); 225498ca4549SVladimir Sementsov-Ogievskiy if (ret < 0) { 225598ca4549SVladimir Sementsov-Ogievskiy return ret; 225698ca4549SVladimir Sementsov-Ogievskiy } 2257f0deecffSVladimir Sementsov-Ogievskiy } 2258f0deecffSVladimir Sementsov-Ogievskiy 2259f0deecffSVladimir Sementsov-Ogievskiy bdrv_inc_in_flight(bs); 2260ebde595cSFam Zheng tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE); 226161007b31SStefan Hajnoczi 226218a59f03SAnton Nefedov if (flags & BDRV_REQ_ZERO_WRITE) { 2263f0deecffSVladimir Sementsov-Ogievskiy assert(!padded); 226485c97ca7SKevin Wolf ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req); 22659eeb6dd1SFam Zheng goto out; 22669eeb6dd1SFam Zheng } 22679eeb6dd1SFam Zheng 2268f0deecffSVladimir Sementsov-Ogievskiy if (padded) { 2269f0deecffSVladimir Sementsov-Ogievskiy /* 2270f0deecffSVladimir Sementsov-Ogievskiy * Request was unaligned to request_alignment and therefore 2271f0deecffSVladimir Sementsov-Ogievskiy * padded. We are going to do read-modify-write, and must 2272f0deecffSVladimir Sementsov-Ogievskiy * serialize the request to prevent interactions of the 2273f0deecffSVladimir Sementsov-Ogievskiy * widened region with other transactions. 2274f0deecffSVladimir Sementsov-Ogievskiy */ 227545e62b46SVladimir Sementsov-Ogievskiy assert(!(flags & BDRV_REQ_NO_WAIT)); 22768ac5aab2SVladimir Sementsov-Ogievskiy bdrv_make_request_serialising(&req, align); 22777a3f542fSVladimir Sementsov-Ogievskiy bdrv_padding_rmw_read(child, &req, &pad, false); 227861007b31SStefan Hajnoczi } 227961007b31SStefan Hajnoczi 228085c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align, 22811acc3466SVladimir Sementsov-Ogievskiy qiov, qiov_offset, flags); 228261007b31SStefan Hajnoczi 22837a3f542fSVladimir Sementsov-Ogievskiy bdrv_padding_destroy(&pad); 228461007b31SStefan Hajnoczi 22859eeb6dd1SFam Zheng out: 22869eeb6dd1SFam Zheng tracked_request_end(&req); 228799723548SPaolo Bonzini bdrv_dec_in_flight(bs); 22887a3f542fSVladimir Sementsov-Ogievskiy 228961007b31SStefan Hajnoczi return ret; 229061007b31SStefan Hajnoczi } 229161007b31SStefan Hajnoczi 2292a03ef88fSKevin Wolf int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset, 2293e9e52efdSVladimir Sementsov-Ogievskiy int64_t bytes, BdrvRequestFlags flags) 229461007b31SStefan Hajnoczi { 2295384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 2296f5a5ca79SManos Pitsidianakis trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags); 229761007b31SStefan Hajnoczi 2298a03ef88fSKevin Wolf if (!(child->bs->open_flags & BDRV_O_UNMAP)) { 229961007b31SStefan Hajnoczi flags &= ~BDRV_REQ_MAY_UNMAP; 230061007b31SStefan Hajnoczi } 230161007b31SStefan Hajnoczi 2302f5a5ca79SManos Pitsidianakis return bdrv_co_pwritev(child, offset, bytes, NULL, 230361007b31SStefan Hajnoczi BDRV_REQ_ZERO_WRITE | flags); 230461007b31SStefan Hajnoczi } 230561007b31SStefan Hajnoczi 23064085f5c7SJohn Snow /* 23074085f5c7SJohn Snow * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not. 23084085f5c7SJohn Snow */ 23094085f5c7SJohn Snow int bdrv_flush_all(void) 23104085f5c7SJohn Snow { 23114085f5c7SJohn Snow BdrvNextIterator it; 23124085f5c7SJohn Snow BlockDriverState *bs = NULL; 23134085f5c7SJohn Snow int result = 0; 23144085f5c7SJohn Snow 2315f791bf7fSEmanuele Giuseppe Esposito GLOBAL_STATE_CODE(); 2316f791bf7fSEmanuele Giuseppe Esposito 2317c8aa7895SPavel Dovgalyuk /* 2318c8aa7895SPavel Dovgalyuk * bdrv queue is managed by record/replay, 2319c8aa7895SPavel Dovgalyuk * creating new flush request for stopping 2320c8aa7895SPavel Dovgalyuk * the VM may break the determinism 2321c8aa7895SPavel Dovgalyuk */ 2322c8aa7895SPavel Dovgalyuk if (replay_events_enabled()) { 2323c8aa7895SPavel Dovgalyuk return result; 2324c8aa7895SPavel Dovgalyuk } 2325c8aa7895SPavel Dovgalyuk 23264085f5c7SJohn Snow for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 23274085f5c7SJohn Snow AioContext *aio_context = bdrv_get_aio_context(bs); 23284085f5c7SJohn Snow int ret; 23294085f5c7SJohn Snow 23304085f5c7SJohn Snow aio_context_acquire(aio_context); 23314085f5c7SJohn Snow ret = bdrv_flush(bs); 23324085f5c7SJohn Snow if (ret < 0 && !result) { 23334085f5c7SJohn Snow result = ret; 23344085f5c7SJohn Snow } 23354085f5c7SJohn Snow aio_context_release(aio_context); 23364085f5c7SJohn Snow } 23374085f5c7SJohn Snow 23384085f5c7SJohn Snow return result; 23394085f5c7SJohn Snow } 23404085f5c7SJohn Snow 234161007b31SStefan Hajnoczi /* 234261007b31SStefan Hajnoczi * Returns the allocation status of the specified sectors. 234361007b31SStefan Hajnoczi * Drivers not implementing the functionality are assumed to not support 234461007b31SStefan Hajnoczi * backing files, hence all their sectors are reported as allocated. 234561007b31SStefan Hajnoczi * 234686a3d5c6SEric Blake * If 'want_zero' is true, the caller is querying for mapping 234786a3d5c6SEric Blake * purposes, with a focus on valid BDRV_BLOCK_OFFSET_VALID, _DATA, and 234886a3d5c6SEric Blake * _ZERO where possible; otherwise, the result favors larger 'pnum', 234986a3d5c6SEric Blake * with a focus on accurate BDRV_BLOCK_ALLOCATED. 2350c9ce8c4dSEric Blake * 23512e8bc787SEric Blake * If 'offset' is beyond the end of the disk image the return value is 2352fb0d8654SEric Blake * BDRV_BLOCK_EOF and 'pnum' is set to 0. 235361007b31SStefan Hajnoczi * 23542e8bc787SEric Blake * 'bytes' is the max value 'pnum' should be set to. If bytes goes 2355fb0d8654SEric Blake * beyond the end of the disk image it will be clamped; if 'pnum' is set to 2356fb0d8654SEric Blake * the end of the image, then the returned value will include BDRV_BLOCK_EOF. 235767a0fd2aSFam Zheng * 23582e8bc787SEric Blake * 'pnum' is set to the number of bytes (including and immediately 23592e8bc787SEric Blake * following the specified offset) that are easily known to be in the 23602e8bc787SEric Blake * same allocated/unallocated state. Note that a second call starting 23612e8bc787SEric Blake * at the original offset plus returned pnum may have the same status. 23622e8bc787SEric Blake * The returned value is non-zero on success except at end-of-file. 23632e8bc787SEric Blake * 23642e8bc787SEric Blake * Returns negative errno on failure. Otherwise, if the 23652e8bc787SEric Blake * BDRV_BLOCK_OFFSET_VALID bit is set, 'map' and 'file' (if non-NULL) are 23662e8bc787SEric Blake * set to the host mapping and BDS corresponding to the guest offset. 236761007b31SStefan Hajnoczi */ 23682e8bc787SEric Blake static int coroutine_fn bdrv_co_block_status(BlockDriverState *bs, 2369c9ce8c4dSEric Blake bool want_zero, 23702e8bc787SEric Blake int64_t offset, int64_t bytes, 23712e8bc787SEric Blake int64_t *pnum, int64_t *map, 237267a0fd2aSFam Zheng BlockDriverState **file) 237361007b31SStefan Hajnoczi { 23742e8bc787SEric Blake int64_t total_size; 23752e8bc787SEric Blake int64_t n; /* bytes */ 2376efa6e2edSEric Blake int ret; 23772e8bc787SEric Blake int64_t local_map = 0; 2378298a1665SEric Blake BlockDriverState *local_file = NULL; 2379efa6e2edSEric Blake int64_t aligned_offset, aligned_bytes; 2380efa6e2edSEric Blake uint32_t align; 2381549ec0d9SMax Reitz bool has_filtered_child; 238261007b31SStefan Hajnoczi 2383298a1665SEric Blake assert(pnum); 2384298a1665SEric Blake *pnum = 0; 23852e8bc787SEric Blake total_size = bdrv_getlength(bs); 23862e8bc787SEric Blake if (total_size < 0) { 23872e8bc787SEric Blake ret = total_size; 2388298a1665SEric Blake goto early_out; 238961007b31SStefan Hajnoczi } 239061007b31SStefan Hajnoczi 23912e8bc787SEric Blake if (offset >= total_size) { 2392298a1665SEric Blake ret = BDRV_BLOCK_EOF; 2393298a1665SEric Blake goto early_out; 239461007b31SStefan Hajnoczi } 23952e8bc787SEric Blake if (!bytes) { 2396298a1665SEric Blake ret = 0; 2397298a1665SEric Blake goto early_out; 23989cdcfd9fSEric Blake } 239961007b31SStefan Hajnoczi 24002e8bc787SEric Blake n = total_size - offset; 24012e8bc787SEric Blake if (n < bytes) { 24022e8bc787SEric Blake bytes = n; 240361007b31SStefan Hajnoczi } 240461007b31SStefan Hajnoczi 2405d470ad42SMax Reitz /* Must be non-NULL or bdrv_getlength() would have failed */ 2406d470ad42SMax Reitz assert(bs->drv); 2407549ec0d9SMax Reitz has_filtered_child = bdrv_filter_child(bs); 2408549ec0d9SMax Reitz if (!bs->drv->bdrv_co_block_status && !has_filtered_child) { 24092e8bc787SEric Blake *pnum = bytes; 241061007b31SStefan Hajnoczi ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED; 24112e8bc787SEric Blake if (offset + bytes == total_size) { 2412fb0d8654SEric Blake ret |= BDRV_BLOCK_EOF; 2413fb0d8654SEric Blake } 241461007b31SStefan Hajnoczi if (bs->drv->protocol_name) { 24152e8bc787SEric Blake ret |= BDRV_BLOCK_OFFSET_VALID; 24162e8bc787SEric Blake local_map = offset; 2417298a1665SEric Blake local_file = bs; 241861007b31SStefan Hajnoczi } 2419298a1665SEric Blake goto early_out; 242061007b31SStefan Hajnoczi } 242161007b31SStefan Hajnoczi 242299723548SPaolo Bonzini bdrv_inc_in_flight(bs); 2423efa6e2edSEric Blake 2424efa6e2edSEric Blake /* Round out to request_alignment boundaries */ 242586a3d5c6SEric Blake align = bs->bl.request_alignment; 2426efa6e2edSEric Blake aligned_offset = QEMU_ALIGN_DOWN(offset, align); 2427efa6e2edSEric Blake aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset; 2428efa6e2edSEric Blake 2429549ec0d9SMax Reitz if (bs->drv->bdrv_co_block_status) { 24300bc329fbSHanna Reitz /* 24310bc329fbSHanna Reitz * Use the block-status cache only for protocol nodes: Format 24320bc329fbSHanna Reitz * drivers are generally quick to inquire the status, but protocol 24330bc329fbSHanna Reitz * drivers often need to get information from outside of qemu, so 24340bc329fbSHanna Reitz * we do not have control over the actual implementation. There 24350bc329fbSHanna Reitz * have been cases where inquiring the status took an unreasonably 24360bc329fbSHanna Reitz * long time, and we can do nothing in qemu to fix it. 24370bc329fbSHanna Reitz * This is especially problematic for images with large data areas, 24380bc329fbSHanna Reitz * because finding the few holes in them and giving them special 24390bc329fbSHanna Reitz * treatment does not gain much performance. Therefore, we try to 24400bc329fbSHanna Reitz * cache the last-identified data region. 24410bc329fbSHanna Reitz * 24420bc329fbSHanna Reitz * Second, limiting ourselves to protocol nodes allows us to assume 24430bc329fbSHanna Reitz * the block status for data regions to be DATA | OFFSET_VALID, and 24440bc329fbSHanna Reitz * that the host offset is the same as the guest offset. 24450bc329fbSHanna Reitz * 24460bc329fbSHanna Reitz * Note that it is possible that external writers zero parts of 24470bc329fbSHanna Reitz * the cached regions without the cache being invalidated, and so 24480bc329fbSHanna Reitz * we may report zeroes as data. This is not catastrophic, 24490bc329fbSHanna Reitz * however, because reporting zeroes as data is fine. 24500bc329fbSHanna Reitz */ 24510bc329fbSHanna Reitz if (QLIST_EMPTY(&bs->children) && 24520bc329fbSHanna Reitz bdrv_bsc_is_data(bs, aligned_offset, pnum)) 24530bc329fbSHanna Reitz { 24540bc329fbSHanna Reitz ret = BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID; 24550bc329fbSHanna Reitz local_file = bs; 24560bc329fbSHanna Reitz local_map = aligned_offset; 24570bc329fbSHanna Reitz } else { 245886a3d5c6SEric Blake ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset, 245986a3d5c6SEric Blake aligned_bytes, pnum, &local_map, 246086a3d5c6SEric Blake &local_file); 24610bc329fbSHanna Reitz 24620bc329fbSHanna Reitz /* 24630bc329fbSHanna Reitz * Note that checking QLIST_EMPTY(&bs->children) is also done when 24640bc329fbSHanna Reitz * the cache is queried above. Technically, we do not need to check 24650bc329fbSHanna Reitz * it here; the worst that can happen is that we fill the cache for 24660bc329fbSHanna Reitz * non-protocol nodes, and then it is never used. However, filling 24670bc329fbSHanna Reitz * the cache requires an RCU update, so double check here to avoid 24680bc329fbSHanna Reitz * such an update if possible. 2469113b727cSHanna Reitz * 2470113b727cSHanna Reitz * Check want_zero, because we only want to update the cache when we 2471113b727cSHanna Reitz * have accurate information about what is zero and what is data. 24720bc329fbSHanna Reitz */ 2473113b727cSHanna Reitz if (want_zero && 2474113b727cSHanna Reitz ret == (BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID) && 24750bc329fbSHanna Reitz QLIST_EMPTY(&bs->children)) 24760bc329fbSHanna Reitz { 24770bc329fbSHanna Reitz /* 24780bc329fbSHanna Reitz * When a protocol driver reports BLOCK_OFFSET_VALID, the 24790bc329fbSHanna Reitz * returned local_map value must be the same as the offset we 24800bc329fbSHanna Reitz * have passed (aligned_offset), and local_bs must be the node 24810bc329fbSHanna Reitz * itself. 24820bc329fbSHanna Reitz * Assert this, because we follow this rule when reading from 24830bc329fbSHanna Reitz * the cache (see the `local_file = bs` and 24840bc329fbSHanna Reitz * `local_map = aligned_offset` assignments above), and the 24850bc329fbSHanna Reitz * result the cache delivers must be the same as the driver 24860bc329fbSHanna Reitz * would deliver. 24870bc329fbSHanna Reitz */ 24880bc329fbSHanna Reitz assert(local_file == bs); 24890bc329fbSHanna Reitz assert(local_map == aligned_offset); 24900bc329fbSHanna Reitz bdrv_bsc_fill(bs, aligned_offset, *pnum); 24910bc329fbSHanna Reitz } 24920bc329fbSHanna Reitz } 2493549ec0d9SMax Reitz } else { 2494549ec0d9SMax Reitz /* Default code for filters */ 2495549ec0d9SMax Reitz 2496549ec0d9SMax Reitz local_file = bdrv_filter_bs(bs); 2497549ec0d9SMax Reitz assert(local_file); 2498549ec0d9SMax Reitz 2499549ec0d9SMax Reitz *pnum = aligned_bytes; 2500549ec0d9SMax Reitz local_map = aligned_offset; 2501549ec0d9SMax Reitz ret = BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID; 2502549ec0d9SMax Reitz } 250386a3d5c6SEric Blake if (ret < 0) { 250486a3d5c6SEric Blake *pnum = 0; 250586a3d5c6SEric Blake goto out; 250686a3d5c6SEric Blake } 2507efa6e2edSEric Blake 2508efa6e2edSEric Blake /* 2509636cb512SEric Blake * The driver's result must be a non-zero multiple of request_alignment. 2510efa6e2edSEric Blake * Clamp pnum and adjust map to original request. 2511efa6e2edSEric Blake */ 2512636cb512SEric Blake assert(*pnum && QEMU_IS_ALIGNED(*pnum, align) && 2513636cb512SEric Blake align > offset - aligned_offset); 251469f47505SVladimir Sementsov-Ogievskiy if (ret & BDRV_BLOCK_RECURSE) { 251569f47505SVladimir Sementsov-Ogievskiy assert(ret & BDRV_BLOCK_DATA); 251669f47505SVladimir Sementsov-Ogievskiy assert(ret & BDRV_BLOCK_OFFSET_VALID); 251769f47505SVladimir Sementsov-Ogievskiy assert(!(ret & BDRV_BLOCK_ZERO)); 251869f47505SVladimir Sementsov-Ogievskiy } 251969f47505SVladimir Sementsov-Ogievskiy 2520efa6e2edSEric Blake *pnum -= offset - aligned_offset; 2521efa6e2edSEric Blake if (*pnum > bytes) { 2522efa6e2edSEric Blake *pnum = bytes; 2523efa6e2edSEric Blake } 2524efa6e2edSEric Blake if (ret & BDRV_BLOCK_OFFSET_VALID) { 2525efa6e2edSEric Blake local_map += offset - aligned_offset; 2526efa6e2edSEric Blake } 252761007b31SStefan Hajnoczi 252861007b31SStefan Hajnoczi if (ret & BDRV_BLOCK_RAW) { 2529298a1665SEric Blake assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file); 25302e8bc787SEric Blake ret = bdrv_co_block_status(local_file, want_zero, local_map, 25312e8bc787SEric Blake *pnum, pnum, &local_map, &local_file); 253299723548SPaolo Bonzini goto out; 253361007b31SStefan Hajnoczi } 253461007b31SStefan Hajnoczi 253561007b31SStefan Hajnoczi if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) { 253661007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ALLOCATED; 2537d40f4a56SAlberto Garcia } else if (bs->drv->supports_backing) { 2538cb850315SMax Reitz BlockDriverState *cow_bs = bdrv_cow_bs(bs); 2539cb850315SMax Reitz 2540d40f4a56SAlberto Garcia if (!cow_bs) { 2541d40f4a56SAlberto Garcia ret |= BDRV_BLOCK_ZERO; 2542d40f4a56SAlberto Garcia } else if (want_zero) { 2543cb850315SMax Reitz int64_t size2 = bdrv_getlength(cow_bs); 2544c9ce8c4dSEric Blake 25452e8bc787SEric Blake if (size2 >= 0 && offset >= size2) { 254661007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 254761007b31SStefan Hajnoczi } 25487b1efe99SVladimir Sementsov-Ogievskiy } 254961007b31SStefan Hajnoczi } 255061007b31SStefan Hajnoczi 255169f47505SVladimir Sementsov-Ogievskiy if (want_zero && ret & BDRV_BLOCK_RECURSE && 255269f47505SVladimir Sementsov-Ogievskiy local_file && local_file != bs && 255361007b31SStefan Hajnoczi (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) && 255461007b31SStefan Hajnoczi (ret & BDRV_BLOCK_OFFSET_VALID)) { 25552e8bc787SEric Blake int64_t file_pnum; 25562e8bc787SEric Blake int ret2; 255761007b31SStefan Hajnoczi 25582e8bc787SEric Blake ret2 = bdrv_co_block_status(local_file, want_zero, local_map, 25592e8bc787SEric Blake *pnum, &file_pnum, NULL, NULL); 256061007b31SStefan Hajnoczi if (ret2 >= 0) { 256161007b31SStefan Hajnoczi /* Ignore errors. This is just providing extra information, it 256261007b31SStefan Hajnoczi * is useful but not necessary. 256361007b31SStefan Hajnoczi */ 2564c61e684eSEric Blake if (ret2 & BDRV_BLOCK_EOF && 2565c61e684eSEric Blake (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) { 2566c61e684eSEric Blake /* 2567c61e684eSEric Blake * It is valid for the format block driver to read 2568c61e684eSEric Blake * beyond the end of the underlying file's current 2569c61e684eSEric Blake * size; such areas read as zero. 2570c61e684eSEric Blake */ 257161007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 257261007b31SStefan Hajnoczi } else { 257361007b31SStefan Hajnoczi /* Limit request to the range reported by the protocol driver */ 257461007b31SStefan Hajnoczi *pnum = file_pnum; 257561007b31SStefan Hajnoczi ret |= (ret2 & BDRV_BLOCK_ZERO); 257661007b31SStefan Hajnoczi } 257761007b31SStefan Hajnoczi } 257861007b31SStefan Hajnoczi } 257961007b31SStefan Hajnoczi 258099723548SPaolo Bonzini out: 258199723548SPaolo Bonzini bdrv_dec_in_flight(bs); 25822e8bc787SEric Blake if (ret >= 0 && offset + *pnum == total_size) { 2583fb0d8654SEric Blake ret |= BDRV_BLOCK_EOF; 2584fb0d8654SEric Blake } 2585298a1665SEric Blake early_out: 2586298a1665SEric Blake if (file) { 2587298a1665SEric Blake *file = local_file; 2588298a1665SEric Blake } 25892e8bc787SEric Blake if (map) { 25902e8bc787SEric Blake *map = local_map; 25912e8bc787SEric Blake } 259261007b31SStefan Hajnoczi return ret; 259361007b31SStefan Hajnoczi } 259461007b31SStefan Hajnoczi 259521c2283eSVladimir Sementsov-Ogievskiy int coroutine_fn 2596f9e694cbSVladimir Sementsov-Ogievskiy bdrv_co_common_block_status_above(BlockDriverState *bs, 2597ba3f0e25SFam Zheng BlockDriverState *base, 25983555a432SVladimir Sementsov-Ogievskiy bool include_base, 2599c9ce8c4dSEric Blake bool want_zero, 26005b648c67SEric Blake int64_t offset, 26015b648c67SEric Blake int64_t bytes, 26025b648c67SEric Blake int64_t *pnum, 26035b648c67SEric Blake int64_t *map, 2604a92b1b06SEric Blake BlockDriverState **file, 2605a92b1b06SEric Blake int *depth) 2606ba3f0e25SFam Zheng { 260767c095c8SVladimir Sementsov-Ogievskiy int ret; 2608ba3f0e25SFam Zheng BlockDriverState *p; 260967c095c8SVladimir Sementsov-Ogievskiy int64_t eof = 0; 2610a92b1b06SEric Blake int dummy; 26111581a70dSEmanuele Giuseppe Esposito IO_CODE(); 2612ba3f0e25SFam Zheng 26133555a432SVladimir Sementsov-Ogievskiy assert(!include_base || base); /* Can't include NULL base */ 261467c095c8SVladimir Sementsov-Ogievskiy 2615a92b1b06SEric Blake if (!depth) { 2616a92b1b06SEric Blake depth = &dummy; 2617a92b1b06SEric Blake } 2618a92b1b06SEric Blake *depth = 0; 2619a92b1b06SEric Blake 2620624f27bbSVladimir Sementsov-Ogievskiy if (!include_base && bs == base) { 2621624f27bbSVladimir Sementsov-Ogievskiy *pnum = bytes; 2622624f27bbSVladimir Sementsov-Ogievskiy return 0; 2623624f27bbSVladimir Sementsov-Ogievskiy } 2624624f27bbSVladimir Sementsov-Ogievskiy 262567c095c8SVladimir Sementsov-Ogievskiy ret = bdrv_co_block_status(bs, want_zero, offset, bytes, pnum, map, file); 2626a92b1b06SEric Blake ++*depth; 26273555a432SVladimir Sementsov-Ogievskiy if (ret < 0 || *pnum == 0 || ret & BDRV_BLOCK_ALLOCATED || bs == base) { 262867c095c8SVladimir Sementsov-Ogievskiy return ret; 262967c095c8SVladimir Sementsov-Ogievskiy } 263067c095c8SVladimir Sementsov-Ogievskiy 263167c095c8SVladimir Sementsov-Ogievskiy if (ret & BDRV_BLOCK_EOF) { 263267c095c8SVladimir Sementsov-Ogievskiy eof = offset + *pnum; 263367c095c8SVladimir Sementsov-Ogievskiy } 263467c095c8SVladimir Sementsov-Ogievskiy 263567c095c8SVladimir Sementsov-Ogievskiy assert(*pnum <= bytes); 263667c095c8SVladimir Sementsov-Ogievskiy bytes = *pnum; 263767c095c8SVladimir Sementsov-Ogievskiy 26383555a432SVladimir Sementsov-Ogievskiy for (p = bdrv_filter_or_cow_bs(bs); include_base || p != base; 263967c095c8SVladimir Sementsov-Ogievskiy p = bdrv_filter_or_cow_bs(p)) 264067c095c8SVladimir Sementsov-Ogievskiy { 26415b648c67SEric Blake ret = bdrv_co_block_status(p, want_zero, offset, bytes, pnum, map, 26425b648c67SEric Blake file); 2643a92b1b06SEric Blake ++*depth; 2644c61e684eSEric Blake if (ret < 0) { 264567c095c8SVladimir Sementsov-Ogievskiy return ret; 2646c61e684eSEric Blake } 264767c095c8SVladimir Sementsov-Ogievskiy if (*pnum == 0) { 2648c61e684eSEric Blake /* 264967c095c8SVladimir Sementsov-Ogievskiy * The top layer deferred to this layer, and because this layer is 265067c095c8SVladimir Sementsov-Ogievskiy * short, any zeroes that we synthesize beyond EOF behave as if they 265167c095c8SVladimir Sementsov-Ogievskiy * were allocated at this layer. 265267c095c8SVladimir Sementsov-Ogievskiy * 265367c095c8SVladimir Sementsov-Ogievskiy * We don't include BDRV_BLOCK_EOF into ret, as upper layer may be 265467c095c8SVladimir Sementsov-Ogievskiy * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see 265567c095c8SVladimir Sementsov-Ogievskiy * below. 2656c61e684eSEric Blake */ 265767c095c8SVladimir Sementsov-Ogievskiy assert(ret & BDRV_BLOCK_EOF); 26585b648c67SEric Blake *pnum = bytes; 265967c095c8SVladimir Sementsov-Ogievskiy if (file) { 266067c095c8SVladimir Sementsov-Ogievskiy *file = p; 2661c61e684eSEric Blake } 266267c095c8SVladimir Sementsov-Ogievskiy ret = BDRV_BLOCK_ZERO | BDRV_BLOCK_ALLOCATED; 2663ba3f0e25SFam Zheng break; 2664ba3f0e25SFam Zheng } 266567c095c8SVladimir Sementsov-Ogievskiy if (ret & BDRV_BLOCK_ALLOCATED) { 266667c095c8SVladimir Sementsov-Ogievskiy /* 266767c095c8SVladimir Sementsov-Ogievskiy * We've found the node and the status, we must break. 266867c095c8SVladimir Sementsov-Ogievskiy * 266967c095c8SVladimir Sementsov-Ogievskiy * Drop BDRV_BLOCK_EOF, as it's not for upper layer, which may be 267067c095c8SVladimir Sementsov-Ogievskiy * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see 267167c095c8SVladimir Sementsov-Ogievskiy * below. 267267c095c8SVladimir Sementsov-Ogievskiy */ 267367c095c8SVladimir Sementsov-Ogievskiy ret &= ~BDRV_BLOCK_EOF; 267467c095c8SVladimir Sementsov-Ogievskiy break; 2675ba3f0e25SFam Zheng } 267667c095c8SVladimir Sementsov-Ogievskiy 26773555a432SVladimir Sementsov-Ogievskiy if (p == base) { 26783555a432SVladimir Sementsov-Ogievskiy assert(include_base); 26793555a432SVladimir Sementsov-Ogievskiy break; 26803555a432SVladimir Sementsov-Ogievskiy } 26813555a432SVladimir Sementsov-Ogievskiy 268267c095c8SVladimir Sementsov-Ogievskiy /* 268367c095c8SVladimir Sementsov-Ogievskiy * OK, [offset, offset + *pnum) region is unallocated on this layer, 268467c095c8SVladimir Sementsov-Ogievskiy * let's continue the diving. 268567c095c8SVladimir Sementsov-Ogievskiy */ 268667c095c8SVladimir Sementsov-Ogievskiy assert(*pnum <= bytes); 268767c095c8SVladimir Sementsov-Ogievskiy bytes = *pnum; 268867c095c8SVladimir Sementsov-Ogievskiy } 268967c095c8SVladimir Sementsov-Ogievskiy 269067c095c8SVladimir Sementsov-Ogievskiy if (offset + *pnum == eof) { 269167c095c8SVladimir Sementsov-Ogievskiy ret |= BDRV_BLOCK_EOF; 269267c095c8SVladimir Sementsov-Ogievskiy } 269367c095c8SVladimir Sementsov-Ogievskiy 2694ba3f0e25SFam Zheng return ret; 2695ba3f0e25SFam Zheng } 2696ba3f0e25SFam Zheng 269731826642SEric Blake int bdrv_block_status_above(BlockDriverState *bs, BlockDriverState *base, 269831826642SEric Blake int64_t offset, int64_t bytes, int64_t *pnum, 269931826642SEric Blake int64_t *map, BlockDriverState **file) 2700c9ce8c4dSEric Blake { 2701384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 27023555a432SVladimir Sementsov-Ogievskiy return bdrv_common_block_status_above(bs, base, false, true, offset, bytes, 2703a92b1b06SEric Blake pnum, map, file, NULL); 2704c9ce8c4dSEric Blake } 2705c9ce8c4dSEric Blake 2706237d78f8SEric Blake int bdrv_block_status(BlockDriverState *bs, int64_t offset, int64_t bytes, 2707237d78f8SEric Blake int64_t *pnum, int64_t *map, BlockDriverState **file) 2708ba3f0e25SFam Zheng { 2709384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 2710cb850315SMax Reitz return bdrv_block_status_above(bs, bdrv_filter_or_cow_bs(bs), 271131826642SEric Blake offset, bytes, pnum, map, file); 2712ba3f0e25SFam Zheng } 2713ba3f0e25SFam Zheng 271446cd1e8aSAlberto Garcia /* 271546cd1e8aSAlberto Garcia * Check @bs (and its backing chain) to see if the range defined 271646cd1e8aSAlberto Garcia * by @offset and @bytes is known to read as zeroes. 271746cd1e8aSAlberto Garcia * Return 1 if that is the case, 0 otherwise and -errno on error. 271846cd1e8aSAlberto Garcia * This test is meant to be fast rather than accurate so returning 0 271946cd1e8aSAlberto Garcia * does not guarantee non-zero data. 272046cd1e8aSAlberto Garcia */ 272146cd1e8aSAlberto Garcia int coroutine_fn bdrv_co_is_zero_fast(BlockDriverState *bs, int64_t offset, 272246cd1e8aSAlberto Garcia int64_t bytes) 272346cd1e8aSAlberto Garcia { 272446cd1e8aSAlberto Garcia int ret; 272546cd1e8aSAlberto Garcia int64_t pnum = bytes; 2726384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 272746cd1e8aSAlberto Garcia 272846cd1e8aSAlberto Garcia if (!bytes) { 272946cd1e8aSAlberto Garcia return 1; 273046cd1e8aSAlberto Garcia } 273146cd1e8aSAlberto Garcia 273246cd1e8aSAlberto Garcia ret = bdrv_common_block_status_above(bs, NULL, false, false, offset, 2733a92b1b06SEric Blake bytes, &pnum, NULL, NULL, NULL); 273446cd1e8aSAlberto Garcia 273546cd1e8aSAlberto Garcia if (ret < 0) { 273646cd1e8aSAlberto Garcia return ret; 273746cd1e8aSAlberto Garcia } 273846cd1e8aSAlberto Garcia 273946cd1e8aSAlberto Garcia return (pnum == bytes) && (ret & BDRV_BLOCK_ZERO); 274046cd1e8aSAlberto Garcia } 274146cd1e8aSAlberto Garcia 2742d6a644bbSEric Blake int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t offset, 2743d6a644bbSEric Blake int64_t bytes, int64_t *pnum) 274461007b31SStefan Hajnoczi { 27457ddb99b9SEric Blake int ret; 27467ddb99b9SEric Blake int64_t dummy; 2747384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 2748d6a644bbSEric Blake 27493555a432SVladimir Sementsov-Ogievskiy ret = bdrv_common_block_status_above(bs, bs, true, false, offset, 27503555a432SVladimir Sementsov-Ogievskiy bytes, pnum ? pnum : &dummy, NULL, 2751a92b1b06SEric Blake NULL, NULL); 275261007b31SStefan Hajnoczi if (ret < 0) { 275361007b31SStefan Hajnoczi return ret; 275461007b31SStefan Hajnoczi } 275561007b31SStefan Hajnoczi return !!(ret & BDRV_BLOCK_ALLOCATED); 275661007b31SStefan Hajnoczi } 275761007b31SStefan Hajnoczi 275861007b31SStefan Hajnoczi /* 275961007b31SStefan Hajnoczi * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP] 276061007b31SStefan Hajnoczi * 2761a92b1b06SEric Blake * Return a positive depth if (a prefix of) the given range is allocated 2762a92b1b06SEric Blake * in any image between BASE and TOP (BASE is only included if include_base 2763a92b1b06SEric Blake * is set). Depth 1 is TOP, 2 is the first backing layer, and so forth. 2764170d3bd3SAndrey Shinkevich * BASE can be NULL to check if the given offset is allocated in any 2765170d3bd3SAndrey Shinkevich * image of the chain. Return 0 otherwise, or negative errno on 2766170d3bd3SAndrey Shinkevich * failure. 276761007b31SStefan Hajnoczi * 276851b0a488SEric Blake * 'pnum' is set to the number of bytes (including and immediately 276951b0a488SEric Blake * following the specified offset) that are known to be in the same 277051b0a488SEric Blake * allocated/unallocated state. Note that a subsequent call starting 277151b0a488SEric Blake * at 'offset + *pnum' may return the same allocation status (in other 277251b0a488SEric Blake * words, the result is not necessarily the maximum possible range); 277351b0a488SEric Blake * but 'pnum' will only be 0 when end of file is reached. 277461007b31SStefan Hajnoczi */ 277561007b31SStefan Hajnoczi int bdrv_is_allocated_above(BlockDriverState *top, 277661007b31SStefan Hajnoczi BlockDriverState *base, 2777170d3bd3SAndrey Shinkevich bool include_base, int64_t offset, 2778170d3bd3SAndrey Shinkevich int64_t bytes, int64_t *pnum) 277961007b31SStefan Hajnoczi { 2780a92b1b06SEric Blake int depth; 27817e7e5100SVladimir Sementsov-Ogievskiy int ret = bdrv_common_block_status_above(top, base, include_base, false, 2782a92b1b06SEric Blake offset, bytes, pnum, NULL, NULL, 2783a92b1b06SEric Blake &depth); 2784384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 278561007b31SStefan Hajnoczi if (ret < 0) { 278661007b31SStefan Hajnoczi return ret; 2787d6a644bbSEric Blake } 278861007b31SStefan Hajnoczi 2789a92b1b06SEric Blake if (ret & BDRV_BLOCK_ALLOCATED) { 2790a92b1b06SEric Blake return depth; 2791a92b1b06SEric Blake } 2792a92b1b06SEric Blake return 0; 279361007b31SStefan Hajnoczi } 279461007b31SStefan Hajnoczi 279521c2283eSVladimir Sementsov-Ogievskiy int coroutine_fn 2796b33b354fSVladimir Sementsov-Ogievskiy bdrv_co_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 27971a8ae822SKevin Wolf { 27981a8ae822SKevin Wolf BlockDriver *drv = bs->drv; 2799c4db2e25SMax Reitz BlockDriverState *child_bs = bdrv_primary_bs(bs); 2800b984b296SVladimir Sementsov-Ogievskiy int ret; 28011581a70dSEmanuele Giuseppe Esposito IO_CODE(); 2802b984b296SVladimir Sementsov-Ogievskiy 2803b984b296SVladimir Sementsov-Ogievskiy ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL); 2804b984b296SVladimir Sementsov-Ogievskiy if (ret < 0) { 2805b984b296SVladimir Sementsov-Ogievskiy return ret; 2806b984b296SVladimir Sementsov-Ogievskiy } 2807dc88a467SStefan Hajnoczi 2808b33b354fSVladimir Sementsov-Ogievskiy if (!drv) { 2809b33b354fSVladimir Sementsov-Ogievskiy return -ENOMEDIUM; 2810b33b354fSVladimir Sementsov-Ogievskiy } 2811b33b354fSVladimir Sementsov-Ogievskiy 2812dc88a467SStefan Hajnoczi bdrv_inc_in_flight(bs); 28131a8ae822SKevin Wolf 2814b33b354fSVladimir Sementsov-Ogievskiy if (drv->bdrv_load_vmstate) { 2815dc88a467SStefan Hajnoczi ret = drv->bdrv_load_vmstate(bs, qiov, pos); 2816c4db2e25SMax Reitz } else if (child_bs) { 2817b33b354fSVladimir Sementsov-Ogievskiy ret = bdrv_co_readv_vmstate(child_bs, qiov, pos); 2818b984b296SVladimir Sementsov-Ogievskiy } else { 2819b984b296SVladimir Sementsov-Ogievskiy ret = -ENOTSUP; 28201a8ae822SKevin Wolf } 28211a8ae822SKevin Wolf 2822dc88a467SStefan Hajnoczi bdrv_dec_in_flight(bs); 2823b33b354fSVladimir Sementsov-Ogievskiy 2824b33b354fSVladimir Sementsov-Ogievskiy return ret; 2825b33b354fSVladimir Sementsov-Ogievskiy } 2826b33b354fSVladimir Sementsov-Ogievskiy 2827b33b354fSVladimir Sementsov-Ogievskiy int coroutine_fn 2828b33b354fSVladimir Sementsov-Ogievskiy bdrv_co_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 2829b33b354fSVladimir Sementsov-Ogievskiy { 2830b33b354fSVladimir Sementsov-Ogievskiy BlockDriver *drv = bs->drv; 2831b33b354fSVladimir Sementsov-Ogievskiy BlockDriverState *child_bs = bdrv_primary_bs(bs); 2832b984b296SVladimir Sementsov-Ogievskiy int ret; 28331581a70dSEmanuele Giuseppe Esposito IO_CODE(); 2834b984b296SVladimir Sementsov-Ogievskiy 2835b984b296SVladimir Sementsov-Ogievskiy ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL); 2836b984b296SVladimir Sementsov-Ogievskiy if (ret < 0) { 2837b984b296SVladimir Sementsov-Ogievskiy return ret; 2838b984b296SVladimir Sementsov-Ogievskiy } 2839b33b354fSVladimir Sementsov-Ogievskiy 2840b33b354fSVladimir Sementsov-Ogievskiy if (!drv) { 2841b33b354fSVladimir Sementsov-Ogievskiy return -ENOMEDIUM; 2842b33b354fSVladimir Sementsov-Ogievskiy } 2843b33b354fSVladimir Sementsov-Ogievskiy 2844b33b354fSVladimir Sementsov-Ogievskiy bdrv_inc_in_flight(bs); 2845b33b354fSVladimir Sementsov-Ogievskiy 2846b33b354fSVladimir Sementsov-Ogievskiy if (drv->bdrv_save_vmstate) { 2847b33b354fSVladimir Sementsov-Ogievskiy ret = drv->bdrv_save_vmstate(bs, qiov, pos); 2848b33b354fSVladimir Sementsov-Ogievskiy } else if (child_bs) { 2849b33b354fSVladimir Sementsov-Ogievskiy ret = bdrv_co_writev_vmstate(child_bs, qiov, pos); 2850b984b296SVladimir Sementsov-Ogievskiy } else { 2851b984b296SVladimir Sementsov-Ogievskiy ret = -ENOTSUP; 2852b33b354fSVladimir Sementsov-Ogievskiy } 2853b33b354fSVladimir Sementsov-Ogievskiy 2854b33b354fSVladimir Sementsov-Ogievskiy bdrv_dec_in_flight(bs); 2855b33b354fSVladimir Sementsov-Ogievskiy 2856dc88a467SStefan Hajnoczi return ret; 28571a8ae822SKevin Wolf } 28581a8ae822SKevin Wolf 285961007b31SStefan Hajnoczi int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf, 286061007b31SStefan Hajnoczi int64_t pos, int size) 286161007b31SStefan Hajnoczi { 28620d93ed08SVladimir Sementsov-Ogievskiy QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size); 2863b33b354fSVladimir Sementsov-Ogievskiy int ret = bdrv_writev_vmstate(bs, &qiov, pos); 2864384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 286561007b31SStefan Hajnoczi 2866b33b354fSVladimir Sementsov-Ogievskiy return ret < 0 ? ret : size; 286761007b31SStefan Hajnoczi } 286861007b31SStefan Hajnoczi 286961007b31SStefan Hajnoczi int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf, 287061007b31SStefan Hajnoczi int64_t pos, int size) 287161007b31SStefan Hajnoczi { 28720d93ed08SVladimir Sementsov-Ogievskiy QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size); 2873b33b354fSVladimir Sementsov-Ogievskiy int ret = bdrv_readv_vmstate(bs, &qiov, pos); 2874384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 28755ddda0b8SKevin Wolf 2876b33b354fSVladimir Sementsov-Ogievskiy return ret < 0 ? ret : size; 287761007b31SStefan Hajnoczi } 287861007b31SStefan Hajnoczi 287961007b31SStefan Hajnoczi /**************************************************************/ 288061007b31SStefan Hajnoczi /* async I/Os */ 288161007b31SStefan Hajnoczi 288261007b31SStefan Hajnoczi void bdrv_aio_cancel(BlockAIOCB *acb) 288361007b31SStefan Hajnoczi { 2884384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 288561007b31SStefan Hajnoczi qemu_aio_ref(acb); 288661007b31SStefan Hajnoczi bdrv_aio_cancel_async(acb); 288761007b31SStefan Hajnoczi while (acb->refcnt > 1) { 288861007b31SStefan Hajnoczi if (acb->aiocb_info->get_aio_context) { 288961007b31SStefan Hajnoczi aio_poll(acb->aiocb_info->get_aio_context(acb), true); 289061007b31SStefan Hajnoczi } else if (acb->bs) { 28912f47da5fSPaolo Bonzini /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so 28922f47da5fSPaolo Bonzini * assert that we're not using an I/O thread. Thread-safe 28932f47da5fSPaolo Bonzini * code should use bdrv_aio_cancel_async exclusively. 28942f47da5fSPaolo Bonzini */ 28952f47da5fSPaolo Bonzini assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context()); 289661007b31SStefan Hajnoczi aio_poll(bdrv_get_aio_context(acb->bs), true); 289761007b31SStefan Hajnoczi } else { 289861007b31SStefan Hajnoczi abort(); 289961007b31SStefan Hajnoczi } 290061007b31SStefan Hajnoczi } 290161007b31SStefan Hajnoczi qemu_aio_unref(acb); 290261007b31SStefan Hajnoczi } 290361007b31SStefan Hajnoczi 290461007b31SStefan Hajnoczi /* Async version of aio cancel. The caller is not blocked if the acb implements 290561007b31SStefan Hajnoczi * cancel_async, otherwise we do nothing and let the request normally complete. 290661007b31SStefan Hajnoczi * In either case the completion callback must be called. */ 290761007b31SStefan Hajnoczi void bdrv_aio_cancel_async(BlockAIOCB *acb) 290861007b31SStefan Hajnoczi { 2909384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 291061007b31SStefan Hajnoczi if (acb->aiocb_info->cancel_async) { 291161007b31SStefan Hajnoczi acb->aiocb_info->cancel_async(acb); 291261007b31SStefan Hajnoczi } 291361007b31SStefan Hajnoczi } 291461007b31SStefan Hajnoczi 291561007b31SStefan Hajnoczi /**************************************************************/ 291661007b31SStefan Hajnoczi /* Coroutine block device emulation */ 291761007b31SStefan Hajnoczi 291861007b31SStefan Hajnoczi int coroutine_fn bdrv_co_flush(BlockDriverState *bs) 291961007b31SStefan Hajnoczi { 2920883833e2SMax Reitz BdrvChild *primary_child = bdrv_primary_child(bs); 2921883833e2SMax Reitz BdrvChild *child; 292249ca6259SFam Zheng int current_gen; 292349ca6259SFam Zheng int ret = 0; 2924384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 292561007b31SStefan Hajnoczi 292699723548SPaolo Bonzini bdrv_inc_in_flight(bs); 2927c32b82afSPavel Dovgalyuk 2928e914404eSFam Zheng if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs) || 292949ca6259SFam Zheng bdrv_is_sg(bs)) { 293049ca6259SFam Zheng goto early_exit; 293149ca6259SFam Zheng } 293249ca6259SFam Zheng 29333783fa3dSPaolo Bonzini qemu_co_mutex_lock(&bs->reqs_lock); 2934d73415a3SStefan Hajnoczi current_gen = qatomic_read(&bs->write_gen); 29353ff2f67aSEvgeny Yakovlev 29363ff2f67aSEvgeny Yakovlev /* Wait until any previous flushes are completed */ 293799723548SPaolo Bonzini while (bs->active_flush_req) { 29383783fa3dSPaolo Bonzini qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock); 29393ff2f67aSEvgeny Yakovlev } 29403ff2f67aSEvgeny Yakovlev 29413783fa3dSPaolo Bonzini /* Flushes reach this point in nondecreasing current_gen order. */ 294299723548SPaolo Bonzini bs->active_flush_req = true; 29433783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&bs->reqs_lock); 29443ff2f67aSEvgeny Yakovlev 2945c32b82afSPavel Dovgalyuk /* Write back all layers by calling one driver function */ 2946c32b82afSPavel Dovgalyuk if (bs->drv->bdrv_co_flush) { 2947c32b82afSPavel Dovgalyuk ret = bs->drv->bdrv_co_flush(bs); 2948c32b82afSPavel Dovgalyuk goto out; 2949c32b82afSPavel Dovgalyuk } 2950c32b82afSPavel Dovgalyuk 295161007b31SStefan Hajnoczi /* Write back cached data to the OS even with cache=unsafe */ 2952883833e2SMax Reitz BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_OS); 295361007b31SStefan Hajnoczi if (bs->drv->bdrv_co_flush_to_os) { 295461007b31SStefan Hajnoczi ret = bs->drv->bdrv_co_flush_to_os(bs); 295561007b31SStefan Hajnoczi if (ret < 0) { 2956cdb5e315SFam Zheng goto out; 295761007b31SStefan Hajnoczi } 295861007b31SStefan Hajnoczi } 295961007b31SStefan Hajnoczi 296061007b31SStefan Hajnoczi /* But don't actually force it to the disk with cache=unsafe */ 296161007b31SStefan Hajnoczi if (bs->open_flags & BDRV_O_NO_FLUSH) { 2962883833e2SMax Reitz goto flush_children; 296361007b31SStefan Hajnoczi } 296461007b31SStefan Hajnoczi 29653ff2f67aSEvgeny Yakovlev /* Check if we really need to flush anything */ 29663ff2f67aSEvgeny Yakovlev if (bs->flushed_gen == current_gen) { 2967883833e2SMax Reitz goto flush_children; 29683ff2f67aSEvgeny Yakovlev } 29693ff2f67aSEvgeny Yakovlev 2970883833e2SMax Reitz BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_DISK); 2971d470ad42SMax Reitz if (!bs->drv) { 2972d470ad42SMax Reitz /* bs->drv->bdrv_co_flush() might have ejected the BDS 2973d470ad42SMax Reitz * (even in case of apparent success) */ 2974d470ad42SMax Reitz ret = -ENOMEDIUM; 2975d470ad42SMax Reitz goto out; 2976d470ad42SMax Reitz } 297761007b31SStefan Hajnoczi if (bs->drv->bdrv_co_flush_to_disk) { 297861007b31SStefan Hajnoczi ret = bs->drv->bdrv_co_flush_to_disk(bs); 297961007b31SStefan Hajnoczi } else if (bs->drv->bdrv_aio_flush) { 298061007b31SStefan Hajnoczi BlockAIOCB *acb; 298161007b31SStefan Hajnoczi CoroutineIOCompletion co = { 298261007b31SStefan Hajnoczi .coroutine = qemu_coroutine_self(), 298361007b31SStefan Hajnoczi }; 298461007b31SStefan Hajnoczi 298561007b31SStefan Hajnoczi acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co); 298661007b31SStefan Hajnoczi if (acb == NULL) { 298761007b31SStefan Hajnoczi ret = -EIO; 298861007b31SStefan Hajnoczi } else { 298961007b31SStefan Hajnoczi qemu_coroutine_yield(); 299061007b31SStefan Hajnoczi ret = co.ret; 299161007b31SStefan Hajnoczi } 299261007b31SStefan Hajnoczi } else { 299361007b31SStefan Hajnoczi /* 299461007b31SStefan Hajnoczi * Some block drivers always operate in either writethrough or unsafe 299561007b31SStefan Hajnoczi * mode and don't support bdrv_flush therefore. Usually qemu doesn't 299661007b31SStefan Hajnoczi * know how the server works (because the behaviour is hardcoded or 299761007b31SStefan Hajnoczi * depends on server-side configuration), so we can't ensure that 299861007b31SStefan Hajnoczi * everything is safe on disk. Returning an error doesn't work because 299961007b31SStefan Hajnoczi * that would break guests even if the server operates in writethrough 300061007b31SStefan Hajnoczi * mode. 300161007b31SStefan Hajnoczi * 300261007b31SStefan Hajnoczi * Let's hope the user knows what he's doing. 300361007b31SStefan Hajnoczi */ 300461007b31SStefan Hajnoczi ret = 0; 300561007b31SStefan Hajnoczi } 30063ff2f67aSEvgeny Yakovlev 300761007b31SStefan Hajnoczi if (ret < 0) { 3008cdb5e315SFam Zheng goto out; 300961007b31SStefan Hajnoczi } 301061007b31SStefan Hajnoczi 301161007b31SStefan Hajnoczi /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH 301261007b31SStefan Hajnoczi * in the case of cache=unsafe, so there are no useless flushes. 301361007b31SStefan Hajnoczi */ 3014883833e2SMax Reitz flush_children: 3015883833e2SMax Reitz ret = 0; 3016883833e2SMax Reitz QLIST_FOREACH(child, &bs->children, next) { 3017883833e2SMax Reitz if (child->perm & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED)) { 3018883833e2SMax Reitz int this_child_ret = bdrv_co_flush(child->bs); 3019883833e2SMax Reitz if (!ret) { 3020883833e2SMax Reitz ret = this_child_ret; 3021883833e2SMax Reitz } 3022883833e2SMax Reitz } 3023883833e2SMax Reitz } 3024883833e2SMax Reitz 3025cdb5e315SFam Zheng out: 30263ff2f67aSEvgeny Yakovlev /* Notify any pending flushes that we have completed */ 3027e6af1e08SKevin Wolf if (ret == 0) { 30283ff2f67aSEvgeny Yakovlev bs->flushed_gen = current_gen; 3029e6af1e08SKevin Wolf } 30303783fa3dSPaolo Bonzini 30313783fa3dSPaolo Bonzini qemu_co_mutex_lock(&bs->reqs_lock); 303299723548SPaolo Bonzini bs->active_flush_req = false; 3033156af3acSDenis V. Lunev /* Return value is ignored - it's ok if wait queue is empty */ 3034156af3acSDenis V. Lunev qemu_co_queue_next(&bs->flush_queue); 30353783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&bs->reqs_lock); 30363ff2f67aSEvgeny Yakovlev 303749ca6259SFam Zheng early_exit: 303899723548SPaolo Bonzini bdrv_dec_in_flight(bs); 3039cdb5e315SFam Zheng return ret; 304061007b31SStefan Hajnoczi } 304161007b31SStefan Hajnoczi 3042d93e5726SVladimir Sementsov-Ogievskiy int coroutine_fn bdrv_co_pdiscard(BdrvChild *child, int64_t offset, 3043d93e5726SVladimir Sementsov-Ogievskiy int64_t bytes) 304461007b31SStefan Hajnoczi { 3045b1066c87SFam Zheng BdrvTrackedRequest req; 304639af49c0SVladimir Sementsov-Ogievskiy int ret; 304739af49c0SVladimir Sementsov-Ogievskiy int64_t max_pdiscard; 30483482b9bcSEric Blake int head, tail, align; 30490b9fd3f4SFam Zheng BlockDriverState *bs = child->bs; 3050384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 305161007b31SStefan Hajnoczi 3052d93e5726SVladimir Sementsov-Ogievskiy if (!bs || !bs->drv || !bdrv_is_inserted(bs)) { 305361007b31SStefan Hajnoczi return -ENOMEDIUM; 305461007b31SStefan Hajnoczi } 305561007b31SStefan Hajnoczi 3056d6883bc9SVladimir Sementsov-Ogievskiy if (bdrv_has_readonly_bitmaps(bs)) { 3057d6883bc9SVladimir Sementsov-Ogievskiy return -EPERM; 3058d6883bc9SVladimir Sementsov-Ogievskiy } 3059d6883bc9SVladimir Sementsov-Ogievskiy 306069b55e03SVladimir Sementsov-Ogievskiy ret = bdrv_check_request(offset, bytes, NULL); 30618b117001SVladimir Sementsov-Ogievskiy if (ret < 0) { 30628b117001SVladimir Sementsov-Ogievskiy return ret; 306361007b31SStefan Hajnoczi } 306461007b31SStefan Hajnoczi 306561007b31SStefan Hajnoczi /* Do nothing if disabled. */ 306661007b31SStefan Hajnoczi if (!(bs->open_flags & BDRV_O_UNMAP)) { 306761007b31SStefan Hajnoczi return 0; 306861007b31SStefan Hajnoczi } 306961007b31SStefan Hajnoczi 307002aefe43SEric Blake if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) { 307161007b31SStefan Hajnoczi return 0; 307261007b31SStefan Hajnoczi } 307361007b31SStefan Hajnoczi 30740bc329fbSHanna Reitz /* Invalidate the cached block-status data range if this discard overlaps */ 30750bc329fbSHanna Reitz bdrv_bsc_invalidate_range(bs, offset, bytes); 30760bc329fbSHanna Reitz 30773482b9bcSEric Blake /* Discard is advisory, but some devices track and coalesce 30783482b9bcSEric Blake * unaligned requests, so we must pass everything down rather than 30793482b9bcSEric Blake * round here. Still, most devices will just silently ignore 30803482b9bcSEric Blake * unaligned requests (by returning -ENOTSUP), so we must fragment 30813482b9bcSEric Blake * the request accordingly. */ 308202aefe43SEric Blake align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment); 3083b8d0a980SEric Blake assert(align % bs->bl.request_alignment == 0); 3084b8d0a980SEric Blake head = offset % align; 3085f5a5ca79SManos Pitsidianakis tail = (offset + bytes) % align; 30869f1963b3SEric Blake 308799723548SPaolo Bonzini bdrv_inc_in_flight(bs); 3088f5a5ca79SManos Pitsidianakis tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD); 308950824995SFam Zheng 309000695c27SFam Zheng ret = bdrv_co_write_req_prepare(child, offset, bytes, &req, 0); 3091ec050f77SDenis V. Lunev if (ret < 0) { 3092ec050f77SDenis V. Lunev goto out; 3093ec050f77SDenis V. Lunev } 3094ec050f77SDenis V. Lunev 30956a8f3dbbSVladimir Sementsov-Ogievskiy max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT64_MAX), 30969f1963b3SEric Blake align); 30973482b9bcSEric Blake assert(max_pdiscard >= bs->bl.request_alignment); 30989f1963b3SEric Blake 3099f5a5ca79SManos Pitsidianakis while (bytes > 0) { 3100d93e5726SVladimir Sementsov-Ogievskiy int64_t num = bytes; 31013482b9bcSEric Blake 31023482b9bcSEric Blake if (head) { 31033482b9bcSEric Blake /* Make small requests to get to alignment boundaries. */ 3104f5a5ca79SManos Pitsidianakis num = MIN(bytes, align - head); 31053482b9bcSEric Blake if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) { 31063482b9bcSEric Blake num %= bs->bl.request_alignment; 31073482b9bcSEric Blake } 31083482b9bcSEric Blake head = (head + num) % align; 31093482b9bcSEric Blake assert(num < max_pdiscard); 31103482b9bcSEric Blake } else if (tail) { 31113482b9bcSEric Blake if (num > align) { 31123482b9bcSEric Blake /* Shorten the request to the last aligned cluster. */ 31133482b9bcSEric Blake num -= tail; 31143482b9bcSEric Blake } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) && 31153482b9bcSEric Blake tail > bs->bl.request_alignment) { 31163482b9bcSEric Blake tail %= bs->bl.request_alignment; 31173482b9bcSEric Blake num -= tail; 31183482b9bcSEric Blake } 31193482b9bcSEric Blake } 31203482b9bcSEric Blake /* limit request size */ 31213482b9bcSEric Blake if (num > max_pdiscard) { 31223482b9bcSEric Blake num = max_pdiscard; 31233482b9bcSEric Blake } 312461007b31SStefan Hajnoczi 3125d470ad42SMax Reitz if (!bs->drv) { 3126d470ad42SMax Reitz ret = -ENOMEDIUM; 3127d470ad42SMax Reitz goto out; 3128d470ad42SMax Reitz } 312947a5486dSEric Blake if (bs->drv->bdrv_co_pdiscard) { 313047a5486dSEric Blake ret = bs->drv->bdrv_co_pdiscard(bs, offset, num); 313161007b31SStefan Hajnoczi } else { 313261007b31SStefan Hajnoczi BlockAIOCB *acb; 313361007b31SStefan Hajnoczi CoroutineIOCompletion co = { 313461007b31SStefan Hajnoczi .coroutine = qemu_coroutine_self(), 313561007b31SStefan Hajnoczi }; 313661007b31SStefan Hajnoczi 31374da444a0SEric Blake acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num, 313861007b31SStefan Hajnoczi bdrv_co_io_em_complete, &co); 313961007b31SStefan Hajnoczi if (acb == NULL) { 3140b1066c87SFam Zheng ret = -EIO; 3141b1066c87SFam Zheng goto out; 314261007b31SStefan Hajnoczi } else { 314361007b31SStefan Hajnoczi qemu_coroutine_yield(); 314461007b31SStefan Hajnoczi ret = co.ret; 314561007b31SStefan Hajnoczi } 314661007b31SStefan Hajnoczi } 314761007b31SStefan Hajnoczi if (ret && ret != -ENOTSUP) { 3148b1066c87SFam Zheng goto out; 314961007b31SStefan Hajnoczi } 315061007b31SStefan Hajnoczi 31519f1963b3SEric Blake offset += num; 3152f5a5ca79SManos Pitsidianakis bytes -= num; 315361007b31SStefan Hajnoczi } 3154b1066c87SFam Zheng ret = 0; 3155b1066c87SFam Zheng out: 315600695c27SFam Zheng bdrv_co_write_req_finish(child, req.offset, req.bytes, &req, ret); 3157b1066c87SFam Zheng tracked_request_end(&req); 315899723548SPaolo Bonzini bdrv_dec_in_flight(bs); 3159b1066c87SFam Zheng return ret; 316061007b31SStefan Hajnoczi } 316161007b31SStefan Hajnoczi 3162881a4c55SPaolo Bonzini int coroutine_fn bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf) 316361007b31SStefan Hajnoczi { 316461007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 31655c5ae76aSFam Zheng CoroutineIOCompletion co = { 31665c5ae76aSFam Zheng .coroutine = qemu_coroutine_self(), 31675c5ae76aSFam Zheng }; 31685c5ae76aSFam Zheng BlockAIOCB *acb; 3169384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 317061007b31SStefan Hajnoczi 317199723548SPaolo Bonzini bdrv_inc_in_flight(bs); 317216a389dcSKevin Wolf if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) { 31735c5ae76aSFam Zheng co.ret = -ENOTSUP; 31745c5ae76aSFam Zheng goto out; 31755c5ae76aSFam Zheng } 31765c5ae76aSFam Zheng 317716a389dcSKevin Wolf if (drv->bdrv_co_ioctl) { 317816a389dcSKevin Wolf co.ret = drv->bdrv_co_ioctl(bs, req, buf); 317916a389dcSKevin Wolf } else { 31805c5ae76aSFam Zheng acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co); 31815c5ae76aSFam Zheng if (!acb) { 3182c8a9fd80SFam Zheng co.ret = -ENOTSUP; 3183c8a9fd80SFam Zheng goto out; 31845c5ae76aSFam Zheng } 31855c5ae76aSFam Zheng qemu_coroutine_yield(); 318616a389dcSKevin Wolf } 31875c5ae76aSFam Zheng out: 318899723548SPaolo Bonzini bdrv_dec_in_flight(bs); 31895c5ae76aSFam Zheng return co.ret; 31905c5ae76aSFam Zheng } 31915c5ae76aSFam Zheng 319261007b31SStefan Hajnoczi void *qemu_blockalign(BlockDriverState *bs, size_t size) 319361007b31SStefan Hajnoczi { 3194384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 319561007b31SStefan Hajnoczi return qemu_memalign(bdrv_opt_mem_align(bs), size); 319661007b31SStefan Hajnoczi } 319761007b31SStefan Hajnoczi 319861007b31SStefan Hajnoczi void *qemu_blockalign0(BlockDriverState *bs, size_t size) 319961007b31SStefan Hajnoczi { 3200384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 320161007b31SStefan Hajnoczi return memset(qemu_blockalign(bs, size), 0, size); 320261007b31SStefan Hajnoczi } 320361007b31SStefan Hajnoczi 320461007b31SStefan Hajnoczi void *qemu_try_blockalign(BlockDriverState *bs, size_t size) 320561007b31SStefan Hajnoczi { 320661007b31SStefan Hajnoczi size_t align = bdrv_opt_mem_align(bs); 3207384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 320861007b31SStefan Hajnoczi 320961007b31SStefan Hajnoczi /* Ensure that NULL is never returned on success */ 321061007b31SStefan Hajnoczi assert(align > 0); 321161007b31SStefan Hajnoczi if (size == 0) { 321261007b31SStefan Hajnoczi size = align; 321361007b31SStefan Hajnoczi } 321461007b31SStefan Hajnoczi 321561007b31SStefan Hajnoczi return qemu_try_memalign(align, size); 321661007b31SStefan Hajnoczi } 321761007b31SStefan Hajnoczi 321861007b31SStefan Hajnoczi void *qemu_try_blockalign0(BlockDriverState *bs, size_t size) 321961007b31SStefan Hajnoczi { 322061007b31SStefan Hajnoczi void *mem = qemu_try_blockalign(bs, size); 3221384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 322261007b31SStefan Hajnoczi 322361007b31SStefan Hajnoczi if (mem) { 322461007b31SStefan Hajnoczi memset(mem, 0, size); 322561007b31SStefan Hajnoczi } 322661007b31SStefan Hajnoczi 322761007b31SStefan Hajnoczi return mem; 322861007b31SStefan Hajnoczi } 322961007b31SStefan Hajnoczi 323061007b31SStefan Hajnoczi void bdrv_io_plug(BlockDriverState *bs) 323161007b31SStefan Hajnoczi { 32326b98bd64SPaolo Bonzini BdrvChild *child; 3233384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 32346b98bd64SPaolo Bonzini 32356b98bd64SPaolo Bonzini QLIST_FOREACH(child, &bs->children, next) { 32366b98bd64SPaolo Bonzini bdrv_io_plug(child->bs); 32376b98bd64SPaolo Bonzini } 32386b98bd64SPaolo Bonzini 3239d73415a3SStefan Hajnoczi if (qatomic_fetch_inc(&bs->io_plugged) == 0) { 324061007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 324161007b31SStefan Hajnoczi if (drv && drv->bdrv_io_plug) { 324261007b31SStefan Hajnoczi drv->bdrv_io_plug(bs); 32436b98bd64SPaolo Bonzini } 324461007b31SStefan Hajnoczi } 324561007b31SStefan Hajnoczi } 324661007b31SStefan Hajnoczi 324761007b31SStefan Hajnoczi void bdrv_io_unplug(BlockDriverState *bs) 324861007b31SStefan Hajnoczi { 32496b98bd64SPaolo Bonzini BdrvChild *child; 3250384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 32516b98bd64SPaolo Bonzini 32526b98bd64SPaolo Bonzini assert(bs->io_plugged); 3253d73415a3SStefan Hajnoczi if (qatomic_fetch_dec(&bs->io_plugged) == 1) { 325461007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 325561007b31SStefan Hajnoczi if (drv && drv->bdrv_io_unplug) { 325661007b31SStefan Hajnoczi drv->bdrv_io_unplug(bs); 325761007b31SStefan Hajnoczi } 325861007b31SStefan Hajnoczi } 325961007b31SStefan Hajnoczi 32606b98bd64SPaolo Bonzini QLIST_FOREACH(child, &bs->children, next) { 32616b98bd64SPaolo Bonzini bdrv_io_unplug(child->bs); 32626b98bd64SPaolo Bonzini } 32636b98bd64SPaolo Bonzini } 326423d0ba93SFam Zheng 326523d0ba93SFam Zheng void bdrv_register_buf(BlockDriverState *bs, void *host, size_t size) 326623d0ba93SFam Zheng { 326723d0ba93SFam Zheng BdrvChild *child; 326823d0ba93SFam Zheng 3269f791bf7fSEmanuele Giuseppe Esposito GLOBAL_STATE_CODE(); 327023d0ba93SFam Zheng if (bs->drv && bs->drv->bdrv_register_buf) { 327123d0ba93SFam Zheng bs->drv->bdrv_register_buf(bs, host, size); 327223d0ba93SFam Zheng } 327323d0ba93SFam Zheng QLIST_FOREACH(child, &bs->children, next) { 327423d0ba93SFam Zheng bdrv_register_buf(child->bs, host, size); 327523d0ba93SFam Zheng } 327623d0ba93SFam Zheng } 327723d0ba93SFam Zheng 3278*4f384011SStefan Hajnoczi void bdrv_unregister_buf(BlockDriverState *bs, void *host, size_t size) 327923d0ba93SFam Zheng { 328023d0ba93SFam Zheng BdrvChild *child; 328123d0ba93SFam Zheng 3282f791bf7fSEmanuele Giuseppe Esposito GLOBAL_STATE_CODE(); 328323d0ba93SFam Zheng if (bs->drv && bs->drv->bdrv_unregister_buf) { 3284*4f384011SStefan Hajnoczi bs->drv->bdrv_unregister_buf(bs, host, size); 328523d0ba93SFam Zheng } 328623d0ba93SFam Zheng QLIST_FOREACH(child, &bs->children, next) { 3287*4f384011SStefan Hajnoczi bdrv_unregister_buf(child->bs, host, size); 328823d0ba93SFam Zheng } 328923d0ba93SFam Zheng } 3290fcc67678SFam Zheng 329167b51fb9SVladimir Sementsov-Ogievskiy static int coroutine_fn bdrv_co_copy_range_internal( 3292a5215b8fSVladimir Sementsov-Ogievskiy BdrvChild *src, int64_t src_offset, BdrvChild *dst, 3293a5215b8fSVladimir Sementsov-Ogievskiy int64_t dst_offset, int64_t bytes, 329467b51fb9SVladimir Sementsov-Ogievskiy BdrvRequestFlags read_flags, BdrvRequestFlags write_flags, 3295fcc67678SFam Zheng bool recurse_src) 3296fcc67678SFam Zheng { 3297999658a0SVladimir Sementsov-Ogievskiy BdrvTrackedRequest req; 3298fcc67678SFam Zheng int ret; 3299fcc67678SFam Zheng 3300fe0480d6SKevin Wolf /* TODO We can support BDRV_REQ_NO_FALLBACK here */ 3301fe0480d6SKevin Wolf assert(!(read_flags & BDRV_REQ_NO_FALLBACK)); 3302fe0480d6SKevin Wolf assert(!(write_flags & BDRV_REQ_NO_FALLBACK)); 330345e62b46SVladimir Sementsov-Ogievskiy assert(!(read_flags & BDRV_REQ_NO_WAIT)); 330445e62b46SVladimir Sementsov-Ogievskiy assert(!(write_flags & BDRV_REQ_NO_WAIT)); 3305fe0480d6SKevin Wolf 3306f4dad307SVladimir Sementsov-Ogievskiy if (!dst || !dst->bs || !bdrv_is_inserted(dst->bs)) { 3307fcc67678SFam Zheng return -ENOMEDIUM; 3308fcc67678SFam Zheng } 330963f4ad11SVladimir Sementsov-Ogievskiy ret = bdrv_check_request32(dst_offset, bytes, NULL, 0); 3310fcc67678SFam Zheng if (ret) { 3311fcc67678SFam Zheng return ret; 3312fcc67678SFam Zheng } 331367b51fb9SVladimir Sementsov-Ogievskiy if (write_flags & BDRV_REQ_ZERO_WRITE) { 331467b51fb9SVladimir Sementsov-Ogievskiy return bdrv_co_pwrite_zeroes(dst, dst_offset, bytes, write_flags); 3315fcc67678SFam Zheng } 3316fcc67678SFam Zheng 3317f4dad307SVladimir Sementsov-Ogievskiy if (!src || !src->bs || !bdrv_is_inserted(src->bs)) { 3318d4d3e5a0SFam Zheng return -ENOMEDIUM; 3319d4d3e5a0SFam Zheng } 332063f4ad11SVladimir Sementsov-Ogievskiy ret = bdrv_check_request32(src_offset, bytes, NULL, 0); 3321d4d3e5a0SFam Zheng if (ret) { 3322d4d3e5a0SFam Zheng return ret; 3323d4d3e5a0SFam Zheng } 3324d4d3e5a0SFam Zheng 3325fcc67678SFam Zheng if (!src->bs->drv->bdrv_co_copy_range_from 3326fcc67678SFam Zheng || !dst->bs->drv->bdrv_co_copy_range_to 3327fcc67678SFam Zheng || src->bs->encrypted || dst->bs->encrypted) { 3328fcc67678SFam Zheng return -ENOTSUP; 3329fcc67678SFam Zheng } 3330999658a0SVladimir Sementsov-Ogievskiy 3331999658a0SVladimir Sementsov-Ogievskiy if (recurse_src) { 3332d4d3e5a0SFam Zheng bdrv_inc_in_flight(src->bs); 3333999658a0SVladimir Sementsov-Ogievskiy tracked_request_begin(&req, src->bs, src_offset, bytes, 3334999658a0SVladimir Sementsov-Ogievskiy BDRV_TRACKED_READ); 333537aec7d7SFam Zheng 333609d2f948SVladimir Sementsov-Ogievskiy /* BDRV_REQ_SERIALISING is only for write operation */ 333709d2f948SVladimir Sementsov-Ogievskiy assert(!(read_flags & BDRV_REQ_SERIALISING)); 3338304d9d7fSMax Reitz bdrv_wait_serialising_requests(&req); 3339999658a0SVladimir Sementsov-Ogievskiy 334037aec7d7SFam Zheng ret = src->bs->drv->bdrv_co_copy_range_from(src->bs, 3341fcc67678SFam Zheng src, src_offset, 3342fcc67678SFam Zheng dst, dst_offset, 334367b51fb9SVladimir Sementsov-Ogievskiy bytes, 334467b51fb9SVladimir Sementsov-Ogievskiy read_flags, write_flags); 3345999658a0SVladimir Sementsov-Ogievskiy 3346999658a0SVladimir Sementsov-Ogievskiy tracked_request_end(&req); 3347999658a0SVladimir Sementsov-Ogievskiy bdrv_dec_in_flight(src->bs); 3348fcc67678SFam Zheng } else { 3349999658a0SVladimir Sementsov-Ogievskiy bdrv_inc_in_flight(dst->bs); 3350999658a0SVladimir Sementsov-Ogievskiy tracked_request_begin(&req, dst->bs, dst_offset, bytes, 3351999658a0SVladimir Sementsov-Ogievskiy BDRV_TRACKED_WRITE); 33520eb1e891SFam Zheng ret = bdrv_co_write_req_prepare(dst, dst_offset, bytes, &req, 33530eb1e891SFam Zheng write_flags); 33540eb1e891SFam Zheng if (!ret) { 335537aec7d7SFam Zheng ret = dst->bs->drv->bdrv_co_copy_range_to(dst->bs, 3356fcc67678SFam Zheng src, src_offset, 3357fcc67678SFam Zheng dst, dst_offset, 335867b51fb9SVladimir Sementsov-Ogievskiy bytes, 335967b51fb9SVladimir Sementsov-Ogievskiy read_flags, write_flags); 33600eb1e891SFam Zheng } 33610eb1e891SFam Zheng bdrv_co_write_req_finish(dst, dst_offset, bytes, &req, ret); 3362999658a0SVladimir Sementsov-Ogievskiy tracked_request_end(&req); 3363d4d3e5a0SFam Zheng bdrv_dec_in_flight(dst->bs); 3364999658a0SVladimir Sementsov-Ogievskiy } 3365999658a0SVladimir Sementsov-Ogievskiy 336637aec7d7SFam Zheng return ret; 3367fcc67678SFam Zheng } 3368fcc67678SFam Zheng 3369fcc67678SFam Zheng /* Copy range from @src to @dst. 3370fcc67678SFam Zheng * 3371fcc67678SFam Zheng * See the comment of bdrv_co_copy_range for the parameter and return value 3372fcc67678SFam Zheng * semantics. */ 3373a5215b8fSVladimir Sementsov-Ogievskiy int coroutine_fn bdrv_co_copy_range_from(BdrvChild *src, int64_t src_offset, 3374a5215b8fSVladimir Sementsov-Ogievskiy BdrvChild *dst, int64_t dst_offset, 3375a5215b8fSVladimir Sementsov-Ogievskiy int64_t bytes, 337667b51fb9SVladimir Sementsov-Ogievskiy BdrvRequestFlags read_flags, 337767b51fb9SVladimir Sementsov-Ogievskiy BdrvRequestFlags write_flags) 3378fcc67678SFam Zheng { 3379967d7905SEmanuele Giuseppe Esposito IO_CODE(); 3380ecc983a5SFam Zheng trace_bdrv_co_copy_range_from(src, src_offset, dst, dst_offset, bytes, 3381ecc983a5SFam Zheng read_flags, write_flags); 3382fcc67678SFam Zheng return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset, 338367b51fb9SVladimir Sementsov-Ogievskiy bytes, read_flags, write_flags, true); 3384fcc67678SFam Zheng } 3385fcc67678SFam Zheng 3386fcc67678SFam Zheng /* Copy range from @src to @dst. 3387fcc67678SFam Zheng * 3388fcc67678SFam Zheng * See the comment of bdrv_co_copy_range for the parameter and return value 3389fcc67678SFam Zheng * semantics. */ 3390a5215b8fSVladimir Sementsov-Ogievskiy int coroutine_fn bdrv_co_copy_range_to(BdrvChild *src, int64_t src_offset, 3391a5215b8fSVladimir Sementsov-Ogievskiy BdrvChild *dst, int64_t dst_offset, 3392a5215b8fSVladimir Sementsov-Ogievskiy int64_t bytes, 339367b51fb9SVladimir Sementsov-Ogievskiy BdrvRequestFlags read_flags, 339467b51fb9SVladimir Sementsov-Ogievskiy BdrvRequestFlags write_flags) 3395fcc67678SFam Zheng { 3396967d7905SEmanuele Giuseppe Esposito IO_CODE(); 3397ecc983a5SFam Zheng trace_bdrv_co_copy_range_to(src, src_offset, dst, dst_offset, bytes, 3398ecc983a5SFam Zheng read_flags, write_flags); 3399fcc67678SFam Zheng return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset, 340067b51fb9SVladimir Sementsov-Ogievskiy bytes, read_flags, write_flags, false); 3401fcc67678SFam Zheng } 3402fcc67678SFam Zheng 3403a5215b8fSVladimir Sementsov-Ogievskiy int coroutine_fn bdrv_co_copy_range(BdrvChild *src, int64_t src_offset, 3404a5215b8fSVladimir Sementsov-Ogievskiy BdrvChild *dst, int64_t dst_offset, 3405a5215b8fSVladimir Sementsov-Ogievskiy int64_t bytes, BdrvRequestFlags read_flags, 340667b51fb9SVladimir Sementsov-Ogievskiy BdrvRequestFlags write_flags) 3407fcc67678SFam Zheng { 3408384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 340937aec7d7SFam Zheng return bdrv_co_copy_range_from(src, src_offset, 3410fcc67678SFam Zheng dst, dst_offset, 341167b51fb9SVladimir Sementsov-Ogievskiy bytes, read_flags, write_flags); 3412fcc67678SFam Zheng } 34133d9f2d2aSKevin Wolf 34143d9f2d2aSKevin Wolf static void bdrv_parent_cb_resize(BlockDriverState *bs) 34153d9f2d2aSKevin Wolf { 34163d9f2d2aSKevin Wolf BdrvChild *c; 34173d9f2d2aSKevin Wolf QLIST_FOREACH(c, &bs->parents, next_parent) { 3418bd86fb99SMax Reitz if (c->klass->resize) { 3419bd86fb99SMax Reitz c->klass->resize(c); 34203d9f2d2aSKevin Wolf } 34213d9f2d2aSKevin Wolf } 34223d9f2d2aSKevin Wolf } 34233d9f2d2aSKevin Wolf 34243d9f2d2aSKevin Wolf /** 34253d9f2d2aSKevin Wolf * Truncate file to 'offset' bytes (needed only for file protocols) 3426c80d8b06SMax Reitz * 3427c80d8b06SMax Reitz * If 'exact' is true, the file must be resized to exactly the given 3428c80d8b06SMax Reitz * 'offset'. Otherwise, it is sufficient for the node to be at least 3429c80d8b06SMax Reitz * 'offset' bytes in length. 34303d9f2d2aSKevin Wolf */ 3431c80d8b06SMax Reitz int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset, bool exact, 34327b8e4857SKevin Wolf PreallocMode prealloc, BdrvRequestFlags flags, 34337b8e4857SKevin Wolf Error **errp) 34343d9f2d2aSKevin Wolf { 34353d9f2d2aSKevin Wolf BlockDriverState *bs = child->bs; 343623b93525SMax Reitz BdrvChild *filtered, *backing; 34373d9f2d2aSKevin Wolf BlockDriver *drv = bs->drv; 34381bc5f09fSKevin Wolf BdrvTrackedRequest req; 34391bc5f09fSKevin Wolf int64_t old_size, new_bytes; 34403d9f2d2aSKevin Wolf int ret; 3441384a48fbSEmanuele Giuseppe Esposito IO_CODE(); 34423d9f2d2aSKevin Wolf 34433d9f2d2aSKevin Wolf /* if bs->drv == NULL, bs is closed, so there's nothing to do here */ 34443d9f2d2aSKevin Wolf if (!drv) { 34453d9f2d2aSKevin Wolf error_setg(errp, "No medium inserted"); 34463d9f2d2aSKevin Wolf return -ENOMEDIUM; 34473d9f2d2aSKevin Wolf } 34483d9f2d2aSKevin Wolf if (offset < 0) { 34493d9f2d2aSKevin Wolf error_setg(errp, "Image size cannot be negative"); 34503d9f2d2aSKevin Wolf return -EINVAL; 34513d9f2d2aSKevin Wolf } 34523d9f2d2aSKevin Wolf 345369b55e03SVladimir Sementsov-Ogievskiy ret = bdrv_check_request(offset, 0, errp); 34548b117001SVladimir Sementsov-Ogievskiy if (ret < 0) { 34558b117001SVladimir Sementsov-Ogievskiy return ret; 34568b117001SVladimir Sementsov-Ogievskiy } 34578b117001SVladimir Sementsov-Ogievskiy 34581bc5f09fSKevin Wolf old_size = bdrv_getlength(bs); 34591bc5f09fSKevin Wolf if (old_size < 0) { 34601bc5f09fSKevin Wolf error_setg_errno(errp, -old_size, "Failed to get old image size"); 34611bc5f09fSKevin Wolf return old_size; 34621bc5f09fSKevin Wolf } 34631bc5f09fSKevin Wolf 346497efa869SEric Blake if (bdrv_is_read_only(bs)) { 346597efa869SEric Blake error_setg(errp, "Image is read-only"); 346697efa869SEric Blake return -EACCES; 346797efa869SEric Blake } 346897efa869SEric Blake 34691bc5f09fSKevin Wolf if (offset > old_size) { 34701bc5f09fSKevin Wolf new_bytes = offset - old_size; 34711bc5f09fSKevin Wolf } else { 34721bc5f09fSKevin Wolf new_bytes = 0; 34731bc5f09fSKevin Wolf } 34741bc5f09fSKevin Wolf 34753d9f2d2aSKevin Wolf bdrv_inc_in_flight(bs); 34765416a11eSFam Zheng tracked_request_begin(&req, bs, offset - new_bytes, new_bytes, 34775416a11eSFam Zheng BDRV_TRACKED_TRUNCATE); 34781bc5f09fSKevin Wolf 34791bc5f09fSKevin Wolf /* If we are growing the image and potentially using preallocation for the 34801bc5f09fSKevin Wolf * new area, we need to make sure that no write requests are made to it 34811bc5f09fSKevin Wolf * concurrently or they might be overwritten by preallocation. */ 34821bc5f09fSKevin Wolf if (new_bytes) { 34838ac5aab2SVladimir Sementsov-Ogievskiy bdrv_make_request_serialising(&req, 1); 3484cd47d792SFam Zheng } 3485cd47d792SFam Zheng ret = bdrv_co_write_req_prepare(child, offset - new_bytes, new_bytes, &req, 3486cd47d792SFam Zheng 0); 3487cd47d792SFam Zheng if (ret < 0) { 3488cd47d792SFam Zheng error_setg_errno(errp, -ret, 3489cd47d792SFam Zheng "Failed to prepare request for truncation"); 3490cd47d792SFam Zheng goto out; 34911bc5f09fSKevin Wolf } 34923d9f2d2aSKevin Wolf 349393393e69SMax Reitz filtered = bdrv_filter_child(bs); 349423b93525SMax Reitz backing = bdrv_cow_child(bs); 349593393e69SMax Reitz 3496955c7d66SKevin Wolf /* 3497955c7d66SKevin Wolf * If the image has a backing file that is large enough that it would 3498955c7d66SKevin Wolf * provide data for the new area, we cannot leave it unallocated because 3499955c7d66SKevin Wolf * then the backing file content would become visible. Instead, zero-fill 3500955c7d66SKevin Wolf * the new area. 3501955c7d66SKevin Wolf * 3502955c7d66SKevin Wolf * Note that if the image has a backing file, but was opened without the 3503955c7d66SKevin Wolf * backing file, taking care of keeping things consistent with that backing 3504955c7d66SKevin Wolf * file is the user's responsibility. 3505955c7d66SKevin Wolf */ 350623b93525SMax Reitz if (new_bytes && backing) { 3507955c7d66SKevin Wolf int64_t backing_len; 3508955c7d66SKevin Wolf 350923b93525SMax Reitz backing_len = bdrv_getlength(backing->bs); 3510955c7d66SKevin Wolf if (backing_len < 0) { 3511955c7d66SKevin Wolf ret = backing_len; 3512955c7d66SKevin Wolf error_setg_errno(errp, -ret, "Could not get backing file size"); 3513955c7d66SKevin Wolf goto out; 3514955c7d66SKevin Wolf } 3515955c7d66SKevin Wolf 3516955c7d66SKevin Wolf if (backing_len > old_size) { 3517955c7d66SKevin Wolf flags |= BDRV_REQ_ZERO_WRITE; 3518955c7d66SKevin Wolf } 3519955c7d66SKevin Wolf } 3520955c7d66SKevin Wolf 35216b7e8f8bSMax Reitz if (drv->bdrv_co_truncate) { 352292b92799SKevin Wolf if (flags & ~bs->supported_truncate_flags) { 352392b92799SKevin Wolf error_setg(errp, "Block driver does not support requested flags"); 352492b92799SKevin Wolf ret = -ENOTSUP; 352592b92799SKevin Wolf goto out; 352692b92799SKevin Wolf } 352792b92799SKevin Wolf ret = drv->bdrv_co_truncate(bs, offset, exact, prealloc, flags, errp); 352893393e69SMax Reitz } else if (filtered) { 352993393e69SMax Reitz ret = bdrv_co_truncate(filtered, offset, exact, prealloc, flags, errp); 35306b7e8f8bSMax Reitz } else { 35313d9f2d2aSKevin Wolf error_setg(errp, "Image format driver does not support resize"); 35323d9f2d2aSKevin Wolf ret = -ENOTSUP; 35333d9f2d2aSKevin Wolf goto out; 35343d9f2d2aSKevin Wolf } 35353d9f2d2aSKevin Wolf if (ret < 0) { 35363d9f2d2aSKevin Wolf goto out; 35373d9f2d2aSKevin Wolf } 35386b7e8f8bSMax Reitz 35393d9f2d2aSKevin Wolf ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS); 35403d9f2d2aSKevin Wolf if (ret < 0) { 35413d9f2d2aSKevin Wolf error_setg_errno(errp, -ret, "Could not refresh total sector count"); 35423d9f2d2aSKevin Wolf } else { 35433d9f2d2aSKevin Wolf offset = bs->total_sectors * BDRV_SECTOR_SIZE; 35443d9f2d2aSKevin Wolf } 3545cd47d792SFam Zheng /* It's possible that truncation succeeded but refresh_total_sectors 3546cd47d792SFam Zheng * failed, but the latter doesn't affect how we should finish the request. 3547cd47d792SFam Zheng * Pass 0 as the last parameter so that dirty bitmaps etc. are handled. */ 3548cd47d792SFam Zheng bdrv_co_write_req_finish(child, offset - new_bytes, new_bytes, &req, 0); 35493d9f2d2aSKevin Wolf 35503d9f2d2aSKevin Wolf out: 35511bc5f09fSKevin Wolf tracked_request_end(&req); 35523d9f2d2aSKevin Wolf bdrv_dec_in_flight(bs); 35531bc5f09fSKevin Wolf 35543d9f2d2aSKevin Wolf return ret; 35553d9f2d2aSKevin Wolf } 3556bd54669aSVladimir Sementsov-Ogievskiy 3557bd54669aSVladimir Sementsov-Ogievskiy void bdrv_cancel_in_flight(BlockDriverState *bs) 3558bd54669aSVladimir Sementsov-Ogievskiy { 3559f791bf7fSEmanuele Giuseppe Esposito GLOBAL_STATE_CODE(); 3560bd54669aSVladimir Sementsov-Ogievskiy if (!bs || !bs->drv) { 3561bd54669aSVladimir Sementsov-Ogievskiy return; 3562bd54669aSVladimir Sementsov-Ogievskiy } 3563bd54669aSVladimir Sementsov-Ogievskiy 3564bd54669aSVladimir Sementsov-Ogievskiy if (bs->drv->bdrv_cancel_in_flight) { 3565bd54669aSVladimir Sementsov-Ogievskiy bs->drv->bdrv_cancel_in_flight(bs); 3566bd54669aSVladimir Sementsov-Ogievskiy } 3567bd54669aSVladimir Sementsov-Ogievskiy } 3568ce14f3b4SVladimir Sementsov-Ogievskiy 3569ce14f3b4SVladimir Sementsov-Ogievskiy int coroutine_fn 3570ce14f3b4SVladimir Sementsov-Ogievskiy bdrv_co_preadv_snapshot(BdrvChild *child, int64_t offset, int64_t bytes, 3571ce14f3b4SVladimir Sementsov-Ogievskiy QEMUIOVector *qiov, size_t qiov_offset) 3572ce14f3b4SVladimir Sementsov-Ogievskiy { 3573ce14f3b4SVladimir Sementsov-Ogievskiy BlockDriverState *bs = child->bs; 3574ce14f3b4SVladimir Sementsov-Ogievskiy BlockDriver *drv = bs->drv; 3575ce14f3b4SVladimir Sementsov-Ogievskiy int ret; 3576ce14f3b4SVladimir Sementsov-Ogievskiy IO_CODE(); 3577ce14f3b4SVladimir Sementsov-Ogievskiy 3578ce14f3b4SVladimir Sementsov-Ogievskiy if (!drv) { 3579ce14f3b4SVladimir Sementsov-Ogievskiy return -ENOMEDIUM; 3580ce14f3b4SVladimir Sementsov-Ogievskiy } 3581ce14f3b4SVladimir Sementsov-Ogievskiy 3582ce14f3b4SVladimir Sementsov-Ogievskiy if (!drv->bdrv_co_preadv_snapshot) { 3583ce14f3b4SVladimir Sementsov-Ogievskiy return -ENOTSUP; 3584ce14f3b4SVladimir Sementsov-Ogievskiy } 3585ce14f3b4SVladimir Sementsov-Ogievskiy 3586ce14f3b4SVladimir Sementsov-Ogievskiy bdrv_inc_in_flight(bs); 3587ce14f3b4SVladimir Sementsov-Ogievskiy ret = drv->bdrv_co_preadv_snapshot(bs, offset, bytes, qiov, qiov_offset); 3588ce14f3b4SVladimir Sementsov-Ogievskiy bdrv_dec_in_flight(bs); 3589ce14f3b4SVladimir Sementsov-Ogievskiy 3590ce14f3b4SVladimir Sementsov-Ogievskiy return ret; 3591ce14f3b4SVladimir Sementsov-Ogievskiy } 3592ce14f3b4SVladimir Sementsov-Ogievskiy 3593ce14f3b4SVladimir Sementsov-Ogievskiy int coroutine_fn 3594ce14f3b4SVladimir Sementsov-Ogievskiy bdrv_co_snapshot_block_status(BlockDriverState *bs, 3595ce14f3b4SVladimir Sementsov-Ogievskiy bool want_zero, int64_t offset, int64_t bytes, 3596ce14f3b4SVladimir Sementsov-Ogievskiy int64_t *pnum, int64_t *map, 3597ce14f3b4SVladimir Sementsov-Ogievskiy BlockDriverState **file) 3598ce14f3b4SVladimir Sementsov-Ogievskiy { 3599ce14f3b4SVladimir Sementsov-Ogievskiy BlockDriver *drv = bs->drv; 3600ce14f3b4SVladimir Sementsov-Ogievskiy int ret; 3601ce14f3b4SVladimir Sementsov-Ogievskiy IO_CODE(); 3602ce14f3b4SVladimir Sementsov-Ogievskiy 3603ce14f3b4SVladimir Sementsov-Ogievskiy if (!drv) { 3604ce14f3b4SVladimir Sementsov-Ogievskiy return -ENOMEDIUM; 3605ce14f3b4SVladimir Sementsov-Ogievskiy } 3606ce14f3b4SVladimir Sementsov-Ogievskiy 3607ce14f3b4SVladimir Sementsov-Ogievskiy if (!drv->bdrv_co_snapshot_block_status) { 3608ce14f3b4SVladimir Sementsov-Ogievskiy return -ENOTSUP; 3609ce14f3b4SVladimir Sementsov-Ogievskiy } 3610ce14f3b4SVladimir Sementsov-Ogievskiy 3611ce14f3b4SVladimir Sementsov-Ogievskiy bdrv_inc_in_flight(bs); 3612ce14f3b4SVladimir Sementsov-Ogievskiy ret = drv->bdrv_co_snapshot_block_status(bs, want_zero, offset, bytes, 3613ce14f3b4SVladimir Sementsov-Ogievskiy pnum, map, file); 3614ce14f3b4SVladimir Sementsov-Ogievskiy bdrv_dec_in_flight(bs); 3615ce14f3b4SVladimir Sementsov-Ogievskiy 3616ce14f3b4SVladimir Sementsov-Ogievskiy return ret; 3617ce14f3b4SVladimir Sementsov-Ogievskiy } 3618ce14f3b4SVladimir Sementsov-Ogievskiy 3619ce14f3b4SVladimir Sementsov-Ogievskiy int coroutine_fn 3620ce14f3b4SVladimir Sementsov-Ogievskiy bdrv_co_pdiscard_snapshot(BlockDriverState *bs, int64_t offset, int64_t bytes) 3621ce14f3b4SVladimir Sementsov-Ogievskiy { 3622ce14f3b4SVladimir Sementsov-Ogievskiy BlockDriver *drv = bs->drv; 3623ce14f3b4SVladimir Sementsov-Ogievskiy int ret; 3624ce14f3b4SVladimir Sementsov-Ogievskiy IO_CODE(); 3625ce14f3b4SVladimir Sementsov-Ogievskiy 3626ce14f3b4SVladimir Sementsov-Ogievskiy if (!drv) { 3627ce14f3b4SVladimir Sementsov-Ogievskiy return -ENOMEDIUM; 3628ce14f3b4SVladimir Sementsov-Ogievskiy } 3629ce14f3b4SVladimir Sementsov-Ogievskiy 3630ce14f3b4SVladimir Sementsov-Ogievskiy if (!drv->bdrv_co_pdiscard_snapshot) { 3631ce14f3b4SVladimir Sementsov-Ogievskiy return -ENOTSUP; 3632ce14f3b4SVladimir Sementsov-Ogievskiy } 3633ce14f3b4SVladimir Sementsov-Ogievskiy 3634ce14f3b4SVladimir Sementsov-Ogievskiy bdrv_inc_in_flight(bs); 3635ce14f3b4SVladimir Sementsov-Ogievskiy ret = drv->bdrv_co_pdiscard_snapshot(bs, offset, bytes); 3636ce14f3b4SVladimir Sementsov-Ogievskiy bdrv_dec_in_flight(bs); 3637ce14f3b4SVladimir Sementsov-Ogievskiy 3638ce14f3b4SVladimir Sementsov-Ogievskiy return ret; 3639ce14f3b4SVladimir Sementsov-Ogievskiy } 3640