161007b31SStefan Hajnoczi /* 261007b31SStefan Hajnoczi * Block layer I/O functions 361007b31SStefan Hajnoczi * 461007b31SStefan Hajnoczi * Copyright (c) 2003 Fabrice Bellard 561007b31SStefan Hajnoczi * 661007b31SStefan Hajnoczi * Permission is hereby granted, free of charge, to any person obtaining a copy 761007b31SStefan Hajnoczi * of this software and associated documentation files (the "Software"), to deal 861007b31SStefan Hajnoczi * in the Software without restriction, including without limitation the rights 961007b31SStefan Hajnoczi * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 1061007b31SStefan Hajnoczi * copies of the Software, and to permit persons to whom the Software is 1161007b31SStefan Hajnoczi * furnished to do so, subject to the following conditions: 1261007b31SStefan Hajnoczi * 1361007b31SStefan Hajnoczi * The above copyright notice and this permission notice shall be included in 1461007b31SStefan Hajnoczi * all copies or substantial portions of the Software. 1561007b31SStefan Hajnoczi * 1661007b31SStefan Hajnoczi * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 1761007b31SStefan Hajnoczi * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 1861007b31SStefan Hajnoczi * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 1961007b31SStefan Hajnoczi * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 2061007b31SStefan Hajnoczi * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 2161007b31SStefan Hajnoczi * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 2261007b31SStefan Hajnoczi * THE SOFTWARE. 2361007b31SStefan Hajnoczi */ 2461007b31SStefan Hajnoczi 2580c71a24SPeter Maydell #include "qemu/osdep.h" 2661007b31SStefan Hajnoczi #include "trace.h" 277f0e9da6SMax Reitz #include "sysemu/block-backend.h" 287719f3c9SStefan Hajnoczi #include "block/aio-wait.h" 2961007b31SStefan Hajnoczi #include "block/blockjob.h" 30f321dcb5SPaolo Bonzini #include "block/blockjob_int.h" 3161007b31SStefan Hajnoczi #include "block/block_int.h" 3221c2283eSVladimir Sementsov-Ogievskiy #include "block/coroutines.h" 33f348b6d1SVeronia Bahaa #include "qemu/cutils.h" 34da34e65cSMarkus Armbruster #include "qapi/error.h" 35d49b6836SMarkus Armbruster #include "qemu/error-report.h" 36db725815SMarkus Armbruster #include "qemu/main-loop.h" 37c8aa7895SPavel Dovgalyuk #include "sysemu/replay.h" 3861007b31SStefan Hajnoczi 39cb2e2878SEric Blake /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */ 40cb2e2878SEric Blake #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS) 41cb2e2878SEric Blake 427f8f03efSFam Zheng static void bdrv_parent_cb_resize(BlockDriverState *bs); 43d05aa8bbSEric Blake static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 44f5a5ca79SManos Pitsidianakis int64_t offset, int bytes, BdrvRequestFlags flags); 4561007b31SStefan Hajnoczi 46f4c8a43bSMax Reitz static void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore, 476cd5c9d7SKevin Wolf bool ignore_bds_parents) 4861007b31SStefan Hajnoczi { 4902d21300SKevin Wolf BdrvChild *c, *next; 5027ccdd52SKevin Wolf 5102d21300SKevin Wolf QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) { 52bd86fb99SMax Reitz if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) { 530152bf40SKevin Wolf continue; 540152bf40SKevin Wolf } 554be6a6d1SKevin Wolf bdrv_parent_drained_begin_single(c, false); 56ce0f1412SPaolo Bonzini } 57ce0f1412SPaolo Bonzini } 58ce0f1412SPaolo Bonzini 59e037c09cSMax Reitz static void bdrv_parent_drained_end_single_no_poll(BdrvChild *c, 60e037c09cSMax Reitz int *drained_end_counter) 61804db8eaSMax Reitz { 62804db8eaSMax Reitz assert(c->parent_quiesce_counter > 0); 63804db8eaSMax Reitz c->parent_quiesce_counter--; 64bd86fb99SMax Reitz if (c->klass->drained_end) { 65bd86fb99SMax Reitz c->klass->drained_end(c, drained_end_counter); 66804db8eaSMax Reitz } 67804db8eaSMax Reitz } 68804db8eaSMax Reitz 69e037c09cSMax Reitz void bdrv_parent_drained_end_single(BdrvChild *c) 70e037c09cSMax Reitz { 71e037c09cSMax Reitz int drained_end_counter = 0; 72e037c09cSMax Reitz bdrv_parent_drained_end_single_no_poll(c, &drained_end_counter); 73d73415a3SStefan Hajnoczi BDRV_POLL_WHILE(c->bs, qatomic_read(&drained_end_counter) > 0); 74e037c09cSMax Reitz } 75e037c09cSMax Reitz 76f4c8a43bSMax Reitz static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore, 77e037c09cSMax Reitz bool ignore_bds_parents, 78e037c09cSMax Reitz int *drained_end_counter) 79ce0f1412SPaolo Bonzini { 8061ad631cSMax Reitz BdrvChild *c; 8127ccdd52SKevin Wolf 8261ad631cSMax Reitz QLIST_FOREACH(c, &bs->parents, next_parent) { 83bd86fb99SMax Reitz if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) { 840152bf40SKevin Wolf continue; 850152bf40SKevin Wolf } 86e037c09cSMax Reitz bdrv_parent_drained_end_single_no_poll(c, drained_end_counter); 87c2066af0SKevin Wolf } 8861007b31SStefan Hajnoczi } 8961007b31SStefan Hajnoczi 904be6a6d1SKevin Wolf static bool bdrv_parent_drained_poll_single(BdrvChild *c) 914be6a6d1SKevin Wolf { 92bd86fb99SMax Reitz if (c->klass->drained_poll) { 93bd86fb99SMax Reitz return c->klass->drained_poll(c); 944be6a6d1SKevin Wolf } 954be6a6d1SKevin Wolf return false; 964be6a6d1SKevin Wolf } 974be6a6d1SKevin Wolf 986cd5c9d7SKevin Wolf static bool bdrv_parent_drained_poll(BlockDriverState *bs, BdrvChild *ignore, 996cd5c9d7SKevin Wolf bool ignore_bds_parents) 10089bd0305SKevin Wolf { 10189bd0305SKevin Wolf BdrvChild *c, *next; 10289bd0305SKevin Wolf bool busy = false; 10389bd0305SKevin Wolf 10489bd0305SKevin Wolf QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) { 105bd86fb99SMax Reitz if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) { 10689bd0305SKevin Wolf continue; 10789bd0305SKevin Wolf } 1084be6a6d1SKevin Wolf busy |= bdrv_parent_drained_poll_single(c); 10989bd0305SKevin Wolf } 11089bd0305SKevin Wolf 11189bd0305SKevin Wolf return busy; 11289bd0305SKevin Wolf } 11389bd0305SKevin Wolf 1144be6a6d1SKevin Wolf void bdrv_parent_drained_begin_single(BdrvChild *c, bool poll) 1154be6a6d1SKevin Wolf { 116804db8eaSMax Reitz c->parent_quiesce_counter++; 117bd86fb99SMax Reitz if (c->klass->drained_begin) { 118bd86fb99SMax Reitz c->klass->drained_begin(c); 1194be6a6d1SKevin Wolf } 1204be6a6d1SKevin Wolf if (poll) { 1214be6a6d1SKevin Wolf BDRV_POLL_WHILE(c->bs, bdrv_parent_drained_poll_single(c)); 1224be6a6d1SKevin Wolf } 1234be6a6d1SKevin Wolf } 1244be6a6d1SKevin Wolf 125d9e0dfa2SEric Blake static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src) 126d9e0dfa2SEric Blake { 127d9e0dfa2SEric Blake dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer); 128d9e0dfa2SEric Blake dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer); 129d9e0dfa2SEric Blake dst->opt_mem_alignment = MAX(dst->opt_mem_alignment, 130d9e0dfa2SEric Blake src->opt_mem_alignment); 131d9e0dfa2SEric Blake dst->min_mem_alignment = MAX(dst->min_mem_alignment, 132d9e0dfa2SEric Blake src->min_mem_alignment); 133d9e0dfa2SEric Blake dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov); 134d9e0dfa2SEric Blake } 135d9e0dfa2SEric Blake 13661007b31SStefan Hajnoczi void bdrv_refresh_limits(BlockDriverState *bs, Error **errp) 13761007b31SStefan Hajnoczi { 13833985614SVladimir Sementsov-Ogievskiy ERRP_GUARD(); 13961007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 14066b129acSMax Reitz BdrvChild *c; 14166b129acSMax Reitz bool have_limits; 14261007b31SStefan Hajnoczi 14361007b31SStefan Hajnoczi memset(&bs->bl, 0, sizeof(bs->bl)); 14461007b31SStefan Hajnoczi 14561007b31SStefan Hajnoczi if (!drv) { 14661007b31SStefan Hajnoczi return; 14761007b31SStefan Hajnoczi } 14861007b31SStefan Hajnoczi 14979ba8c98SEric Blake /* Default alignment based on whether driver has byte interface */ 150e31f6864SEric Blake bs->bl.request_alignment = (drv->bdrv_co_preadv || 151ac850bf0SVladimir Sementsov-Ogievskiy drv->bdrv_aio_preadv || 152ac850bf0SVladimir Sementsov-Ogievskiy drv->bdrv_co_preadv_part) ? 1 : 512; 15379ba8c98SEric Blake 15461007b31SStefan Hajnoczi /* Take some limits from the children as a default */ 15566b129acSMax Reitz have_limits = false; 15666b129acSMax Reitz QLIST_FOREACH(c, &bs->children, next) { 15766b129acSMax Reitz if (c->role & (BDRV_CHILD_DATA | BDRV_CHILD_FILTERED | BDRV_CHILD_COW)) 15866b129acSMax Reitz { 15933985614SVladimir Sementsov-Ogievskiy bdrv_refresh_limits(c->bs, errp); 16033985614SVladimir Sementsov-Ogievskiy if (*errp) { 16161007b31SStefan Hajnoczi return; 16261007b31SStefan Hajnoczi } 16366b129acSMax Reitz bdrv_merge_limits(&bs->bl, &c->bs->bl); 16466b129acSMax Reitz have_limits = true; 16566b129acSMax Reitz } 16666b129acSMax Reitz } 16766b129acSMax Reitz 16866b129acSMax Reitz if (!have_limits) { 1694196d2f0SDenis V. Lunev bs->bl.min_mem_alignment = 512; 170038adc2fSWei Yang bs->bl.opt_mem_alignment = qemu_real_host_page_size; 171bd44feb7SStefan Hajnoczi 172bd44feb7SStefan Hajnoczi /* Safe default since most protocols use readv()/writev()/etc */ 173bd44feb7SStefan Hajnoczi bs->bl.max_iov = IOV_MAX; 17461007b31SStefan Hajnoczi } 17561007b31SStefan Hajnoczi 17661007b31SStefan Hajnoczi /* Then let the driver override it */ 17761007b31SStefan Hajnoczi if (drv->bdrv_refresh_limits) { 17861007b31SStefan Hajnoczi drv->bdrv_refresh_limits(bs, errp); 1798b117001SVladimir Sementsov-Ogievskiy if (*errp) { 1808b117001SVladimir Sementsov-Ogievskiy return; 1818b117001SVladimir Sementsov-Ogievskiy } 1828b117001SVladimir Sementsov-Ogievskiy } 1838b117001SVladimir Sementsov-Ogievskiy 1848b117001SVladimir Sementsov-Ogievskiy if (bs->bl.request_alignment > BDRV_MAX_ALIGNMENT) { 1858b117001SVladimir Sementsov-Ogievskiy error_setg(errp, "Driver requires too large request alignment"); 18661007b31SStefan Hajnoczi } 18761007b31SStefan Hajnoczi } 18861007b31SStefan Hajnoczi 18961007b31SStefan Hajnoczi /** 19061007b31SStefan Hajnoczi * The copy-on-read flag is actually a reference count so multiple users may 19161007b31SStefan Hajnoczi * use the feature without worrying about clobbering its previous state. 19261007b31SStefan Hajnoczi * Copy-on-read stays enabled until all users have called to disable it. 19361007b31SStefan Hajnoczi */ 19461007b31SStefan Hajnoczi void bdrv_enable_copy_on_read(BlockDriverState *bs) 19561007b31SStefan Hajnoczi { 196d73415a3SStefan Hajnoczi qatomic_inc(&bs->copy_on_read); 19761007b31SStefan Hajnoczi } 19861007b31SStefan Hajnoczi 19961007b31SStefan Hajnoczi void bdrv_disable_copy_on_read(BlockDriverState *bs) 20061007b31SStefan Hajnoczi { 201d73415a3SStefan Hajnoczi int old = qatomic_fetch_dec(&bs->copy_on_read); 202d3faa13eSPaolo Bonzini assert(old >= 1); 20361007b31SStefan Hajnoczi } 20461007b31SStefan Hajnoczi 20561124f03SPaolo Bonzini typedef struct { 20661124f03SPaolo Bonzini Coroutine *co; 20761124f03SPaolo Bonzini BlockDriverState *bs; 20861124f03SPaolo Bonzini bool done; 209481cad48SManos Pitsidianakis bool begin; 210b0165585SKevin Wolf bool recursive; 211fe4f0614SKevin Wolf bool poll; 2120152bf40SKevin Wolf BdrvChild *parent; 2136cd5c9d7SKevin Wolf bool ignore_bds_parents; 2148e1da77eSMax Reitz int *drained_end_counter; 21561124f03SPaolo Bonzini } BdrvCoDrainData; 21661124f03SPaolo Bonzini 21761124f03SPaolo Bonzini static void coroutine_fn bdrv_drain_invoke_entry(void *opaque) 21861124f03SPaolo Bonzini { 21961124f03SPaolo Bonzini BdrvCoDrainData *data = opaque; 22061124f03SPaolo Bonzini BlockDriverState *bs = data->bs; 22161124f03SPaolo Bonzini 222481cad48SManos Pitsidianakis if (data->begin) { 223f8ea8dacSManos Pitsidianakis bs->drv->bdrv_co_drain_begin(bs); 224481cad48SManos Pitsidianakis } else { 225481cad48SManos Pitsidianakis bs->drv->bdrv_co_drain_end(bs); 226481cad48SManos Pitsidianakis } 22761124f03SPaolo Bonzini 22865181d63SMax Reitz /* Set data->done and decrement drained_end_counter before bdrv_wakeup() */ 229d73415a3SStefan Hajnoczi qatomic_mb_set(&data->done, true); 230e037c09cSMax Reitz if (!data->begin) { 231d73415a3SStefan Hajnoczi qatomic_dec(data->drained_end_counter); 2328e1da77eSMax Reitz } 23365181d63SMax Reitz bdrv_dec_in_flight(bs); 2348e1da77eSMax Reitz 2350109e7e6SKevin Wolf g_free(data); 2360109e7e6SKevin Wolf } 23761124f03SPaolo Bonzini 238db0289b9SKevin Wolf /* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */ 2398e1da77eSMax Reitz static void bdrv_drain_invoke(BlockDriverState *bs, bool begin, 2408e1da77eSMax Reitz int *drained_end_counter) 24161124f03SPaolo Bonzini { 2420109e7e6SKevin Wolf BdrvCoDrainData *data; 24361124f03SPaolo Bonzini 244f8ea8dacSManos Pitsidianakis if (!bs->drv || (begin && !bs->drv->bdrv_co_drain_begin) || 245481cad48SManos Pitsidianakis (!begin && !bs->drv->bdrv_co_drain_end)) { 24661124f03SPaolo Bonzini return; 24761124f03SPaolo Bonzini } 24861124f03SPaolo Bonzini 2490109e7e6SKevin Wolf data = g_new(BdrvCoDrainData, 1); 2500109e7e6SKevin Wolf *data = (BdrvCoDrainData) { 2510109e7e6SKevin Wolf .bs = bs, 2520109e7e6SKevin Wolf .done = false, 2538e1da77eSMax Reitz .begin = begin, 2548e1da77eSMax Reitz .drained_end_counter = drained_end_counter, 2550109e7e6SKevin Wolf }; 2560109e7e6SKevin Wolf 257e037c09cSMax Reitz if (!begin) { 258d73415a3SStefan Hajnoczi qatomic_inc(drained_end_counter); 2598e1da77eSMax Reitz } 2608e1da77eSMax Reitz 2610109e7e6SKevin Wolf /* Make sure the driver callback completes during the polling phase for 2620109e7e6SKevin Wolf * drain_begin. */ 2630109e7e6SKevin Wolf bdrv_inc_in_flight(bs); 2640109e7e6SKevin Wolf data->co = qemu_coroutine_create(bdrv_drain_invoke_entry, data); 2650109e7e6SKevin Wolf aio_co_schedule(bdrv_get_aio_context(bs), data->co); 26661124f03SPaolo Bonzini } 26761124f03SPaolo Bonzini 2681cc8e54aSKevin Wolf /* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */ 269fe4f0614SKevin Wolf bool bdrv_drain_poll(BlockDriverState *bs, bool recursive, 2706cd5c9d7SKevin Wolf BdrvChild *ignore_parent, bool ignore_bds_parents) 27189bd0305SKevin Wolf { 272fe4f0614SKevin Wolf BdrvChild *child, *next; 273fe4f0614SKevin Wolf 2746cd5c9d7SKevin Wolf if (bdrv_parent_drained_poll(bs, ignore_parent, ignore_bds_parents)) { 27589bd0305SKevin Wolf return true; 27689bd0305SKevin Wolf } 27789bd0305SKevin Wolf 278d73415a3SStefan Hajnoczi if (qatomic_read(&bs->in_flight)) { 279fe4f0614SKevin Wolf return true; 28089bd0305SKevin Wolf } 28189bd0305SKevin Wolf 282fe4f0614SKevin Wolf if (recursive) { 2836cd5c9d7SKevin Wolf assert(!ignore_bds_parents); 284fe4f0614SKevin Wolf QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 2856cd5c9d7SKevin Wolf if (bdrv_drain_poll(child->bs, recursive, child, false)) { 286fe4f0614SKevin Wolf return true; 287fe4f0614SKevin Wolf } 288fe4f0614SKevin Wolf } 289fe4f0614SKevin Wolf } 290fe4f0614SKevin Wolf 291fe4f0614SKevin Wolf return false; 292fe4f0614SKevin Wolf } 293fe4f0614SKevin Wolf 294fe4f0614SKevin Wolf static bool bdrv_drain_poll_top_level(BlockDriverState *bs, bool recursive, 29589bd0305SKevin Wolf BdrvChild *ignore_parent) 2961cc8e54aSKevin Wolf { 2976cd5c9d7SKevin Wolf return bdrv_drain_poll(bs, recursive, ignore_parent, false); 2981cc8e54aSKevin Wolf } 2991cc8e54aSKevin Wolf 300b0165585SKevin Wolf static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive, 3016cd5c9d7SKevin Wolf BdrvChild *parent, bool ignore_bds_parents, 3026cd5c9d7SKevin Wolf bool poll); 303b0165585SKevin Wolf static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive, 3048e1da77eSMax Reitz BdrvChild *parent, bool ignore_bds_parents, 3058e1da77eSMax Reitz int *drained_end_counter); 3060152bf40SKevin Wolf 307a77fd4bbSFam Zheng static void bdrv_co_drain_bh_cb(void *opaque) 308a77fd4bbSFam Zheng { 309a77fd4bbSFam Zheng BdrvCoDrainData *data = opaque; 310a77fd4bbSFam Zheng Coroutine *co = data->co; 31199723548SPaolo Bonzini BlockDriverState *bs = data->bs; 312a77fd4bbSFam Zheng 313c8ca33d0SKevin Wolf if (bs) { 314aa1361d5SKevin Wolf AioContext *ctx = bdrv_get_aio_context(bs); 315aa1361d5SKevin Wolf aio_context_acquire(ctx); 31699723548SPaolo Bonzini bdrv_dec_in_flight(bs); 317481cad48SManos Pitsidianakis if (data->begin) { 318e037c09cSMax Reitz assert(!data->drained_end_counter); 3196cd5c9d7SKevin Wolf bdrv_do_drained_begin(bs, data->recursive, data->parent, 3206cd5c9d7SKevin Wolf data->ignore_bds_parents, data->poll); 321481cad48SManos Pitsidianakis } else { 322e037c09cSMax Reitz assert(!data->poll); 3236cd5c9d7SKevin Wolf bdrv_do_drained_end(bs, data->recursive, data->parent, 3248e1da77eSMax Reitz data->ignore_bds_parents, 3258e1da77eSMax Reitz data->drained_end_counter); 326481cad48SManos Pitsidianakis } 327aa1361d5SKevin Wolf aio_context_release(ctx); 328c8ca33d0SKevin Wolf } else { 329c8ca33d0SKevin Wolf assert(data->begin); 330c8ca33d0SKevin Wolf bdrv_drain_all_begin(); 331c8ca33d0SKevin Wolf } 332481cad48SManos Pitsidianakis 333a77fd4bbSFam Zheng data->done = true; 3341919631eSPaolo Bonzini aio_co_wake(co); 335a77fd4bbSFam Zheng } 336a77fd4bbSFam Zheng 337481cad48SManos Pitsidianakis static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs, 338b0165585SKevin Wolf bool begin, bool recursive, 3396cd5c9d7SKevin Wolf BdrvChild *parent, 3406cd5c9d7SKevin Wolf bool ignore_bds_parents, 3418e1da77eSMax Reitz bool poll, 3428e1da77eSMax Reitz int *drained_end_counter) 343a77fd4bbSFam Zheng { 344a77fd4bbSFam Zheng BdrvCoDrainData data; 345*960d5fb3SKevin Wolf Coroutine *self = qemu_coroutine_self(); 346*960d5fb3SKevin Wolf AioContext *ctx = bdrv_get_aio_context(bs); 347*960d5fb3SKevin Wolf AioContext *co_ctx = qemu_coroutine_get_aio_context(self); 348a77fd4bbSFam Zheng 349a77fd4bbSFam Zheng /* Calling bdrv_drain() from a BH ensures the current coroutine yields and 350c40a2545SStefan Hajnoczi * other coroutines run if they were queued by aio_co_enter(). */ 351a77fd4bbSFam Zheng 352a77fd4bbSFam Zheng assert(qemu_in_coroutine()); 353a77fd4bbSFam Zheng data = (BdrvCoDrainData) { 354*960d5fb3SKevin Wolf .co = self, 355a77fd4bbSFam Zheng .bs = bs, 356a77fd4bbSFam Zheng .done = false, 357481cad48SManos Pitsidianakis .begin = begin, 358b0165585SKevin Wolf .recursive = recursive, 3590152bf40SKevin Wolf .parent = parent, 3606cd5c9d7SKevin Wolf .ignore_bds_parents = ignore_bds_parents, 361fe4f0614SKevin Wolf .poll = poll, 3628e1da77eSMax Reitz .drained_end_counter = drained_end_counter, 363a77fd4bbSFam Zheng }; 3648e1da77eSMax Reitz 365c8ca33d0SKevin Wolf if (bs) { 36699723548SPaolo Bonzini bdrv_inc_in_flight(bs); 367c8ca33d0SKevin Wolf } 368*960d5fb3SKevin Wolf 369*960d5fb3SKevin Wolf /* 370*960d5fb3SKevin Wolf * Temporarily drop the lock across yield or we would get deadlocks. 371*960d5fb3SKevin Wolf * bdrv_co_drain_bh_cb() reaquires the lock as needed. 372*960d5fb3SKevin Wolf * 373*960d5fb3SKevin Wolf * When we yield below, the lock for the current context will be 374*960d5fb3SKevin Wolf * released, so if this is actually the lock that protects bs, don't drop 375*960d5fb3SKevin Wolf * it a second time. 376*960d5fb3SKevin Wolf */ 377*960d5fb3SKevin Wolf if (ctx != co_ctx) { 378*960d5fb3SKevin Wolf aio_context_release(ctx); 379*960d5fb3SKevin Wolf } 380*960d5fb3SKevin Wolf replay_bh_schedule_oneshot_event(ctx, bdrv_co_drain_bh_cb, &data); 381a77fd4bbSFam Zheng 382a77fd4bbSFam Zheng qemu_coroutine_yield(); 383a77fd4bbSFam Zheng /* If we are resumed from some other event (such as an aio completion or a 384a77fd4bbSFam Zheng * timer callback), it is a bug in the caller that should be fixed. */ 385a77fd4bbSFam Zheng assert(data.done); 386*960d5fb3SKevin Wolf 387*960d5fb3SKevin Wolf /* Reaquire the AioContext of bs if we dropped it */ 388*960d5fb3SKevin Wolf if (ctx != co_ctx) { 389*960d5fb3SKevin Wolf aio_context_acquire(ctx); 390*960d5fb3SKevin Wolf } 391a77fd4bbSFam Zheng } 392a77fd4bbSFam Zheng 393dcf94a23SKevin Wolf void bdrv_do_drained_begin_quiesce(BlockDriverState *bs, 3946cd5c9d7SKevin Wolf BdrvChild *parent, bool ignore_bds_parents) 395dcf94a23SKevin Wolf { 396dcf94a23SKevin Wolf assert(!qemu_in_coroutine()); 397dcf94a23SKevin Wolf 398dcf94a23SKevin Wolf /* Stop things in parent-to-child order */ 399d73415a3SStefan Hajnoczi if (qatomic_fetch_inc(&bs->quiesce_counter) == 0) { 400dcf94a23SKevin Wolf aio_disable_external(bdrv_get_aio_context(bs)); 401dcf94a23SKevin Wolf } 402dcf94a23SKevin Wolf 4036cd5c9d7SKevin Wolf bdrv_parent_drained_begin(bs, parent, ignore_bds_parents); 4048e1da77eSMax Reitz bdrv_drain_invoke(bs, true, NULL); 405dcf94a23SKevin Wolf } 406dcf94a23SKevin Wolf 407dcf94a23SKevin Wolf static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive, 4086cd5c9d7SKevin Wolf BdrvChild *parent, bool ignore_bds_parents, 4096cd5c9d7SKevin Wolf bool poll) 4106820643fSKevin Wolf { 411b0165585SKevin Wolf BdrvChild *child, *next; 412b0165585SKevin Wolf 413d42cf288SPaolo Bonzini if (qemu_in_coroutine()) { 4146cd5c9d7SKevin Wolf bdrv_co_yield_to_drain(bs, true, recursive, parent, ignore_bds_parents, 4158e1da77eSMax Reitz poll, NULL); 416d42cf288SPaolo Bonzini return; 417d42cf288SPaolo Bonzini } 418d42cf288SPaolo Bonzini 4196cd5c9d7SKevin Wolf bdrv_do_drained_begin_quiesce(bs, parent, ignore_bds_parents); 420d30b8e64SKevin Wolf 421b0165585SKevin Wolf if (recursive) { 4226cd5c9d7SKevin Wolf assert(!ignore_bds_parents); 423d736f119SKevin Wolf bs->recursive_quiesce_counter++; 424b0165585SKevin Wolf QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 4256cd5c9d7SKevin Wolf bdrv_do_drained_begin(child->bs, true, child, ignore_bds_parents, 4266cd5c9d7SKevin Wolf false); 427b0165585SKevin Wolf } 428b0165585SKevin Wolf } 429fe4f0614SKevin Wolf 430fe4f0614SKevin Wolf /* 431fe4f0614SKevin Wolf * Wait for drained requests to finish. 432fe4f0614SKevin Wolf * 433fe4f0614SKevin Wolf * Calling BDRV_POLL_WHILE() only once for the top-level node is okay: The 434fe4f0614SKevin Wolf * call is needed so things in this AioContext can make progress even 435fe4f0614SKevin Wolf * though we don't return to the main AioContext loop - this automatically 436fe4f0614SKevin Wolf * includes other nodes in the same AioContext and therefore all child 437fe4f0614SKevin Wolf * nodes. 438fe4f0614SKevin Wolf */ 439fe4f0614SKevin Wolf if (poll) { 4406cd5c9d7SKevin Wolf assert(!ignore_bds_parents); 441fe4f0614SKevin Wolf BDRV_POLL_WHILE(bs, bdrv_drain_poll_top_level(bs, recursive, parent)); 442fe4f0614SKevin Wolf } 4436820643fSKevin Wolf } 4446820643fSKevin Wolf 4450152bf40SKevin Wolf void bdrv_drained_begin(BlockDriverState *bs) 4460152bf40SKevin Wolf { 4476cd5c9d7SKevin Wolf bdrv_do_drained_begin(bs, false, NULL, false, true); 4480152bf40SKevin Wolf } 4490152bf40SKevin Wolf 450b0165585SKevin Wolf void bdrv_subtree_drained_begin(BlockDriverState *bs) 4516820643fSKevin Wolf { 4526cd5c9d7SKevin Wolf bdrv_do_drained_begin(bs, true, NULL, false, true); 453b0165585SKevin Wolf } 454b0165585SKevin Wolf 455e037c09cSMax Reitz /** 456e037c09cSMax Reitz * This function does not poll, nor must any of its recursively called 457e037c09cSMax Reitz * functions. The *drained_end_counter pointee will be incremented 458e037c09cSMax Reitz * once for every background operation scheduled, and decremented once 459e037c09cSMax Reitz * the operation settles. Therefore, the pointer must remain valid 460e037c09cSMax Reitz * until the pointee reaches 0. That implies that whoever sets up the 461e037c09cSMax Reitz * pointee has to poll until it is 0. 462e037c09cSMax Reitz * 463e037c09cSMax Reitz * We use atomic operations to access *drained_end_counter, because 464e037c09cSMax Reitz * (1) when called from bdrv_set_aio_context_ignore(), the subgraph of 465e037c09cSMax Reitz * @bs may contain nodes in different AioContexts, 466e037c09cSMax Reitz * (2) bdrv_drain_all_end() uses the same counter for all nodes, 467e037c09cSMax Reitz * regardless of which AioContext they are in. 468e037c09cSMax Reitz */ 4696cd5c9d7SKevin Wolf static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive, 4708e1da77eSMax Reitz BdrvChild *parent, bool ignore_bds_parents, 4718e1da77eSMax Reitz int *drained_end_counter) 472b0165585SKevin Wolf { 47361ad631cSMax Reitz BdrvChild *child; 4740f115168SKevin Wolf int old_quiesce_counter; 4750f115168SKevin Wolf 476e037c09cSMax Reitz assert(drained_end_counter != NULL); 477e037c09cSMax Reitz 478481cad48SManos Pitsidianakis if (qemu_in_coroutine()) { 4796cd5c9d7SKevin Wolf bdrv_co_yield_to_drain(bs, false, recursive, parent, ignore_bds_parents, 4808e1da77eSMax Reitz false, drained_end_counter); 481481cad48SManos Pitsidianakis return; 482481cad48SManos Pitsidianakis } 4836820643fSKevin Wolf assert(bs->quiesce_counter > 0); 4846820643fSKevin Wolf 48560369b86SKevin Wolf /* Re-enable things in child-to-parent order */ 4868e1da77eSMax Reitz bdrv_drain_invoke(bs, false, drained_end_counter); 487e037c09cSMax Reitz bdrv_parent_drained_end(bs, parent, ignore_bds_parents, 488e037c09cSMax Reitz drained_end_counter); 4895cb2737eSMax Reitz 490d73415a3SStefan Hajnoczi old_quiesce_counter = qatomic_fetch_dec(&bs->quiesce_counter); 4910f115168SKevin Wolf if (old_quiesce_counter == 1) { 4926820643fSKevin Wolf aio_enable_external(bdrv_get_aio_context(bs)); 4936820643fSKevin Wolf } 494b0165585SKevin Wolf 495b0165585SKevin Wolf if (recursive) { 4966cd5c9d7SKevin Wolf assert(!ignore_bds_parents); 497d736f119SKevin Wolf bs->recursive_quiesce_counter--; 49861ad631cSMax Reitz QLIST_FOREACH(child, &bs->children, next) { 4998e1da77eSMax Reitz bdrv_do_drained_end(child->bs, true, child, ignore_bds_parents, 5008e1da77eSMax Reitz drained_end_counter); 501b0165585SKevin Wolf } 502b0165585SKevin Wolf } 5030f115168SKevin Wolf } 5046820643fSKevin Wolf 5050152bf40SKevin Wolf void bdrv_drained_end(BlockDriverState *bs) 5060152bf40SKevin Wolf { 507e037c09cSMax Reitz int drained_end_counter = 0; 508e037c09cSMax Reitz bdrv_do_drained_end(bs, false, NULL, false, &drained_end_counter); 509d73415a3SStefan Hajnoczi BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0); 510e037c09cSMax Reitz } 511e037c09cSMax Reitz 512e037c09cSMax Reitz void bdrv_drained_end_no_poll(BlockDriverState *bs, int *drained_end_counter) 513e037c09cSMax Reitz { 514e037c09cSMax Reitz bdrv_do_drained_end(bs, false, NULL, false, drained_end_counter); 515b0165585SKevin Wolf } 516b0165585SKevin Wolf 517b0165585SKevin Wolf void bdrv_subtree_drained_end(BlockDriverState *bs) 518b0165585SKevin Wolf { 519e037c09cSMax Reitz int drained_end_counter = 0; 520e037c09cSMax Reitz bdrv_do_drained_end(bs, true, NULL, false, &drained_end_counter); 521d73415a3SStefan Hajnoczi BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0); 5220152bf40SKevin Wolf } 5230152bf40SKevin Wolf 524d736f119SKevin Wolf void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent) 525d736f119SKevin Wolf { 526d736f119SKevin Wolf int i; 527d736f119SKevin Wolf 528d736f119SKevin Wolf for (i = 0; i < new_parent->recursive_quiesce_counter; i++) { 5296cd5c9d7SKevin Wolf bdrv_do_drained_begin(child->bs, true, child, false, true); 530d736f119SKevin Wolf } 531d736f119SKevin Wolf } 532d736f119SKevin Wolf 533d736f119SKevin Wolf void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent) 534d736f119SKevin Wolf { 535e037c09cSMax Reitz int drained_end_counter = 0; 536d736f119SKevin Wolf int i; 537d736f119SKevin Wolf 538d736f119SKevin Wolf for (i = 0; i < old_parent->recursive_quiesce_counter; i++) { 539e037c09cSMax Reitz bdrv_do_drained_end(child->bs, true, child, false, 540e037c09cSMax Reitz &drained_end_counter); 541d736f119SKevin Wolf } 542e037c09cSMax Reitz 543d73415a3SStefan Hajnoczi BDRV_POLL_WHILE(child->bs, qatomic_read(&drained_end_counter) > 0); 544d736f119SKevin Wolf } 545d736f119SKevin Wolf 54661007b31SStefan Hajnoczi /* 54767da1dc5SFam Zheng * Wait for pending requests to complete on a single BlockDriverState subtree, 54867da1dc5SFam Zheng * and suspend block driver's internal I/O until next request arrives. 54961007b31SStefan Hajnoczi * 55061007b31SStefan Hajnoczi * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState 55161007b31SStefan Hajnoczi * AioContext. 55261007b31SStefan Hajnoczi */ 553b6e84c97SPaolo Bonzini void coroutine_fn bdrv_co_drain(BlockDriverState *bs) 554b6e84c97SPaolo Bonzini { 5556820643fSKevin Wolf assert(qemu_in_coroutine()); 5566820643fSKevin Wolf bdrv_drained_begin(bs); 5576820643fSKevin Wolf bdrv_drained_end(bs); 558b6e84c97SPaolo Bonzini } 559b6e84c97SPaolo Bonzini 56061007b31SStefan Hajnoczi void bdrv_drain(BlockDriverState *bs) 56161007b31SStefan Hajnoczi { 5626820643fSKevin Wolf bdrv_drained_begin(bs); 5636820643fSKevin Wolf bdrv_drained_end(bs); 56461007b31SStefan Hajnoczi } 56561007b31SStefan Hajnoczi 566c13ad59fSKevin Wolf static void bdrv_drain_assert_idle(BlockDriverState *bs) 567c13ad59fSKevin Wolf { 568c13ad59fSKevin Wolf BdrvChild *child, *next; 569c13ad59fSKevin Wolf 570d73415a3SStefan Hajnoczi assert(qatomic_read(&bs->in_flight) == 0); 571c13ad59fSKevin Wolf QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 572c13ad59fSKevin Wolf bdrv_drain_assert_idle(child->bs); 573c13ad59fSKevin Wolf } 574c13ad59fSKevin Wolf } 575c13ad59fSKevin Wolf 5760f12264eSKevin Wolf unsigned int bdrv_drain_all_count = 0; 5770f12264eSKevin Wolf 5780f12264eSKevin Wolf static bool bdrv_drain_all_poll(void) 5790f12264eSKevin Wolf { 5800f12264eSKevin Wolf BlockDriverState *bs = NULL; 5810f12264eSKevin Wolf bool result = false; 5820f12264eSKevin Wolf 5830f12264eSKevin Wolf /* bdrv_drain_poll() can't make changes to the graph and we are holding the 5840f12264eSKevin Wolf * main AioContext lock, so iterating bdrv_next_all_states() is safe. */ 5850f12264eSKevin Wolf while ((bs = bdrv_next_all_states(bs))) { 5860f12264eSKevin Wolf AioContext *aio_context = bdrv_get_aio_context(bs); 5870f12264eSKevin Wolf aio_context_acquire(aio_context); 5880f12264eSKevin Wolf result |= bdrv_drain_poll(bs, false, NULL, true); 5890f12264eSKevin Wolf aio_context_release(aio_context); 5900f12264eSKevin Wolf } 5910f12264eSKevin Wolf 5920f12264eSKevin Wolf return result; 5930f12264eSKevin Wolf } 5940f12264eSKevin Wolf 59561007b31SStefan Hajnoczi /* 59661007b31SStefan Hajnoczi * Wait for pending requests to complete across all BlockDriverStates 59761007b31SStefan Hajnoczi * 59861007b31SStefan Hajnoczi * This function does not flush data to disk, use bdrv_flush_all() for that 59961007b31SStefan Hajnoczi * after calling this function. 600c0778f66SAlberto Garcia * 601c0778f66SAlberto Garcia * This pauses all block jobs and disables external clients. It must 602c0778f66SAlberto Garcia * be paired with bdrv_drain_all_end(). 603c0778f66SAlberto Garcia * 604c0778f66SAlberto Garcia * NOTE: no new block jobs or BlockDriverStates can be created between 605c0778f66SAlberto Garcia * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls. 60661007b31SStefan Hajnoczi */ 607c0778f66SAlberto Garcia void bdrv_drain_all_begin(void) 60861007b31SStefan Hajnoczi { 6090f12264eSKevin Wolf BlockDriverState *bs = NULL; 61061007b31SStefan Hajnoczi 611c8ca33d0SKevin Wolf if (qemu_in_coroutine()) { 6128e1da77eSMax Reitz bdrv_co_yield_to_drain(NULL, true, false, NULL, true, true, NULL); 613c8ca33d0SKevin Wolf return; 614c8ca33d0SKevin Wolf } 615c8ca33d0SKevin Wolf 616c8aa7895SPavel Dovgalyuk /* 617c8aa7895SPavel Dovgalyuk * bdrv queue is managed by record/replay, 618c8aa7895SPavel Dovgalyuk * waiting for finishing the I/O requests may 619c8aa7895SPavel Dovgalyuk * be infinite 620c8aa7895SPavel Dovgalyuk */ 621c8aa7895SPavel Dovgalyuk if (replay_events_enabled()) { 622c8aa7895SPavel Dovgalyuk return; 623c8aa7895SPavel Dovgalyuk } 624c8aa7895SPavel Dovgalyuk 6250f12264eSKevin Wolf /* AIO_WAIT_WHILE() with a NULL context can only be called from the main 6260f12264eSKevin Wolf * loop AioContext, so make sure we're in the main context. */ 6279a7e86c8SKevin Wolf assert(qemu_get_current_aio_context() == qemu_get_aio_context()); 6280f12264eSKevin Wolf assert(bdrv_drain_all_count < INT_MAX); 6290f12264eSKevin Wolf bdrv_drain_all_count++; 6309a7e86c8SKevin Wolf 6310f12264eSKevin Wolf /* Quiesce all nodes, without polling in-flight requests yet. The graph 6320f12264eSKevin Wolf * cannot change during this loop. */ 6330f12264eSKevin Wolf while ((bs = bdrv_next_all_states(bs))) { 63461007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 63561007b31SStefan Hajnoczi 63661007b31SStefan Hajnoczi aio_context_acquire(aio_context); 6370f12264eSKevin Wolf bdrv_do_drained_begin(bs, false, NULL, true, false); 63861007b31SStefan Hajnoczi aio_context_release(aio_context); 63961007b31SStefan Hajnoczi } 64061007b31SStefan Hajnoczi 6410f12264eSKevin Wolf /* Now poll the in-flight requests */ 642cfe29d82SKevin Wolf AIO_WAIT_WHILE(NULL, bdrv_drain_all_poll()); 6430f12264eSKevin Wolf 6440f12264eSKevin Wolf while ((bs = bdrv_next_all_states(bs))) { 645c13ad59fSKevin Wolf bdrv_drain_assert_idle(bs); 646f406c03cSAlexander Yarygin } 647f406c03cSAlexander Yarygin } 648c0778f66SAlberto Garcia 6491a6d3bd2SGreg Kurz void bdrv_drain_all_end_quiesce(BlockDriverState *bs) 6501a6d3bd2SGreg Kurz { 6511a6d3bd2SGreg Kurz int drained_end_counter = 0; 6521a6d3bd2SGreg Kurz 6531a6d3bd2SGreg Kurz g_assert(bs->quiesce_counter > 0); 6541a6d3bd2SGreg Kurz g_assert(!bs->refcnt); 6551a6d3bd2SGreg Kurz 6561a6d3bd2SGreg Kurz while (bs->quiesce_counter) { 6571a6d3bd2SGreg Kurz bdrv_do_drained_end(bs, false, NULL, true, &drained_end_counter); 6581a6d3bd2SGreg Kurz } 6591a6d3bd2SGreg Kurz BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0); 6601a6d3bd2SGreg Kurz } 6611a6d3bd2SGreg Kurz 662c0778f66SAlberto Garcia void bdrv_drain_all_end(void) 663c0778f66SAlberto Garcia { 6640f12264eSKevin Wolf BlockDriverState *bs = NULL; 665e037c09cSMax Reitz int drained_end_counter = 0; 666c0778f66SAlberto Garcia 667c8aa7895SPavel Dovgalyuk /* 668c8aa7895SPavel Dovgalyuk * bdrv queue is managed by record/replay, 669c8aa7895SPavel Dovgalyuk * waiting for finishing the I/O requests may 670c8aa7895SPavel Dovgalyuk * be endless 671c8aa7895SPavel Dovgalyuk */ 672c8aa7895SPavel Dovgalyuk if (replay_events_enabled()) { 673c8aa7895SPavel Dovgalyuk return; 674c8aa7895SPavel Dovgalyuk } 675c8aa7895SPavel Dovgalyuk 6760f12264eSKevin Wolf while ((bs = bdrv_next_all_states(bs))) { 67761007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 67861007b31SStefan Hajnoczi 67961007b31SStefan Hajnoczi aio_context_acquire(aio_context); 680e037c09cSMax Reitz bdrv_do_drained_end(bs, false, NULL, true, &drained_end_counter); 68161007b31SStefan Hajnoczi aio_context_release(aio_context); 68261007b31SStefan Hajnoczi } 6830f12264eSKevin Wolf 684e037c09cSMax Reitz assert(qemu_get_current_aio_context() == qemu_get_aio_context()); 685d73415a3SStefan Hajnoczi AIO_WAIT_WHILE(NULL, qatomic_read(&drained_end_counter) > 0); 686e037c09cSMax Reitz 6870f12264eSKevin Wolf assert(bdrv_drain_all_count > 0); 6880f12264eSKevin Wolf bdrv_drain_all_count--; 68961007b31SStefan Hajnoczi } 69061007b31SStefan Hajnoczi 691c0778f66SAlberto Garcia void bdrv_drain_all(void) 692c0778f66SAlberto Garcia { 693c0778f66SAlberto Garcia bdrv_drain_all_begin(); 694c0778f66SAlberto Garcia bdrv_drain_all_end(); 695c0778f66SAlberto Garcia } 696c0778f66SAlberto Garcia 69761007b31SStefan Hajnoczi /** 69861007b31SStefan Hajnoczi * Remove an active request from the tracked requests list 69961007b31SStefan Hajnoczi * 70061007b31SStefan Hajnoczi * This function should be called when a tracked request is completing. 70161007b31SStefan Hajnoczi */ 70261007b31SStefan Hajnoczi static void tracked_request_end(BdrvTrackedRequest *req) 70361007b31SStefan Hajnoczi { 70461007b31SStefan Hajnoczi if (req->serialising) { 705d73415a3SStefan Hajnoczi qatomic_dec(&req->bs->serialising_in_flight); 70661007b31SStefan Hajnoczi } 70761007b31SStefan Hajnoczi 7083783fa3dSPaolo Bonzini qemu_co_mutex_lock(&req->bs->reqs_lock); 70961007b31SStefan Hajnoczi QLIST_REMOVE(req, list); 71061007b31SStefan Hajnoczi qemu_co_queue_restart_all(&req->wait_queue); 7113783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&req->bs->reqs_lock); 71261007b31SStefan Hajnoczi } 71361007b31SStefan Hajnoczi 71461007b31SStefan Hajnoczi /** 71561007b31SStefan Hajnoczi * Add an active request to the tracked requests list 71661007b31SStefan Hajnoczi */ 71761007b31SStefan Hajnoczi static void tracked_request_begin(BdrvTrackedRequest *req, 71861007b31SStefan Hajnoczi BlockDriverState *bs, 71961007b31SStefan Hajnoczi int64_t offset, 72022931a15SFam Zheng uint64_t bytes, 721ebde595cSFam Zheng enum BdrvTrackedRequestType type) 72261007b31SStefan Hajnoczi { 72322931a15SFam Zheng assert(bytes <= INT64_MAX && offset <= INT64_MAX - bytes); 72422931a15SFam Zheng 72561007b31SStefan Hajnoczi *req = (BdrvTrackedRequest){ 72661007b31SStefan Hajnoczi .bs = bs, 72761007b31SStefan Hajnoczi .offset = offset, 72861007b31SStefan Hajnoczi .bytes = bytes, 729ebde595cSFam Zheng .type = type, 73061007b31SStefan Hajnoczi .co = qemu_coroutine_self(), 73161007b31SStefan Hajnoczi .serialising = false, 73261007b31SStefan Hajnoczi .overlap_offset = offset, 73361007b31SStefan Hajnoczi .overlap_bytes = bytes, 73461007b31SStefan Hajnoczi }; 73561007b31SStefan Hajnoczi 73661007b31SStefan Hajnoczi qemu_co_queue_init(&req->wait_queue); 73761007b31SStefan Hajnoczi 7383783fa3dSPaolo Bonzini qemu_co_mutex_lock(&bs->reqs_lock); 73961007b31SStefan Hajnoczi QLIST_INSERT_HEAD(&bs->tracked_requests, req, list); 7403783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&bs->reqs_lock); 74161007b31SStefan Hajnoczi } 74261007b31SStefan Hajnoczi 7433ba0e1a0SPaolo Bonzini static bool tracked_request_overlaps(BdrvTrackedRequest *req, 7443ba0e1a0SPaolo Bonzini int64_t offset, uint64_t bytes) 7453ba0e1a0SPaolo Bonzini { 7463ba0e1a0SPaolo Bonzini /* aaaa bbbb */ 7473ba0e1a0SPaolo Bonzini if (offset >= req->overlap_offset + req->overlap_bytes) { 7483ba0e1a0SPaolo Bonzini return false; 7493ba0e1a0SPaolo Bonzini } 7503ba0e1a0SPaolo Bonzini /* bbbb aaaa */ 7513ba0e1a0SPaolo Bonzini if (req->overlap_offset >= offset + bytes) { 7523ba0e1a0SPaolo Bonzini return false; 7533ba0e1a0SPaolo Bonzini } 7543ba0e1a0SPaolo Bonzini return true; 7553ba0e1a0SPaolo Bonzini } 7563ba0e1a0SPaolo Bonzini 7573ba0e1a0SPaolo Bonzini static bool coroutine_fn 7583ba0e1a0SPaolo Bonzini bdrv_wait_serialising_requests_locked(BlockDriverState *bs, 7593ba0e1a0SPaolo Bonzini BdrvTrackedRequest *self) 7603ba0e1a0SPaolo Bonzini { 7613ba0e1a0SPaolo Bonzini BdrvTrackedRequest *req; 7623ba0e1a0SPaolo Bonzini bool retry; 7633ba0e1a0SPaolo Bonzini bool waited = false; 7643ba0e1a0SPaolo Bonzini 7653ba0e1a0SPaolo Bonzini do { 7663ba0e1a0SPaolo Bonzini retry = false; 7673ba0e1a0SPaolo Bonzini QLIST_FOREACH(req, &bs->tracked_requests, list) { 7683ba0e1a0SPaolo Bonzini if (req == self || (!req->serialising && !self->serialising)) { 7693ba0e1a0SPaolo Bonzini continue; 7703ba0e1a0SPaolo Bonzini } 7713ba0e1a0SPaolo Bonzini if (tracked_request_overlaps(req, self->overlap_offset, 7723ba0e1a0SPaolo Bonzini self->overlap_bytes)) 7733ba0e1a0SPaolo Bonzini { 7743ba0e1a0SPaolo Bonzini /* Hitting this means there was a reentrant request, for 7753ba0e1a0SPaolo Bonzini * example, a block driver issuing nested requests. This must 7763ba0e1a0SPaolo Bonzini * never happen since it means deadlock. 7773ba0e1a0SPaolo Bonzini */ 7783ba0e1a0SPaolo Bonzini assert(qemu_coroutine_self() != req->co); 7793ba0e1a0SPaolo Bonzini 7803ba0e1a0SPaolo Bonzini /* If the request is already (indirectly) waiting for us, or 7813ba0e1a0SPaolo Bonzini * will wait for us as soon as it wakes up, then just go on 7823ba0e1a0SPaolo Bonzini * (instead of producing a deadlock in the former case). */ 7833ba0e1a0SPaolo Bonzini if (!req->waiting_for) { 7843ba0e1a0SPaolo Bonzini self->waiting_for = req; 7853ba0e1a0SPaolo Bonzini qemu_co_queue_wait(&req->wait_queue, &bs->reqs_lock); 7863ba0e1a0SPaolo Bonzini self->waiting_for = NULL; 7873ba0e1a0SPaolo Bonzini retry = true; 7883ba0e1a0SPaolo Bonzini waited = true; 7893ba0e1a0SPaolo Bonzini break; 7903ba0e1a0SPaolo Bonzini } 7913ba0e1a0SPaolo Bonzini } 7923ba0e1a0SPaolo Bonzini } 7933ba0e1a0SPaolo Bonzini } while (retry); 7943ba0e1a0SPaolo Bonzini return waited; 7953ba0e1a0SPaolo Bonzini } 7963ba0e1a0SPaolo Bonzini 79718fbd0deSPaolo Bonzini bool bdrv_mark_request_serialising(BdrvTrackedRequest *req, uint64_t align) 79861007b31SStefan Hajnoczi { 7993ba0e1a0SPaolo Bonzini BlockDriverState *bs = req->bs; 80061007b31SStefan Hajnoczi int64_t overlap_offset = req->offset & ~(align - 1); 80122931a15SFam Zheng uint64_t overlap_bytes = ROUND_UP(req->offset + req->bytes, align) 80261007b31SStefan Hajnoczi - overlap_offset; 8033ba0e1a0SPaolo Bonzini bool waited; 80461007b31SStefan Hajnoczi 8053ba0e1a0SPaolo Bonzini qemu_co_mutex_lock(&bs->reqs_lock); 80661007b31SStefan Hajnoczi if (!req->serialising) { 807d73415a3SStefan Hajnoczi qatomic_inc(&req->bs->serialising_in_flight); 80861007b31SStefan Hajnoczi req->serialising = true; 80961007b31SStefan Hajnoczi } 81061007b31SStefan Hajnoczi 81161007b31SStefan Hajnoczi req->overlap_offset = MIN(req->overlap_offset, overlap_offset); 81261007b31SStefan Hajnoczi req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes); 8133ba0e1a0SPaolo Bonzini waited = bdrv_wait_serialising_requests_locked(bs, req); 8143ba0e1a0SPaolo Bonzini qemu_co_mutex_unlock(&bs->reqs_lock); 8153ba0e1a0SPaolo Bonzini return waited; 81609d2f948SVladimir Sementsov-Ogievskiy } 81709d2f948SVladimir Sementsov-Ogievskiy 81861007b31SStefan Hajnoczi /** 819c28107e9SMax Reitz * Return the tracked request on @bs for the current coroutine, or 820c28107e9SMax Reitz * NULL if there is none. 821c28107e9SMax Reitz */ 822c28107e9SMax Reitz BdrvTrackedRequest *coroutine_fn bdrv_co_get_self_request(BlockDriverState *bs) 823c28107e9SMax Reitz { 824c28107e9SMax Reitz BdrvTrackedRequest *req; 825c28107e9SMax Reitz Coroutine *self = qemu_coroutine_self(); 826c28107e9SMax Reitz 827c28107e9SMax Reitz QLIST_FOREACH(req, &bs->tracked_requests, list) { 828c28107e9SMax Reitz if (req->co == self) { 829c28107e9SMax Reitz return req; 830c28107e9SMax Reitz } 831c28107e9SMax Reitz } 832c28107e9SMax Reitz 833c28107e9SMax Reitz return NULL; 834c28107e9SMax Reitz } 835c28107e9SMax Reitz 836c28107e9SMax Reitz /** 837244483e6SKevin Wolf * Round a region to cluster boundaries 838244483e6SKevin Wolf */ 839244483e6SKevin Wolf void bdrv_round_to_clusters(BlockDriverState *bs, 8407cfd5275SEric Blake int64_t offset, int64_t bytes, 841244483e6SKevin Wolf int64_t *cluster_offset, 8427cfd5275SEric Blake int64_t *cluster_bytes) 843244483e6SKevin Wolf { 844244483e6SKevin Wolf BlockDriverInfo bdi; 845244483e6SKevin Wolf 846244483e6SKevin Wolf if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 847244483e6SKevin Wolf *cluster_offset = offset; 848244483e6SKevin Wolf *cluster_bytes = bytes; 849244483e6SKevin Wolf } else { 850244483e6SKevin Wolf int64_t c = bdi.cluster_size; 851244483e6SKevin Wolf *cluster_offset = QEMU_ALIGN_DOWN(offset, c); 852244483e6SKevin Wolf *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c); 853244483e6SKevin Wolf } 854244483e6SKevin Wolf } 855244483e6SKevin Wolf 85661007b31SStefan Hajnoczi static int bdrv_get_cluster_size(BlockDriverState *bs) 85761007b31SStefan Hajnoczi { 85861007b31SStefan Hajnoczi BlockDriverInfo bdi; 85961007b31SStefan Hajnoczi int ret; 86061007b31SStefan Hajnoczi 86161007b31SStefan Hajnoczi ret = bdrv_get_info(bs, &bdi); 86261007b31SStefan Hajnoczi if (ret < 0 || bdi.cluster_size == 0) { 863a5b8dd2cSEric Blake return bs->bl.request_alignment; 86461007b31SStefan Hajnoczi } else { 86561007b31SStefan Hajnoczi return bdi.cluster_size; 86661007b31SStefan Hajnoczi } 86761007b31SStefan Hajnoczi } 86861007b31SStefan Hajnoczi 86999723548SPaolo Bonzini void bdrv_inc_in_flight(BlockDriverState *bs) 87099723548SPaolo Bonzini { 871d73415a3SStefan Hajnoczi qatomic_inc(&bs->in_flight); 87299723548SPaolo Bonzini } 87399723548SPaolo Bonzini 874c9d1a561SPaolo Bonzini void bdrv_wakeup(BlockDriverState *bs) 875c9d1a561SPaolo Bonzini { 876cfe29d82SKevin Wolf aio_wait_kick(); 877c9d1a561SPaolo Bonzini } 878c9d1a561SPaolo Bonzini 87999723548SPaolo Bonzini void bdrv_dec_in_flight(BlockDriverState *bs) 88099723548SPaolo Bonzini { 881d73415a3SStefan Hajnoczi qatomic_dec(&bs->in_flight); 882c9d1a561SPaolo Bonzini bdrv_wakeup(bs); 88399723548SPaolo Bonzini } 88499723548SPaolo Bonzini 88518fbd0deSPaolo Bonzini static bool coroutine_fn bdrv_wait_serialising_requests(BdrvTrackedRequest *self) 88661007b31SStefan Hajnoczi { 88761007b31SStefan Hajnoczi BlockDriverState *bs = self->bs; 88861007b31SStefan Hajnoczi bool waited = false; 88961007b31SStefan Hajnoczi 890d73415a3SStefan Hajnoczi if (!qatomic_read(&bs->serialising_in_flight)) { 89161007b31SStefan Hajnoczi return false; 89261007b31SStefan Hajnoczi } 89361007b31SStefan Hajnoczi 8943783fa3dSPaolo Bonzini qemu_co_mutex_lock(&bs->reqs_lock); 8953ba0e1a0SPaolo Bonzini waited = bdrv_wait_serialising_requests_locked(bs, self); 8963783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&bs->reqs_lock); 89761007b31SStefan Hajnoczi 89861007b31SStefan Hajnoczi return waited; 89961007b31SStefan Hajnoczi } 90061007b31SStefan Hajnoczi 9018b117001SVladimir Sementsov-Ogievskiy int bdrv_check_request(int64_t offset, int64_t bytes) 90261007b31SStefan Hajnoczi { 9038b117001SVladimir Sementsov-Ogievskiy if (offset < 0 || bytes < 0) { 90461007b31SStefan Hajnoczi return -EIO; 90561007b31SStefan Hajnoczi } 90661007b31SStefan Hajnoczi 9078b117001SVladimir Sementsov-Ogievskiy if (bytes > BDRV_MAX_LENGTH) { 9088b117001SVladimir Sementsov-Ogievskiy return -EIO; 9098b117001SVladimir Sementsov-Ogievskiy } 9108b117001SVladimir Sementsov-Ogievskiy 9118b117001SVladimir Sementsov-Ogievskiy if (offset > BDRV_MAX_LENGTH - bytes) { 9128b117001SVladimir Sementsov-Ogievskiy return -EIO; 9138b117001SVladimir Sementsov-Ogievskiy } 9148b117001SVladimir Sementsov-Ogievskiy 9158b117001SVladimir Sementsov-Ogievskiy return 0; 9168b117001SVladimir Sementsov-Ogievskiy } 9178b117001SVladimir Sementsov-Ogievskiy 9188b117001SVladimir Sementsov-Ogievskiy static int bdrv_check_request32(int64_t offset, int64_t bytes) 9198b117001SVladimir Sementsov-Ogievskiy { 9208b117001SVladimir Sementsov-Ogievskiy int ret = bdrv_check_request(offset, bytes); 9218b117001SVladimir Sementsov-Ogievskiy if (ret < 0) { 9228b117001SVladimir Sementsov-Ogievskiy return ret; 9238b117001SVladimir Sementsov-Ogievskiy } 9248b117001SVladimir Sementsov-Ogievskiy 9258b117001SVladimir Sementsov-Ogievskiy if (bytes > BDRV_REQUEST_MAX_BYTES) { 92661007b31SStefan Hajnoczi return -EIO; 92761007b31SStefan Hajnoczi } 92861007b31SStefan Hajnoczi 92961007b31SStefan Hajnoczi return 0; 93061007b31SStefan Hajnoczi } 93161007b31SStefan Hajnoczi 932720ff280SKevin Wolf int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset, 933f5a5ca79SManos Pitsidianakis int bytes, BdrvRequestFlags flags) 93461007b31SStefan Hajnoczi { 935fae2681aSVladimir Sementsov-Ogievskiy return bdrv_pwritev(child, offset, bytes, NULL, 936fae2681aSVladimir Sementsov-Ogievskiy BDRV_REQ_ZERO_WRITE | flags); 93761007b31SStefan Hajnoczi } 93861007b31SStefan Hajnoczi 93961007b31SStefan Hajnoczi /* 94074021bc4SEric Blake * Completely zero out a block device with the help of bdrv_pwrite_zeroes. 94161007b31SStefan Hajnoczi * The operation is sped up by checking the block status and only writing 94261007b31SStefan Hajnoczi * zeroes to the device if they currently do not return zeroes. Optional 94374021bc4SEric Blake * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP, 944465fe887SEric Blake * BDRV_REQ_FUA). 94561007b31SStefan Hajnoczi * 946f4649069SEric Blake * Returns < 0 on error, 0 on success. For error codes see bdrv_pwrite(). 94761007b31SStefan Hajnoczi */ 948720ff280SKevin Wolf int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags) 94961007b31SStefan Hajnoczi { 950237d78f8SEric Blake int ret; 951237d78f8SEric Blake int64_t target_size, bytes, offset = 0; 952720ff280SKevin Wolf BlockDriverState *bs = child->bs; 95361007b31SStefan Hajnoczi 9547286d610SEric Blake target_size = bdrv_getlength(bs); 9557286d610SEric Blake if (target_size < 0) { 9567286d610SEric Blake return target_size; 95761007b31SStefan Hajnoczi } 95861007b31SStefan Hajnoczi 95961007b31SStefan Hajnoczi for (;;) { 9607286d610SEric Blake bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES); 9617286d610SEric Blake if (bytes <= 0) { 96261007b31SStefan Hajnoczi return 0; 96361007b31SStefan Hajnoczi } 964237d78f8SEric Blake ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL); 96561007b31SStefan Hajnoczi if (ret < 0) { 96661007b31SStefan Hajnoczi return ret; 96761007b31SStefan Hajnoczi } 96861007b31SStefan Hajnoczi if (ret & BDRV_BLOCK_ZERO) { 969237d78f8SEric Blake offset += bytes; 97061007b31SStefan Hajnoczi continue; 97161007b31SStefan Hajnoczi } 972237d78f8SEric Blake ret = bdrv_pwrite_zeroes(child, offset, bytes, flags); 97361007b31SStefan Hajnoczi if (ret < 0) { 97461007b31SStefan Hajnoczi return ret; 97561007b31SStefan Hajnoczi } 976237d78f8SEric Blake offset += bytes; 97761007b31SStefan Hajnoczi } 97861007b31SStefan Hajnoczi } 97961007b31SStefan Hajnoczi 9802e11d756SAlberto Garcia /* See bdrv_pwrite() for the return codes */ 981cf2ab8fcSKevin Wolf int bdrv_pread(BdrvChild *child, int64_t offset, void *buf, int bytes) 98261007b31SStefan Hajnoczi { 983fae2681aSVladimir Sementsov-Ogievskiy int ret; 9840d93ed08SVladimir Sementsov-Ogievskiy QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes); 98561007b31SStefan Hajnoczi 98661007b31SStefan Hajnoczi if (bytes < 0) { 98761007b31SStefan Hajnoczi return -EINVAL; 98861007b31SStefan Hajnoczi } 98961007b31SStefan Hajnoczi 990fae2681aSVladimir Sementsov-Ogievskiy ret = bdrv_preadv(child, offset, bytes, &qiov, 0); 99161007b31SStefan Hajnoczi 992fae2681aSVladimir Sementsov-Ogievskiy return ret < 0 ? ret : bytes; 99361007b31SStefan Hajnoczi } 99461007b31SStefan Hajnoczi 9952e11d756SAlberto Garcia /* Return no. of bytes on success or < 0 on error. Important errors are: 9962e11d756SAlberto Garcia -EIO generic I/O error (may happen for all errors) 9972e11d756SAlberto Garcia -ENOMEDIUM No media inserted. 9982e11d756SAlberto Garcia -EINVAL Invalid offset or number of bytes 9992e11d756SAlberto Garcia -EACCES Trying to write a read-only device 10002e11d756SAlberto Garcia */ 1001d9ca2ea2SKevin Wolf int bdrv_pwrite(BdrvChild *child, int64_t offset, const void *buf, int bytes) 100261007b31SStefan Hajnoczi { 1003fae2681aSVladimir Sementsov-Ogievskiy int ret; 10040d93ed08SVladimir Sementsov-Ogievskiy QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes); 100561007b31SStefan Hajnoczi 100661007b31SStefan Hajnoczi if (bytes < 0) { 100761007b31SStefan Hajnoczi return -EINVAL; 100861007b31SStefan Hajnoczi } 100961007b31SStefan Hajnoczi 1010fae2681aSVladimir Sementsov-Ogievskiy ret = bdrv_pwritev(child, offset, bytes, &qiov, 0); 1011fae2681aSVladimir Sementsov-Ogievskiy 1012fae2681aSVladimir Sementsov-Ogievskiy return ret < 0 ? ret : bytes; 101361007b31SStefan Hajnoczi } 101461007b31SStefan Hajnoczi 101561007b31SStefan Hajnoczi /* 101661007b31SStefan Hajnoczi * Writes to the file and ensures that no writes are reordered across this 101761007b31SStefan Hajnoczi * request (acts as a barrier) 101861007b31SStefan Hajnoczi * 101961007b31SStefan Hajnoczi * Returns 0 on success, -errno in error cases. 102061007b31SStefan Hajnoczi */ 1021d9ca2ea2SKevin Wolf int bdrv_pwrite_sync(BdrvChild *child, int64_t offset, 102261007b31SStefan Hajnoczi const void *buf, int count) 102361007b31SStefan Hajnoczi { 102461007b31SStefan Hajnoczi int ret; 102561007b31SStefan Hajnoczi 1026d9ca2ea2SKevin Wolf ret = bdrv_pwrite(child, offset, buf, count); 102761007b31SStefan Hajnoczi if (ret < 0) { 102861007b31SStefan Hajnoczi return ret; 102961007b31SStefan Hajnoczi } 103061007b31SStefan Hajnoczi 1031d9ca2ea2SKevin Wolf ret = bdrv_flush(child->bs); 1032855a6a93SKevin Wolf if (ret < 0) { 1033855a6a93SKevin Wolf return ret; 103461007b31SStefan Hajnoczi } 103561007b31SStefan Hajnoczi 103661007b31SStefan Hajnoczi return 0; 103761007b31SStefan Hajnoczi } 103861007b31SStefan Hajnoczi 103908844473SKevin Wolf typedef struct CoroutineIOCompletion { 104008844473SKevin Wolf Coroutine *coroutine; 104108844473SKevin Wolf int ret; 104208844473SKevin Wolf } CoroutineIOCompletion; 104308844473SKevin Wolf 104408844473SKevin Wolf static void bdrv_co_io_em_complete(void *opaque, int ret) 104508844473SKevin Wolf { 104608844473SKevin Wolf CoroutineIOCompletion *co = opaque; 104708844473SKevin Wolf 104808844473SKevin Wolf co->ret = ret; 1049b9e413ddSPaolo Bonzini aio_co_wake(co->coroutine); 105008844473SKevin Wolf } 105108844473SKevin Wolf 1052166fe960SKevin Wolf static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs, 1053166fe960SKevin Wolf uint64_t offset, uint64_t bytes, 1054ac850bf0SVladimir Sementsov-Ogievskiy QEMUIOVector *qiov, 1055ac850bf0SVladimir Sementsov-Ogievskiy size_t qiov_offset, int flags) 1056166fe960SKevin Wolf { 1057166fe960SKevin Wolf BlockDriver *drv = bs->drv; 10583fb06697SKevin Wolf int64_t sector_num; 10593fb06697SKevin Wolf unsigned int nb_sectors; 1060ac850bf0SVladimir Sementsov-Ogievskiy QEMUIOVector local_qiov; 1061ac850bf0SVladimir Sementsov-Ogievskiy int ret; 10623fb06697SKevin Wolf 1063fa166538SEric Blake assert(!(flags & ~BDRV_REQ_MASK)); 1064fe0480d6SKevin Wolf assert(!(flags & BDRV_REQ_NO_FALLBACK)); 1065fa166538SEric Blake 1066d470ad42SMax Reitz if (!drv) { 1067d470ad42SMax Reitz return -ENOMEDIUM; 1068d470ad42SMax Reitz } 1069d470ad42SMax Reitz 1070ac850bf0SVladimir Sementsov-Ogievskiy if (drv->bdrv_co_preadv_part) { 1071ac850bf0SVladimir Sementsov-Ogievskiy return drv->bdrv_co_preadv_part(bs, offset, bytes, qiov, qiov_offset, 1072ac850bf0SVladimir Sementsov-Ogievskiy flags); 1073ac850bf0SVladimir Sementsov-Ogievskiy } 1074ac850bf0SVladimir Sementsov-Ogievskiy 1075ac850bf0SVladimir Sementsov-Ogievskiy if (qiov_offset > 0 || bytes != qiov->size) { 1076ac850bf0SVladimir Sementsov-Ogievskiy qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1077ac850bf0SVladimir Sementsov-Ogievskiy qiov = &local_qiov; 1078ac850bf0SVladimir Sementsov-Ogievskiy } 1079ac850bf0SVladimir Sementsov-Ogievskiy 10803fb06697SKevin Wolf if (drv->bdrv_co_preadv) { 1081ac850bf0SVladimir Sementsov-Ogievskiy ret = drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags); 1082ac850bf0SVladimir Sementsov-Ogievskiy goto out; 10833fb06697SKevin Wolf } 10843fb06697SKevin Wolf 1085edfab6a0SEric Blake if (drv->bdrv_aio_preadv) { 108608844473SKevin Wolf BlockAIOCB *acb; 108708844473SKevin Wolf CoroutineIOCompletion co = { 108808844473SKevin Wolf .coroutine = qemu_coroutine_self(), 108908844473SKevin Wolf }; 109008844473SKevin Wolf 1091e31f6864SEric Blake acb = drv->bdrv_aio_preadv(bs, offset, bytes, qiov, flags, 109208844473SKevin Wolf bdrv_co_io_em_complete, &co); 109308844473SKevin Wolf if (acb == NULL) { 1094ac850bf0SVladimir Sementsov-Ogievskiy ret = -EIO; 1095ac850bf0SVladimir Sementsov-Ogievskiy goto out; 109608844473SKevin Wolf } else { 109708844473SKevin Wolf qemu_coroutine_yield(); 1098ac850bf0SVladimir Sementsov-Ogievskiy ret = co.ret; 1099ac850bf0SVladimir Sementsov-Ogievskiy goto out; 110008844473SKevin Wolf } 110108844473SKevin Wolf } 1102edfab6a0SEric Blake 1103edfab6a0SEric Blake sector_num = offset >> BDRV_SECTOR_BITS; 1104edfab6a0SEric Blake nb_sectors = bytes >> BDRV_SECTOR_BITS; 1105edfab6a0SEric Blake 11061bbbf32dSNir Soffer assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE)); 11071bbbf32dSNir Soffer assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE)); 110841ae31e3SAlberto Garcia assert(bytes <= BDRV_REQUEST_MAX_BYTES); 1109edfab6a0SEric Blake assert(drv->bdrv_co_readv); 1110edfab6a0SEric Blake 1111ac850bf0SVladimir Sementsov-Ogievskiy ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov); 1112ac850bf0SVladimir Sementsov-Ogievskiy 1113ac850bf0SVladimir Sementsov-Ogievskiy out: 1114ac850bf0SVladimir Sementsov-Ogievskiy if (qiov == &local_qiov) { 1115ac850bf0SVladimir Sementsov-Ogievskiy qemu_iovec_destroy(&local_qiov); 1116ac850bf0SVladimir Sementsov-Ogievskiy } 1117ac850bf0SVladimir Sementsov-Ogievskiy 1118ac850bf0SVladimir Sementsov-Ogievskiy return ret; 1119166fe960SKevin Wolf } 1120166fe960SKevin Wolf 112178a07294SKevin Wolf static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs, 112278a07294SKevin Wolf uint64_t offset, uint64_t bytes, 1123ac850bf0SVladimir Sementsov-Ogievskiy QEMUIOVector *qiov, 1124ac850bf0SVladimir Sementsov-Ogievskiy size_t qiov_offset, int flags) 112578a07294SKevin Wolf { 112678a07294SKevin Wolf BlockDriver *drv = bs->drv; 11273fb06697SKevin Wolf int64_t sector_num; 11283fb06697SKevin Wolf unsigned int nb_sectors; 1129ac850bf0SVladimir Sementsov-Ogievskiy QEMUIOVector local_qiov; 113078a07294SKevin Wolf int ret; 113178a07294SKevin Wolf 1132fa166538SEric Blake assert(!(flags & ~BDRV_REQ_MASK)); 1133fe0480d6SKevin Wolf assert(!(flags & BDRV_REQ_NO_FALLBACK)); 1134fa166538SEric Blake 1135d470ad42SMax Reitz if (!drv) { 1136d470ad42SMax Reitz return -ENOMEDIUM; 1137d470ad42SMax Reitz } 1138d470ad42SMax Reitz 1139ac850bf0SVladimir Sementsov-Ogievskiy if (drv->bdrv_co_pwritev_part) { 1140ac850bf0SVladimir Sementsov-Ogievskiy ret = drv->bdrv_co_pwritev_part(bs, offset, bytes, qiov, qiov_offset, 1141ac850bf0SVladimir Sementsov-Ogievskiy flags & bs->supported_write_flags); 1142ac850bf0SVladimir Sementsov-Ogievskiy flags &= ~bs->supported_write_flags; 1143ac850bf0SVladimir Sementsov-Ogievskiy goto emulate_flags; 1144ac850bf0SVladimir Sementsov-Ogievskiy } 1145ac850bf0SVladimir Sementsov-Ogievskiy 1146ac850bf0SVladimir Sementsov-Ogievskiy if (qiov_offset > 0 || bytes != qiov->size) { 1147ac850bf0SVladimir Sementsov-Ogievskiy qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1148ac850bf0SVladimir Sementsov-Ogievskiy qiov = &local_qiov; 1149ac850bf0SVladimir Sementsov-Ogievskiy } 1150ac850bf0SVladimir Sementsov-Ogievskiy 11513fb06697SKevin Wolf if (drv->bdrv_co_pwritev) { 1152515c2f43SKevin Wolf ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, 1153515c2f43SKevin Wolf flags & bs->supported_write_flags); 1154515c2f43SKevin Wolf flags &= ~bs->supported_write_flags; 11553fb06697SKevin Wolf goto emulate_flags; 11563fb06697SKevin Wolf } 11573fb06697SKevin Wolf 1158edfab6a0SEric Blake if (drv->bdrv_aio_pwritev) { 115908844473SKevin Wolf BlockAIOCB *acb; 116008844473SKevin Wolf CoroutineIOCompletion co = { 116108844473SKevin Wolf .coroutine = qemu_coroutine_self(), 116208844473SKevin Wolf }; 116308844473SKevin Wolf 1164e31f6864SEric Blake acb = drv->bdrv_aio_pwritev(bs, offset, bytes, qiov, 1165e31f6864SEric Blake flags & bs->supported_write_flags, 116608844473SKevin Wolf bdrv_co_io_em_complete, &co); 1167e31f6864SEric Blake flags &= ~bs->supported_write_flags; 116808844473SKevin Wolf if (acb == NULL) { 11693fb06697SKevin Wolf ret = -EIO; 117008844473SKevin Wolf } else { 117108844473SKevin Wolf qemu_coroutine_yield(); 11723fb06697SKevin Wolf ret = co.ret; 117308844473SKevin Wolf } 1174edfab6a0SEric Blake goto emulate_flags; 1175edfab6a0SEric Blake } 1176edfab6a0SEric Blake 1177edfab6a0SEric Blake sector_num = offset >> BDRV_SECTOR_BITS; 1178edfab6a0SEric Blake nb_sectors = bytes >> BDRV_SECTOR_BITS; 1179edfab6a0SEric Blake 11801bbbf32dSNir Soffer assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE)); 11811bbbf32dSNir Soffer assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE)); 118241ae31e3SAlberto Garcia assert(bytes <= BDRV_REQUEST_MAX_BYTES); 1183edfab6a0SEric Blake 1184e18a58b4SEric Blake assert(drv->bdrv_co_writev); 1185e18a58b4SEric Blake ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov, 1186edfab6a0SEric Blake flags & bs->supported_write_flags); 1187edfab6a0SEric Blake flags &= ~bs->supported_write_flags; 118878a07294SKevin Wolf 11893fb06697SKevin Wolf emulate_flags: 11904df863f3SEric Blake if (ret == 0 && (flags & BDRV_REQ_FUA)) { 119178a07294SKevin Wolf ret = bdrv_co_flush(bs); 119278a07294SKevin Wolf } 119378a07294SKevin Wolf 1194ac850bf0SVladimir Sementsov-Ogievskiy if (qiov == &local_qiov) { 1195ac850bf0SVladimir Sementsov-Ogievskiy qemu_iovec_destroy(&local_qiov); 1196ac850bf0SVladimir Sementsov-Ogievskiy } 1197ac850bf0SVladimir Sementsov-Ogievskiy 119878a07294SKevin Wolf return ret; 119978a07294SKevin Wolf } 120078a07294SKevin Wolf 120129a298afSPavel Butsykin static int coroutine_fn 120229a298afSPavel Butsykin bdrv_driver_pwritev_compressed(BlockDriverState *bs, uint64_t offset, 1203ac850bf0SVladimir Sementsov-Ogievskiy uint64_t bytes, QEMUIOVector *qiov, 1204ac850bf0SVladimir Sementsov-Ogievskiy size_t qiov_offset) 120529a298afSPavel Butsykin { 120629a298afSPavel Butsykin BlockDriver *drv = bs->drv; 1207ac850bf0SVladimir Sementsov-Ogievskiy QEMUIOVector local_qiov; 1208ac850bf0SVladimir Sementsov-Ogievskiy int ret; 120929a298afSPavel Butsykin 1210d470ad42SMax Reitz if (!drv) { 1211d470ad42SMax Reitz return -ENOMEDIUM; 1212d470ad42SMax Reitz } 1213d470ad42SMax Reitz 1214ac850bf0SVladimir Sementsov-Ogievskiy if (!block_driver_can_compress(drv)) { 121529a298afSPavel Butsykin return -ENOTSUP; 121629a298afSPavel Butsykin } 121729a298afSPavel Butsykin 1218ac850bf0SVladimir Sementsov-Ogievskiy if (drv->bdrv_co_pwritev_compressed_part) { 1219ac850bf0SVladimir Sementsov-Ogievskiy return drv->bdrv_co_pwritev_compressed_part(bs, offset, bytes, 1220ac850bf0SVladimir Sementsov-Ogievskiy qiov, qiov_offset); 1221ac850bf0SVladimir Sementsov-Ogievskiy } 1222ac850bf0SVladimir Sementsov-Ogievskiy 1223ac850bf0SVladimir Sementsov-Ogievskiy if (qiov_offset == 0) { 122429a298afSPavel Butsykin return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov); 122529a298afSPavel Butsykin } 122629a298afSPavel Butsykin 1227ac850bf0SVladimir Sementsov-Ogievskiy qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1228ac850bf0SVladimir Sementsov-Ogievskiy ret = drv->bdrv_co_pwritev_compressed(bs, offset, bytes, &local_qiov); 1229ac850bf0SVladimir Sementsov-Ogievskiy qemu_iovec_destroy(&local_qiov); 1230ac850bf0SVladimir Sementsov-Ogievskiy 1231ac850bf0SVladimir Sementsov-Ogievskiy return ret; 1232ac850bf0SVladimir Sementsov-Ogievskiy } 1233ac850bf0SVladimir Sementsov-Ogievskiy 123485c97ca7SKevin Wolf static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child, 12353299e5ecSVladimir Sementsov-Ogievskiy int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 12361143ec5eSVladimir Sementsov-Ogievskiy size_t qiov_offset, int flags) 123761007b31SStefan Hajnoczi { 123885c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 123985c97ca7SKevin Wolf 124061007b31SStefan Hajnoczi /* Perform I/O through a temporary buffer so that users who scribble over 124161007b31SStefan Hajnoczi * their read buffer while the operation is in progress do not end up 124261007b31SStefan Hajnoczi * modifying the image file. This is critical for zero-copy guest I/O 124361007b31SStefan Hajnoczi * where anything might happen inside guest memory. 124461007b31SStefan Hajnoczi */ 12452275cc90SVladimir Sementsov-Ogievskiy void *bounce_buffer = NULL; 124661007b31SStefan Hajnoczi 124761007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 1248244483e6SKevin Wolf int64_t cluster_offset; 12497cfd5275SEric Blake int64_t cluster_bytes; 125061007b31SStefan Hajnoczi size_t skip_bytes; 125161007b31SStefan Hajnoczi int ret; 1252cb2e2878SEric Blake int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, 1253cb2e2878SEric Blake BDRV_REQUEST_MAX_BYTES); 1254cb2e2878SEric Blake unsigned int progress = 0; 12558644476eSMax Reitz bool skip_write; 125661007b31SStefan Hajnoczi 1257d470ad42SMax Reitz if (!drv) { 1258d470ad42SMax Reitz return -ENOMEDIUM; 1259d470ad42SMax Reitz } 1260d470ad42SMax Reitz 12618644476eSMax Reitz /* 12628644476eSMax Reitz * Do not write anything when the BDS is inactive. That is not 12638644476eSMax Reitz * allowed, and it would not help. 12648644476eSMax Reitz */ 12658644476eSMax Reitz skip_write = (bs->open_flags & BDRV_O_INACTIVE); 12668644476eSMax Reitz 12671bf03e66SKevin Wolf /* FIXME We cannot require callers to have write permissions when all they 12681bf03e66SKevin Wolf * are doing is a read request. If we did things right, write permissions 12691bf03e66SKevin Wolf * would be obtained anyway, but internally by the copy-on-read code. As 1270765d9df9SEric Blake * long as it is implemented here rather than in a separate filter driver, 12711bf03e66SKevin Wolf * the copy-on-read code doesn't have its own BdrvChild, however, for which 12721bf03e66SKevin Wolf * it could request permissions. Therefore we have to bypass the permission 12731bf03e66SKevin Wolf * system for the moment. */ 12741bf03e66SKevin Wolf // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE)); 1275afa4b293SKevin Wolf 127661007b31SStefan Hajnoczi /* Cover entire cluster so no additional backing file I/O is required when 1277cb2e2878SEric Blake * allocating cluster in the image file. Note that this value may exceed 1278cb2e2878SEric Blake * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which 1279cb2e2878SEric Blake * is one reason we loop rather than doing it all at once. 128061007b31SStefan Hajnoczi */ 1281244483e6SKevin Wolf bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes); 1282cb2e2878SEric Blake skip_bytes = offset - cluster_offset; 128361007b31SStefan Hajnoczi 1284244483e6SKevin Wolf trace_bdrv_co_do_copy_on_readv(bs, offset, bytes, 1285244483e6SKevin Wolf cluster_offset, cluster_bytes); 128661007b31SStefan Hajnoczi 1287cb2e2878SEric Blake while (cluster_bytes) { 1288cb2e2878SEric Blake int64_t pnum; 128961007b31SStefan Hajnoczi 12908644476eSMax Reitz if (skip_write) { 12918644476eSMax Reitz ret = 1; /* "already allocated", so nothing will be copied */ 12928644476eSMax Reitz pnum = MIN(cluster_bytes, max_transfer); 12938644476eSMax Reitz } else { 1294cb2e2878SEric Blake ret = bdrv_is_allocated(bs, cluster_offset, 1295cb2e2878SEric Blake MIN(cluster_bytes, max_transfer), &pnum); 1296cb2e2878SEric Blake if (ret < 0) { 12978644476eSMax Reitz /* 12988644476eSMax Reitz * Safe to treat errors in querying allocation as if 1299cb2e2878SEric Blake * unallocated; we'll probably fail again soon on the 1300cb2e2878SEric Blake * read, but at least that will set a decent errno. 1301cb2e2878SEric Blake */ 1302cb2e2878SEric Blake pnum = MIN(cluster_bytes, max_transfer); 1303cb2e2878SEric Blake } 1304cb2e2878SEric Blake 1305b0ddcbbbSKevin Wolf /* Stop at EOF if the image ends in the middle of the cluster */ 1306b0ddcbbbSKevin Wolf if (ret == 0 && pnum == 0) { 1307b0ddcbbbSKevin Wolf assert(progress >= bytes); 1308b0ddcbbbSKevin Wolf break; 1309b0ddcbbbSKevin Wolf } 1310b0ddcbbbSKevin Wolf 1311cb2e2878SEric Blake assert(skip_bytes < pnum); 13128644476eSMax Reitz } 1313cb2e2878SEric Blake 1314cb2e2878SEric Blake if (ret <= 0) { 13151143ec5eSVladimir Sementsov-Ogievskiy QEMUIOVector local_qiov; 13161143ec5eSVladimir Sementsov-Ogievskiy 1317cb2e2878SEric Blake /* Must copy-on-read; use the bounce buffer */ 13180d93ed08SVladimir Sementsov-Ogievskiy pnum = MIN(pnum, MAX_BOUNCE_BUFFER); 13192275cc90SVladimir Sementsov-Ogievskiy if (!bounce_buffer) { 13202275cc90SVladimir Sementsov-Ogievskiy int64_t max_we_need = MAX(pnum, cluster_bytes - pnum); 13212275cc90SVladimir Sementsov-Ogievskiy int64_t max_allowed = MIN(max_transfer, MAX_BOUNCE_BUFFER); 13222275cc90SVladimir Sementsov-Ogievskiy int64_t bounce_buffer_len = MIN(max_we_need, max_allowed); 13232275cc90SVladimir Sementsov-Ogievskiy 13242275cc90SVladimir Sementsov-Ogievskiy bounce_buffer = qemu_try_blockalign(bs, bounce_buffer_len); 13252275cc90SVladimir Sementsov-Ogievskiy if (!bounce_buffer) { 13262275cc90SVladimir Sementsov-Ogievskiy ret = -ENOMEM; 13272275cc90SVladimir Sementsov-Ogievskiy goto err; 13282275cc90SVladimir Sementsov-Ogievskiy } 13292275cc90SVladimir Sementsov-Ogievskiy } 13300d93ed08SVladimir Sementsov-Ogievskiy qemu_iovec_init_buf(&local_qiov, bounce_buffer, pnum); 1331cb2e2878SEric Blake 1332cb2e2878SEric Blake ret = bdrv_driver_preadv(bs, cluster_offset, pnum, 1333ac850bf0SVladimir Sementsov-Ogievskiy &local_qiov, 0, 0); 133461007b31SStefan Hajnoczi if (ret < 0) { 133561007b31SStefan Hajnoczi goto err; 133661007b31SStefan Hajnoczi } 133761007b31SStefan Hajnoczi 1338d855ebcdSEric Blake bdrv_debug_event(bs, BLKDBG_COR_WRITE); 1339c1499a5eSEric Blake if (drv->bdrv_co_pwrite_zeroes && 1340cb2e2878SEric Blake buffer_is_zero(bounce_buffer, pnum)) { 1341a604fa2bSEric Blake /* FIXME: Should we (perhaps conditionally) be setting 1342a604fa2bSEric Blake * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy 1343a604fa2bSEric Blake * that still correctly reads as zero? */ 13447adcf59fSMax Reitz ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, pnum, 13457adcf59fSMax Reitz BDRV_REQ_WRITE_UNCHANGED); 134661007b31SStefan Hajnoczi } else { 1347cb2e2878SEric Blake /* This does not change the data on the disk, it is not 1348cb2e2878SEric Blake * necessary to flush even in cache=writethrough mode. 134961007b31SStefan Hajnoczi */ 1350cb2e2878SEric Blake ret = bdrv_driver_pwritev(bs, cluster_offset, pnum, 1351ac850bf0SVladimir Sementsov-Ogievskiy &local_qiov, 0, 13527adcf59fSMax Reitz BDRV_REQ_WRITE_UNCHANGED); 135361007b31SStefan Hajnoczi } 135461007b31SStefan Hajnoczi 135561007b31SStefan Hajnoczi if (ret < 0) { 1356cb2e2878SEric Blake /* It might be okay to ignore write errors for guest 1357cb2e2878SEric Blake * requests. If this is a deliberate copy-on-read 1358cb2e2878SEric Blake * then we don't want to ignore the error. Simply 1359cb2e2878SEric Blake * report it in all cases. 136061007b31SStefan Hajnoczi */ 136161007b31SStefan Hajnoczi goto err; 136261007b31SStefan Hajnoczi } 136361007b31SStefan Hajnoczi 13643299e5ecSVladimir Sementsov-Ogievskiy if (!(flags & BDRV_REQ_PREFETCH)) { 13651143ec5eSVladimir Sementsov-Ogievskiy qemu_iovec_from_buf(qiov, qiov_offset + progress, 13661143ec5eSVladimir Sementsov-Ogievskiy bounce_buffer + skip_bytes, 13674ab78b19SVladimir Sementsov-Ogievskiy MIN(pnum - skip_bytes, bytes - progress)); 13683299e5ecSVladimir Sementsov-Ogievskiy } 13693299e5ecSVladimir Sementsov-Ogievskiy } else if (!(flags & BDRV_REQ_PREFETCH)) { 1370cb2e2878SEric Blake /* Read directly into the destination */ 13711143ec5eSVladimir Sementsov-Ogievskiy ret = bdrv_driver_preadv(bs, offset + progress, 13721143ec5eSVladimir Sementsov-Ogievskiy MIN(pnum - skip_bytes, bytes - progress), 13731143ec5eSVladimir Sementsov-Ogievskiy qiov, qiov_offset + progress, 0); 1374cb2e2878SEric Blake if (ret < 0) { 1375cb2e2878SEric Blake goto err; 1376cb2e2878SEric Blake } 1377cb2e2878SEric Blake } 1378cb2e2878SEric Blake 1379cb2e2878SEric Blake cluster_offset += pnum; 1380cb2e2878SEric Blake cluster_bytes -= pnum; 1381cb2e2878SEric Blake progress += pnum - skip_bytes; 1382cb2e2878SEric Blake skip_bytes = 0; 1383cb2e2878SEric Blake } 1384cb2e2878SEric Blake ret = 0; 138561007b31SStefan Hajnoczi 138661007b31SStefan Hajnoczi err: 138761007b31SStefan Hajnoczi qemu_vfree(bounce_buffer); 138861007b31SStefan Hajnoczi return ret; 138961007b31SStefan Hajnoczi } 139061007b31SStefan Hajnoczi 139161007b31SStefan Hajnoczi /* 139261007b31SStefan Hajnoczi * Forwards an already correctly aligned request to the BlockDriver. This 13931a62d0acSEric Blake * handles copy on read, zeroing after EOF, and fragmentation of large 13941a62d0acSEric Blake * reads; any other features must be implemented by the caller. 139561007b31SStefan Hajnoczi */ 139685c97ca7SKevin Wolf static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child, 139761007b31SStefan Hajnoczi BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 139865cd4424SVladimir Sementsov-Ogievskiy int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags) 139961007b31SStefan Hajnoczi { 140085c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 1401c9d20029SKevin Wolf int64_t total_bytes, max_bytes; 14021a62d0acSEric Blake int ret = 0; 14031a62d0acSEric Blake uint64_t bytes_remaining = bytes; 14041a62d0acSEric Blake int max_transfer; 140561007b31SStefan Hajnoczi 140649c07526SKevin Wolf assert(is_power_of_2(align)); 140749c07526SKevin Wolf assert((offset & (align - 1)) == 0); 140849c07526SKevin Wolf assert((bytes & (align - 1)) == 0); 1409abb06c5aSDaniel P. Berrange assert((bs->open_flags & BDRV_O_NO_IO) == 0); 14101a62d0acSEric Blake max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 14111a62d0acSEric Blake align); 1412a604fa2bSEric Blake 1413a604fa2bSEric Blake /* TODO: We would need a per-BDS .supported_read_flags and 1414a604fa2bSEric Blake * potential fallback support, if we ever implement any read flags 1415a604fa2bSEric Blake * to pass through to drivers. For now, there aren't any 1416a604fa2bSEric Blake * passthrough flags. */ 1417c53cb427SPaolo Bonzini assert(!(flags & ~(BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH))); 141861007b31SStefan Hajnoczi 141961007b31SStefan Hajnoczi /* Handle Copy on Read and associated serialisation */ 142061007b31SStefan Hajnoczi if (flags & BDRV_REQ_COPY_ON_READ) { 142161007b31SStefan Hajnoczi /* If we touch the same cluster it counts as an overlap. This 142261007b31SStefan Hajnoczi * guarantees that allocating writes will be serialized and not race 142361007b31SStefan Hajnoczi * with each other for the same cluster. For example, in copy-on-read 142461007b31SStefan Hajnoczi * it ensures that the CoR read and write operations are atomic and 142561007b31SStefan Hajnoczi * guest writes cannot interleave between them. */ 1426304d9d7fSMax Reitz bdrv_mark_request_serialising(req, bdrv_get_cluster_size(bs)); 142718fbd0deSPaolo Bonzini } else { 1428304d9d7fSMax Reitz bdrv_wait_serialising_requests(req); 142918fbd0deSPaolo Bonzini } 143061007b31SStefan Hajnoczi 143161007b31SStefan Hajnoczi if (flags & BDRV_REQ_COPY_ON_READ) { 1432d6a644bbSEric Blake int64_t pnum; 143361007b31SStefan Hajnoczi 143488e63df2SEric Blake ret = bdrv_is_allocated(bs, offset, bytes, &pnum); 143561007b31SStefan Hajnoczi if (ret < 0) { 143661007b31SStefan Hajnoczi goto out; 143761007b31SStefan Hajnoczi } 143861007b31SStefan Hajnoczi 143988e63df2SEric Blake if (!ret || pnum != bytes) { 144065cd4424SVladimir Sementsov-Ogievskiy ret = bdrv_co_do_copy_on_readv(child, offset, bytes, 144165cd4424SVladimir Sementsov-Ogievskiy qiov, qiov_offset, flags); 14423299e5ecSVladimir Sementsov-Ogievskiy goto out; 14433299e5ecSVladimir Sementsov-Ogievskiy } else if (flags & BDRV_REQ_PREFETCH) { 144461007b31SStefan Hajnoczi goto out; 144561007b31SStefan Hajnoczi } 144661007b31SStefan Hajnoczi } 144761007b31SStefan Hajnoczi 14481a62d0acSEric Blake /* Forward the request to the BlockDriver, possibly fragmenting it */ 144949c07526SKevin Wolf total_bytes = bdrv_getlength(bs); 145049c07526SKevin Wolf if (total_bytes < 0) { 145149c07526SKevin Wolf ret = total_bytes; 145261007b31SStefan Hajnoczi goto out; 145361007b31SStefan Hajnoczi } 145461007b31SStefan Hajnoczi 145549c07526SKevin Wolf max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align); 14561a62d0acSEric Blake if (bytes <= max_bytes && bytes <= max_transfer) { 145765cd4424SVladimir Sementsov-Ogievskiy ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, 0); 14581a62d0acSEric Blake goto out; 145961007b31SStefan Hajnoczi } 146061007b31SStefan Hajnoczi 14611a62d0acSEric Blake while (bytes_remaining) { 14621a62d0acSEric Blake int num; 14631a62d0acSEric Blake 14641a62d0acSEric Blake if (max_bytes) { 14651a62d0acSEric Blake num = MIN(bytes_remaining, MIN(max_bytes, max_transfer)); 14661a62d0acSEric Blake assert(num); 14671a62d0acSEric Blake 14681a62d0acSEric Blake ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining, 1469134b7decSMax Reitz num, qiov, 1470134b7decSMax Reitz qiov_offset + bytes - bytes_remaining, 0); 14711a62d0acSEric Blake max_bytes -= num; 14721a62d0acSEric Blake } else { 14731a62d0acSEric Blake num = bytes_remaining; 1474134b7decSMax Reitz ret = qemu_iovec_memset(qiov, qiov_offset + bytes - bytes_remaining, 1475134b7decSMax Reitz 0, bytes_remaining); 14761a62d0acSEric Blake } 14771a62d0acSEric Blake if (ret < 0) { 14781a62d0acSEric Blake goto out; 14791a62d0acSEric Blake } 14801a62d0acSEric Blake bytes_remaining -= num; 148161007b31SStefan Hajnoczi } 148261007b31SStefan Hajnoczi 148361007b31SStefan Hajnoczi out: 14841a62d0acSEric Blake return ret < 0 ? ret : 0; 148561007b31SStefan Hajnoczi } 148661007b31SStefan Hajnoczi 148761007b31SStefan Hajnoczi /* 14887a3f542fSVladimir Sementsov-Ogievskiy * Request padding 14897a3f542fSVladimir Sementsov-Ogievskiy * 14907a3f542fSVladimir Sementsov-Ogievskiy * |<---- align ----->| |<----- align ---->| 14917a3f542fSVladimir Sementsov-Ogievskiy * |<- head ->|<------------- bytes ------------->|<-- tail -->| 14927a3f542fSVladimir Sementsov-Ogievskiy * | | | | | | 14937a3f542fSVladimir Sementsov-Ogievskiy * -*----------$-------*-------- ... --------*-----$------------*--- 14947a3f542fSVladimir Sementsov-Ogievskiy * | | | | | | 14957a3f542fSVladimir Sementsov-Ogievskiy * | offset | | end | 14967a3f542fSVladimir Sementsov-Ogievskiy * ALIGN_DOWN(offset) ALIGN_UP(offset) ALIGN_DOWN(end) ALIGN_UP(end) 14977a3f542fSVladimir Sementsov-Ogievskiy * [buf ... ) [tail_buf ) 14987a3f542fSVladimir Sementsov-Ogievskiy * 14997a3f542fSVladimir Sementsov-Ogievskiy * @buf is an aligned allocation needed to store @head and @tail paddings. @head 15007a3f542fSVladimir Sementsov-Ogievskiy * is placed at the beginning of @buf and @tail at the @end. 15017a3f542fSVladimir Sementsov-Ogievskiy * 15027a3f542fSVladimir Sementsov-Ogievskiy * @tail_buf is a pointer to sub-buffer, corresponding to align-sized chunk 15037a3f542fSVladimir Sementsov-Ogievskiy * around tail, if tail exists. 15047a3f542fSVladimir Sementsov-Ogievskiy * 15057a3f542fSVladimir Sementsov-Ogievskiy * @merge_reads is true for small requests, 15067a3f542fSVladimir Sementsov-Ogievskiy * if @buf_len == @head + bytes + @tail. In this case it is possible that both 15077a3f542fSVladimir Sementsov-Ogievskiy * head and tail exist but @buf_len == align and @tail_buf == @buf. 150861007b31SStefan Hajnoczi */ 15097a3f542fSVladimir Sementsov-Ogievskiy typedef struct BdrvRequestPadding { 15107a3f542fSVladimir Sementsov-Ogievskiy uint8_t *buf; 15117a3f542fSVladimir Sementsov-Ogievskiy size_t buf_len; 15127a3f542fSVladimir Sementsov-Ogievskiy uint8_t *tail_buf; 15137a3f542fSVladimir Sementsov-Ogievskiy size_t head; 15147a3f542fSVladimir Sementsov-Ogievskiy size_t tail; 15157a3f542fSVladimir Sementsov-Ogievskiy bool merge_reads; 15167a3f542fSVladimir Sementsov-Ogievskiy QEMUIOVector local_qiov; 15177a3f542fSVladimir Sementsov-Ogievskiy } BdrvRequestPadding; 15187a3f542fSVladimir Sementsov-Ogievskiy 15197a3f542fSVladimir Sementsov-Ogievskiy static bool bdrv_init_padding(BlockDriverState *bs, 15207a3f542fSVladimir Sementsov-Ogievskiy int64_t offset, int64_t bytes, 15217a3f542fSVladimir Sementsov-Ogievskiy BdrvRequestPadding *pad) 15227a3f542fSVladimir Sementsov-Ogievskiy { 15237a3f542fSVladimir Sementsov-Ogievskiy uint64_t align = bs->bl.request_alignment; 15247a3f542fSVladimir Sementsov-Ogievskiy size_t sum; 15257a3f542fSVladimir Sementsov-Ogievskiy 15267a3f542fSVladimir Sementsov-Ogievskiy memset(pad, 0, sizeof(*pad)); 15277a3f542fSVladimir Sementsov-Ogievskiy 15287a3f542fSVladimir Sementsov-Ogievskiy pad->head = offset & (align - 1); 15297a3f542fSVladimir Sementsov-Ogievskiy pad->tail = ((offset + bytes) & (align - 1)); 15307a3f542fSVladimir Sementsov-Ogievskiy if (pad->tail) { 15317a3f542fSVladimir Sementsov-Ogievskiy pad->tail = align - pad->tail; 15327a3f542fSVladimir Sementsov-Ogievskiy } 15337a3f542fSVladimir Sementsov-Ogievskiy 1534ac9d00bfSVladimir Sementsov-Ogievskiy if (!pad->head && !pad->tail) { 15357a3f542fSVladimir Sementsov-Ogievskiy return false; 15367a3f542fSVladimir Sementsov-Ogievskiy } 15377a3f542fSVladimir Sementsov-Ogievskiy 1538ac9d00bfSVladimir Sementsov-Ogievskiy assert(bytes); /* Nothing good in aligning zero-length requests */ 1539ac9d00bfSVladimir Sementsov-Ogievskiy 15407a3f542fSVladimir Sementsov-Ogievskiy sum = pad->head + bytes + pad->tail; 15417a3f542fSVladimir Sementsov-Ogievskiy pad->buf_len = (sum > align && pad->head && pad->tail) ? 2 * align : align; 15427a3f542fSVladimir Sementsov-Ogievskiy pad->buf = qemu_blockalign(bs, pad->buf_len); 15437a3f542fSVladimir Sementsov-Ogievskiy pad->merge_reads = sum == pad->buf_len; 15447a3f542fSVladimir Sementsov-Ogievskiy if (pad->tail) { 15457a3f542fSVladimir Sementsov-Ogievskiy pad->tail_buf = pad->buf + pad->buf_len - align; 15467a3f542fSVladimir Sementsov-Ogievskiy } 15477a3f542fSVladimir Sementsov-Ogievskiy 15487a3f542fSVladimir Sementsov-Ogievskiy return true; 15497a3f542fSVladimir Sementsov-Ogievskiy } 15507a3f542fSVladimir Sementsov-Ogievskiy 15517a3f542fSVladimir Sementsov-Ogievskiy static int bdrv_padding_rmw_read(BdrvChild *child, 15527a3f542fSVladimir Sementsov-Ogievskiy BdrvTrackedRequest *req, 15537a3f542fSVladimir Sementsov-Ogievskiy BdrvRequestPadding *pad, 15547a3f542fSVladimir Sementsov-Ogievskiy bool zero_middle) 15557a3f542fSVladimir Sementsov-Ogievskiy { 15567a3f542fSVladimir Sementsov-Ogievskiy QEMUIOVector local_qiov; 15577a3f542fSVladimir Sementsov-Ogievskiy BlockDriverState *bs = child->bs; 15587a3f542fSVladimir Sementsov-Ogievskiy uint64_t align = bs->bl.request_alignment; 15597a3f542fSVladimir Sementsov-Ogievskiy int ret; 15607a3f542fSVladimir Sementsov-Ogievskiy 15617a3f542fSVladimir Sementsov-Ogievskiy assert(req->serialising && pad->buf); 15627a3f542fSVladimir Sementsov-Ogievskiy 15637a3f542fSVladimir Sementsov-Ogievskiy if (pad->head || pad->merge_reads) { 15647a3f542fSVladimir Sementsov-Ogievskiy uint64_t bytes = pad->merge_reads ? pad->buf_len : align; 15657a3f542fSVladimir Sementsov-Ogievskiy 15667a3f542fSVladimir Sementsov-Ogievskiy qemu_iovec_init_buf(&local_qiov, pad->buf, bytes); 15677a3f542fSVladimir Sementsov-Ogievskiy 15687a3f542fSVladimir Sementsov-Ogievskiy if (pad->head) { 15697a3f542fSVladimir Sementsov-Ogievskiy bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 15707a3f542fSVladimir Sementsov-Ogievskiy } 15717a3f542fSVladimir Sementsov-Ogievskiy if (pad->merge_reads && pad->tail) { 15727a3f542fSVladimir Sementsov-Ogievskiy bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 15737a3f542fSVladimir Sementsov-Ogievskiy } 15747a3f542fSVladimir Sementsov-Ogievskiy ret = bdrv_aligned_preadv(child, req, req->overlap_offset, bytes, 157565cd4424SVladimir Sementsov-Ogievskiy align, &local_qiov, 0, 0); 15767a3f542fSVladimir Sementsov-Ogievskiy if (ret < 0) { 15777a3f542fSVladimir Sementsov-Ogievskiy return ret; 15787a3f542fSVladimir Sementsov-Ogievskiy } 15797a3f542fSVladimir Sementsov-Ogievskiy if (pad->head) { 15807a3f542fSVladimir Sementsov-Ogievskiy bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 15817a3f542fSVladimir Sementsov-Ogievskiy } 15827a3f542fSVladimir Sementsov-Ogievskiy if (pad->merge_reads && pad->tail) { 15837a3f542fSVladimir Sementsov-Ogievskiy bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 15847a3f542fSVladimir Sementsov-Ogievskiy } 15857a3f542fSVladimir Sementsov-Ogievskiy 15867a3f542fSVladimir Sementsov-Ogievskiy if (pad->merge_reads) { 15877a3f542fSVladimir Sementsov-Ogievskiy goto zero_mem; 15887a3f542fSVladimir Sementsov-Ogievskiy } 15897a3f542fSVladimir Sementsov-Ogievskiy } 15907a3f542fSVladimir Sementsov-Ogievskiy 15917a3f542fSVladimir Sementsov-Ogievskiy if (pad->tail) { 15927a3f542fSVladimir Sementsov-Ogievskiy qemu_iovec_init_buf(&local_qiov, pad->tail_buf, align); 15937a3f542fSVladimir Sementsov-Ogievskiy 15947a3f542fSVladimir Sementsov-Ogievskiy bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 15957a3f542fSVladimir Sementsov-Ogievskiy ret = bdrv_aligned_preadv( 15967a3f542fSVladimir Sementsov-Ogievskiy child, req, 15977a3f542fSVladimir Sementsov-Ogievskiy req->overlap_offset + req->overlap_bytes - align, 159865cd4424SVladimir Sementsov-Ogievskiy align, align, &local_qiov, 0, 0); 15997a3f542fSVladimir Sementsov-Ogievskiy if (ret < 0) { 16007a3f542fSVladimir Sementsov-Ogievskiy return ret; 16017a3f542fSVladimir Sementsov-Ogievskiy } 16027a3f542fSVladimir Sementsov-Ogievskiy bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 16037a3f542fSVladimir Sementsov-Ogievskiy } 16047a3f542fSVladimir Sementsov-Ogievskiy 16057a3f542fSVladimir Sementsov-Ogievskiy zero_mem: 16067a3f542fSVladimir Sementsov-Ogievskiy if (zero_middle) { 16077a3f542fSVladimir Sementsov-Ogievskiy memset(pad->buf + pad->head, 0, pad->buf_len - pad->head - pad->tail); 16087a3f542fSVladimir Sementsov-Ogievskiy } 16097a3f542fSVladimir Sementsov-Ogievskiy 16107a3f542fSVladimir Sementsov-Ogievskiy return 0; 16117a3f542fSVladimir Sementsov-Ogievskiy } 16127a3f542fSVladimir Sementsov-Ogievskiy 16137a3f542fSVladimir Sementsov-Ogievskiy static void bdrv_padding_destroy(BdrvRequestPadding *pad) 16147a3f542fSVladimir Sementsov-Ogievskiy { 16157a3f542fSVladimir Sementsov-Ogievskiy if (pad->buf) { 16167a3f542fSVladimir Sementsov-Ogievskiy qemu_vfree(pad->buf); 16177a3f542fSVladimir Sementsov-Ogievskiy qemu_iovec_destroy(&pad->local_qiov); 16187a3f542fSVladimir Sementsov-Ogievskiy } 16197a3f542fSVladimir Sementsov-Ogievskiy } 16207a3f542fSVladimir Sementsov-Ogievskiy 16217a3f542fSVladimir Sementsov-Ogievskiy /* 16227a3f542fSVladimir Sementsov-Ogievskiy * bdrv_pad_request 16237a3f542fSVladimir Sementsov-Ogievskiy * 16247a3f542fSVladimir Sementsov-Ogievskiy * Exchange request parameters with padded request if needed. Don't include RMW 16257a3f542fSVladimir Sementsov-Ogievskiy * read of padding, bdrv_padding_rmw_read() should be called separately if 16267a3f542fSVladimir Sementsov-Ogievskiy * needed. 16277a3f542fSVladimir Sementsov-Ogievskiy * 16287a3f542fSVladimir Sementsov-Ogievskiy * All parameters except @bs are in-out: they represent original request at 16297a3f542fSVladimir Sementsov-Ogievskiy * function call and padded (if padding needed) at function finish. 16307a3f542fSVladimir Sementsov-Ogievskiy * 16317a3f542fSVladimir Sementsov-Ogievskiy * Function always succeeds. 16327a3f542fSVladimir Sementsov-Ogievskiy */ 16331acc3466SVladimir Sementsov-Ogievskiy static bool bdrv_pad_request(BlockDriverState *bs, 16341acc3466SVladimir Sementsov-Ogievskiy QEMUIOVector **qiov, size_t *qiov_offset, 16357a3f542fSVladimir Sementsov-Ogievskiy int64_t *offset, unsigned int *bytes, 16367a3f542fSVladimir Sementsov-Ogievskiy BdrvRequestPadding *pad) 16377a3f542fSVladimir Sementsov-Ogievskiy { 16387a3f542fSVladimir Sementsov-Ogievskiy if (!bdrv_init_padding(bs, *offset, *bytes, pad)) { 16397a3f542fSVladimir Sementsov-Ogievskiy return false; 16407a3f542fSVladimir Sementsov-Ogievskiy } 16417a3f542fSVladimir Sementsov-Ogievskiy 16427a3f542fSVladimir Sementsov-Ogievskiy qemu_iovec_init_extended(&pad->local_qiov, pad->buf, pad->head, 16431acc3466SVladimir Sementsov-Ogievskiy *qiov, *qiov_offset, *bytes, 16447a3f542fSVladimir Sementsov-Ogievskiy pad->buf + pad->buf_len - pad->tail, pad->tail); 16457a3f542fSVladimir Sementsov-Ogievskiy *bytes += pad->head + pad->tail; 16467a3f542fSVladimir Sementsov-Ogievskiy *offset -= pad->head; 16477a3f542fSVladimir Sementsov-Ogievskiy *qiov = &pad->local_qiov; 16481acc3466SVladimir Sementsov-Ogievskiy *qiov_offset = 0; 16497a3f542fSVladimir Sementsov-Ogievskiy 16507a3f542fSVladimir Sementsov-Ogievskiy return true; 16517a3f542fSVladimir Sementsov-Ogievskiy } 16527a3f542fSVladimir Sementsov-Ogievskiy 1653a03ef88fSKevin Wolf int coroutine_fn bdrv_co_preadv(BdrvChild *child, 165461007b31SStefan Hajnoczi int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 165561007b31SStefan Hajnoczi BdrvRequestFlags flags) 165661007b31SStefan Hajnoczi { 16571acc3466SVladimir Sementsov-Ogievskiy return bdrv_co_preadv_part(child, offset, bytes, qiov, 0, flags); 16581acc3466SVladimir Sementsov-Ogievskiy } 16591acc3466SVladimir Sementsov-Ogievskiy 16601acc3466SVladimir Sementsov-Ogievskiy int coroutine_fn bdrv_co_preadv_part(BdrvChild *child, 16611acc3466SVladimir Sementsov-Ogievskiy int64_t offset, unsigned int bytes, 16621acc3466SVladimir Sementsov-Ogievskiy QEMUIOVector *qiov, size_t qiov_offset, 16631acc3466SVladimir Sementsov-Ogievskiy BdrvRequestFlags flags) 16641acc3466SVladimir Sementsov-Ogievskiy { 1665a03ef88fSKevin Wolf BlockDriverState *bs = child->bs; 166661007b31SStefan Hajnoczi BdrvTrackedRequest req; 16677a3f542fSVladimir Sementsov-Ogievskiy BdrvRequestPadding pad; 166861007b31SStefan Hajnoczi int ret; 166961007b31SStefan Hajnoczi 16707a3f542fSVladimir Sementsov-Ogievskiy trace_bdrv_co_preadv(bs, offset, bytes, flags); 167161007b31SStefan Hajnoczi 1672f4dad307SVladimir Sementsov-Ogievskiy if (!bdrv_is_inserted(bs)) { 1673f4dad307SVladimir Sementsov-Ogievskiy return -ENOMEDIUM; 1674f4dad307SVladimir Sementsov-Ogievskiy } 1675f4dad307SVladimir Sementsov-Ogievskiy 16768b117001SVladimir Sementsov-Ogievskiy ret = bdrv_check_request32(offset, bytes); 167761007b31SStefan Hajnoczi if (ret < 0) { 167861007b31SStefan Hajnoczi return ret; 167961007b31SStefan Hajnoczi } 168061007b31SStefan Hajnoczi 1681ac9d00bfSVladimir Sementsov-Ogievskiy if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) { 1682ac9d00bfSVladimir Sementsov-Ogievskiy /* 1683ac9d00bfSVladimir Sementsov-Ogievskiy * Aligning zero request is nonsense. Even if driver has special meaning 1684ac9d00bfSVladimir Sementsov-Ogievskiy * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass 1685ac9d00bfSVladimir Sementsov-Ogievskiy * it to driver due to request_alignment. 1686ac9d00bfSVladimir Sementsov-Ogievskiy * 1687ac9d00bfSVladimir Sementsov-Ogievskiy * Still, no reason to return an error if someone do unaligned 1688ac9d00bfSVladimir Sementsov-Ogievskiy * zero-length read occasionally. 1689ac9d00bfSVladimir Sementsov-Ogievskiy */ 1690ac9d00bfSVladimir Sementsov-Ogievskiy return 0; 1691ac9d00bfSVladimir Sementsov-Ogievskiy } 1692ac9d00bfSVladimir Sementsov-Ogievskiy 169399723548SPaolo Bonzini bdrv_inc_in_flight(bs); 169499723548SPaolo Bonzini 16959568b511SWen Congyang /* Don't do copy-on-read if we read data before write operation */ 1696d73415a3SStefan Hajnoczi if (qatomic_read(&bs->copy_on_read)) { 169761007b31SStefan Hajnoczi flags |= BDRV_REQ_COPY_ON_READ; 169861007b31SStefan Hajnoczi } 169961007b31SStefan Hajnoczi 17001acc3466SVladimir Sementsov-Ogievskiy bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad); 170161007b31SStefan Hajnoczi 1702ebde595cSFam Zheng tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ); 17037a3f542fSVladimir Sementsov-Ogievskiy ret = bdrv_aligned_preadv(child, &req, offset, bytes, 17047a3f542fSVladimir Sementsov-Ogievskiy bs->bl.request_alignment, 17051acc3466SVladimir Sementsov-Ogievskiy qiov, qiov_offset, flags); 170661007b31SStefan Hajnoczi tracked_request_end(&req); 170799723548SPaolo Bonzini bdrv_dec_in_flight(bs); 170861007b31SStefan Hajnoczi 17097a3f542fSVladimir Sementsov-Ogievskiy bdrv_padding_destroy(&pad); 171061007b31SStefan Hajnoczi 171161007b31SStefan Hajnoczi return ret; 171261007b31SStefan Hajnoczi } 171361007b31SStefan Hajnoczi 1714d05aa8bbSEric Blake static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 1715f5a5ca79SManos Pitsidianakis int64_t offset, int bytes, BdrvRequestFlags flags) 171661007b31SStefan Hajnoczi { 171761007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 171861007b31SStefan Hajnoczi QEMUIOVector qiov; 17190d93ed08SVladimir Sementsov-Ogievskiy void *buf = NULL; 172061007b31SStefan Hajnoczi int ret = 0; 1721465fe887SEric Blake bool need_flush = false; 1722443668caSDenis V. Lunev int head = 0; 1723443668caSDenis V. Lunev int tail = 0; 172461007b31SStefan Hajnoczi 1725cf081fcaSEric Blake int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, INT_MAX); 1726a5b8dd2cSEric Blake int alignment = MAX(bs->bl.pwrite_zeroes_alignment, 1727a5b8dd2cSEric Blake bs->bl.request_alignment); 1728cb2e2878SEric Blake int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER); 1729cf081fcaSEric Blake 1730d470ad42SMax Reitz if (!drv) { 1731d470ad42SMax Reitz return -ENOMEDIUM; 1732d470ad42SMax Reitz } 1733d470ad42SMax Reitz 1734fe0480d6SKevin Wolf if ((flags & ~bs->supported_zero_flags) & BDRV_REQ_NO_FALLBACK) { 1735fe0480d6SKevin Wolf return -ENOTSUP; 1736fe0480d6SKevin Wolf } 1737fe0480d6SKevin Wolf 1738b8d0a980SEric Blake assert(alignment % bs->bl.request_alignment == 0); 1739b8d0a980SEric Blake head = offset % alignment; 1740f5a5ca79SManos Pitsidianakis tail = (offset + bytes) % alignment; 1741b8d0a980SEric Blake max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment); 1742b8d0a980SEric Blake assert(max_write_zeroes >= bs->bl.request_alignment); 174361007b31SStefan Hajnoczi 1744f5a5ca79SManos Pitsidianakis while (bytes > 0 && !ret) { 1745f5a5ca79SManos Pitsidianakis int num = bytes; 174661007b31SStefan Hajnoczi 174761007b31SStefan Hajnoczi /* Align request. Block drivers can expect the "bulk" of the request 1748443668caSDenis V. Lunev * to be aligned, and that unaligned requests do not cross cluster 1749443668caSDenis V. Lunev * boundaries. 175061007b31SStefan Hajnoczi */ 1751443668caSDenis V. Lunev if (head) { 1752b2f95feeSEric Blake /* Make a small request up to the first aligned sector. For 1753b2f95feeSEric Blake * convenience, limit this request to max_transfer even if 1754b2f95feeSEric Blake * we don't need to fall back to writes. */ 1755f5a5ca79SManos Pitsidianakis num = MIN(MIN(bytes, max_transfer), alignment - head); 1756b2f95feeSEric Blake head = (head + num) % alignment; 1757b2f95feeSEric Blake assert(num < max_write_zeroes); 1758d05aa8bbSEric Blake } else if (tail && num > alignment) { 1759443668caSDenis V. Lunev /* Shorten the request to the last aligned sector. */ 1760443668caSDenis V. Lunev num -= tail; 176161007b31SStefan Hajnoczi } 176261007b31SStefan Hajnoczi 176361007b31SStefan Hajnoczi /* limit request size */ 176461007b31SStefan Hajnoczi if (num > max_write_zeroes) { 176561007b31SStefan Hajnoczi num = max_write_zeroes; 176661007b31SStefan Hajnoczi } 176761007b31SStefan Hajnoczi 176861007b31SStefan Hajnoczi ret = -ENOTSUP; 176961007b31SStefan Hajnoczi /* First try the efficient write zeroes operation */ 1770d05aa8bbSEric Blake if (drv->bdrv_co_pwrite_zeroes) { 1771d05aa8bbSEric Blake ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num, 1772d05aa8bbSEric Blake flags & bs->supported_zero_flags); 1773d05aa8bbSEric Blake if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) && 1774d05aa8bbSEric Blake !(bs->supported_zero_flags & BDRV_REQ_FUA)) { 1775d05aa8bbSEric Blake need_flush = true; 1776d05aa8bbSEric Blake } 1777465fe887SEric Blake } else { 1778465fe887SEric Blake assert(!bs->supported_zero_flags); 177961007b31SStefan Hajnoczi } 178061007b31SStefan Hajnoczi 1781294682ccSAndrey Shinkevich if (ret == -ENOTSUP && !(flags & BDRV_REQ_NO_FALLBACK)) { 178261007b31SStefan Hajnoczi /* Fall back to bounce buffer if write zeroes is unsupported */ 1783465fe887SEric Blake BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE; 1784465fe887SEric Blake 1785465fe887SEric Blake if ((flags & BDRV_REQ_FUA) && 1786465fe887SEric Blake !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1787465fe887SEric Blake /* No need for bdrv_driver_pwrite() to do a fallback 1788465fe887SEric Blake * flush on each chunk; use just one at the end */ 1789465fe887SEric Blake write_flags &= ~BDRV_REQ_FUA; 1790465fe887SEric Blake need_flush = true; 1791465fe887SEric Blake } 17925def6b80SEric Blake num = MIN(num, max_transfer); 17930d93ed08SVladimir Sementsov-Ogievskiy if (buf == NULL) { 17940d93ed08SVladimir Sementsov-Ogievskiy buf = qemu_try_blockalign0(bs, num); 17950d93ed08SVladimir Sementsov-Ogievskiy if (buf == NULL) { 179661007b31SStefan Hajnoczi ret = -ENOMEM; 179761007b31SStefan Hajnoczi goto fail; 179861007b31SStefan Hajnoczi } 179961007b31SStefan Hajnoczi } 18000d93ed08SVladimir Sementsov-Ogievskiy qemu_iovec_init_buf(&qiov, buf, num); 180161007b31SStefan Hajnoczi 1802ac850bf0SVladimir Sementsov-Ogievskiy ret = bdrv_driver_pwritev(bs, offset, num, &qiov, 0, write_flags); 180361007b31SStefan Hajnoczi 180461007b31SStefan Hajnoczi /* Keep bounce buffer around if it is big enough for all 180561007b31SStefan Hajnoczi * all future requests. 180661007b31SStefan Hajnoczi */ 18075def6b80SEric Blake if (num < max_transfer) { 18080d93ed08SVladimir Sementsov-Ogievskiy qemu_vfree(buf); 18090d93ed08SVladimir Sementsov-Ogievskiy buf = NULL; 181061007b31SStefan Hajnoczi } 181161007b31SStefan Hajnoczi } 181261007b31SStefan Hajnoczi 1813d05aa8bbSEric Blake offset += num; 1814f5a5ca79SManos Pitsidianakis bytes -= num; 181561007b31SStefan Hajnoczi } 181661007b31SStefan Hajnoczi 181761007b31SStefan Hajnoczi fail: 1818465fe887SEric Blake if (ret == 0 && need_flush) { 1819465fe887SEric Blake ret = bdrv_co_flush(bs); 1820465fe887SEric Blake } 18210d93ed08SVladimir Sementsov-Ogievskiy qemu_vfree(buf); 182261007b31SStefan Hajnoczi return ret; 182361007b31SStefan Hajnoczi } 182461007b31SStefan Hajnoczi 182585fe2479SFam Zheng static inline int coroutine_fn 182685fe2479SFam Zheng bdrv_co_write_req_prepare(BdrvChild *child, int64_t offset, uint64_t bytes, 182785fe2479SFam Zheng BdrvTrackedRequest *req, int flags) 182885fe2479SFam Zheng { 182985fe2479SFam Zheng BlockDriverState *bs = child->bs; 183085fe2479SFam Zheng bool waited; 183185fe2479SFam Zheng int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 183285fe2479SFam Zheng 183385fe2479SFam Zheng if (bs->read_only) { 183485fe2479SFam Zheng return -EPERM; 183585fe2479SFam Zheng } 183685fe2479SFam Zheng 183785fe2479SFam Zheng assert(!(bs->open_flags & BDRV_O_INACTIVE)); 183885fe2479SFam Zheng assert((bs->open_flags & BDRV_O_NO_IO) == 0); 183985fe2479SFam Zheng assert(!(flags & ~BDRV_REQ_MASK)); 184085fe2479SFam Zheng 184185fe2479SFam Zheng if (flags & BDRV_REQ_SERIALISING) { 184218fbd0deSPaolo Bonzini waited = bdrv_mark_request_serialising(req, bdrv_get_cluster_size(bs)); 184318fbd0deSPaolo Bonzini /* 184418fbd0deSPaolo Bonzini * For a misaligned request we should have already waited earlier, 184518fbd0deSPaolo Bonzini * because we come after bdrv_padding_rmw_read which must be called 184618fbd0deSPaolo Bonzini * with the request already marked as serialising. 184718fbd0deSPaolo Bonzini */ 184818fbd0deSPaolo Bonzini assert(!waited || 184918fbd0deSPaolo Bonzini (req->offset == req->overlap_offset && 185018fbd0deSPaolo Bonzini req->bytes == req->overlap_bytes)); 185118fbd0deSPaolo Bonzini } else { 185218fbd0deSPaolo Bonzini bdrv_wait_serialising_requests(req); 185385fe2479SFam Zheng } 185485fe2479SFam Zheng 185585fe2479SFam Zheng assert(req->overlap_offset <= offset); 185685fe2479SFam Zheng assert(offset + bytes <= req->overlap_offset + req->overlap_bytes); 1857cd47d792SFam Zheng assert(end_sector <= bs->total_sectors || child->perm & BLK_PERM_RESIZE); 185885fe2479SFam Zheng 1859cd47d792SFam Zheng switch (req->type) { 1860cd47d792SFam Zheng case BDRV_TRACKED_WRITE: 1861cd47d792SFam Zheng case BDRV_TRACKED_DISCARD: 186285fe2479SFam Zheng if (flags & BDRV_REQ_WRITE_UNCHANGED) { 186385fe2479SFam Zheng assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE)); 186485fe2479SFam Zheng } else { 186585fe2479SFam Zheng assert(child->perm & BLK_PERM_WRITE); 186685fe2479SFam Zheng } 1867cd47d792SFam Zheng return notifier_with_return_list_notify(&bs->before_write_notifiers, 1868cd47d792SFam Zheng req); 1869cd47d792SFam Zheng case BDRV_TRACKED_TRUNCATE: 1870cd47d792SFam Zheng assert(child->perm & BLK_PERM_RESIZE); 1871cd47d792SFam Zheng return 0; 1872cd47d792SFam Zheng default: 1873cd47d792SFam Zheng abort(); 1874cd47d792SFam Zheng } 187585fe2479SFam Zheng } 187685fe2479SFam Zheng 187785fe2479SFam Zheng static inline void coroutine_fn 187885fe2479SFam Zheng bdrv_co_write_req_finish(BdrvChild *child, int64_t offset, uint64_t bytes, 187985fe2479SFam Zheng BdrvTrackedRequest *req, int ret) 188085fe2479SFam Zheng { 188185fe2479SFam Zheng int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 188285fe2479SFam Zheng BlockDriverState *bs = child->bs; 188385fe2479SFam Zheng 1884d73415a3SStefan Hajnoczi qatomic_inc(&bs->write_gen); 188585fe2479SFam Zheng 188600695c27SFam Zheng /* 188700695c27SFam Zheng * Discard cannot extend the image, but in error handling cases, such as 188800695c27SFam Zheng * when reverting a qcow2 cluster allocation, the discarded range can pass 188900695c27SFam Zheng * the end of image file, so we cannot assert about BDRV_TRACKED_DISCARD 189000695c27SFam Zheng * here. Instead, just skip it, since semantically a discard request 189100695c27SFam Zheng * beyond EOF cannot expand the image anyway. 189200695c27SFam Zheng */ 18937f8f03efSFam Zheng if (ret == 0 && 1894cd47d792SFam Zheng (req->type == BDRV_TRACKED_TRUNCATE || 1895cd47d792SFam Zheng end_sector > bs->total_sectors) && 189600695c27SFam Zheng req->type != BDRV_TRACKED_DISCARD) { 18977f8f03efSFam Zheng bs->total_sectors = end_sector; 18987f8f03efSFam Zheng bdrv_parent_cb_resize(bs); 18997f8f03efSFam Zheng bdrv_dirty_bitmap_truncate(bs, end_sector << BDRV_SECTOR_BITS); 190085fe2479SFam Zheng } 190100695c27SFam Zheng if (req->bytes) { 190200695c27SFam Zheng switch (req->type) { 190300695c27SFam Zheng case BDRV_TRACKED_WRITE: 190400695c27SFam Zheng stat64_max(&bs->wr_highest_offset, offset + bytes); 190500695c27SFam Zheng /* fall through, to set dirty bits */ 190600695c27SFam Zheng case BDRV_TRACKED_DISCARD: 19077f8f03efSFam Zheng bdrv_set_dirty(bs, offset, bytes); 190800695c27SFam Zheng break; 190900695c27SFam Zheng default: 191000695c27SFam Zheng break; 191100695c27SFam Zheng } 191200695c27SFam Zheng } 191385fe2479SFam Zheng } 191485fe2479SFam Zheng 191561007b31SStefan Hajnoczi /* 191604ed95f4SEric Blake * Forwards an already correctly aligned write request to the BlockDriver, 191704ed95f4SEric Blake * after possibly fragmenting it. 191861007b31SStefan Hajnoczi */ 191985c97ca7SKevin Wolf static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child, 192061007b31SStefan Hajnoczi BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 192128c4da28SVladimir Sementsov-Ogievskiy int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags) 192261007b31SStefan Hajnoczi { 192385c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 192461007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 192561007b31SStefan Hajnoczi int ret; 192661007b31SStefan Hajnoczi 192704ed95f4SEric Blake uint64_t bytes_remaining = bytes; 192804ed95f4SEric Blake int max_transfer; 192961007b31SStefan Hajnoczi 1930d470ad42SMax Reitz if (!drv) { 1931d470ad42SMax Reitz return -ENOMEDIUM; 1932d470ad42SMax Reitz } 1933d470ad42SMax Reitz 1934d6883bc9SVladimir Sementsov-Ogievskiy if (bdrv_has_readonly_bitmaps(bs)) { 1935d6883bc9SVladimir Sementsov-Ogievskiy return -EPERM; 1936d6883bc9SVladimir Sementsov-Ogievskiy } 1937d6883bc9SVladimir Sementsov-Ogievskiy 1938cff86b38SEric Blake assert(is_power_of_2(align)); 1939cff86b38SEric Blake assert((offset & (align - 1)) == 0); 1940cff86b38SEric Blake assert((bytes & (align - 1)) == 0); 194128c4da28SVladimir Sementsov-Ogievskiy assert(!qiov || qiov_offset + bytes <= qiov->size); 194204ed95f4SEric Blake max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 194304ed95f4SEric Blake align); 194461007b31SStefan Hajnoczi 194585fe2479SFam Zheng ret = bdrv_co_write_req_prepare(child, offset, bytes, req, flags); 194661007b31SStefan Hajnoczi 194761007b31SStefan Hajnoczi if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF && 1948c1499a5eSEric Blake !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes && 194928c4da28SVladimir Sementsov-Ogievskiy qemu_iovec_is_zero(qiov, qiov_offset, bytes)) { 195061007b31SStefan Hajnoczi flags |= BDRV_REQ_ZERO_WRITE; 195161007b31SStefan Hajnoczi if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) { 195261007b31SStefan Hajnoczi flags |= BDRV_REQ_MAY_UNMAP; 195361007b31SStefan Hajnoczi } 195461007b31SStefan Hajnoczi } 195561007b31SStefan Hajnoczi 195661007b31SStefan Hajnoczi if (ret < 0) { 195761007b31SStefan Hajnoczi /* Do nothing, write notifier decided to fail this request */ 195861007b31SStefan Hajnoczi } else if (flags & BDRV_REQ_ZERO_WRITE) { 19599a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO); 19609896c876SKevin Wolf ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags); 19613ea1a091SPavel Butsykin } else if (flags & BDRV_REQ_WRITE_COMPRESSED) { 196228c4da28SVladimir Sementsov-Ogievskiy ret = bdrv_driver_pwritev_compressed(bs, offset, bytes, 196328c4da28SVladimir Sementsov-Ogievskiy qiov, qiov_offset); 196404ed95f4SEric Blake } else if (bytes <= max_transfer) { 19659a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV); 196628c4da28SVladimir Sementsov-Ogievskiy ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, qiov_offset, flags); 196704ed95f4SEric Blake } else { 196804ed95f4SEric Blake bdrv_debug_event(bs, BLKDBG_PWRITEV); 196904ed95f4SEric Blake while (bytes_remaining) { 197004ed95f4SEric Blake int num = MIN(bytes_remaining, max_transfer); 197104ed95f4SEric Blake int local_flags = flags; 197204ed95f4SEric Blake 197304ed95f4SEric Blake assert(num); 197404ed95f4SEric Blake if (num < bytes_remaining && (flags & BDRV_REQ_FUA) && 197504ed95f4SEric Blake !(bs->supported_write_flags & BDRV_REQ_FUA)) { 197604ed95f4SEric Blake /* If FUA is going to be emulated by flush, we only 197704ed95f4SEric Blake * need to flush on the last iteration */ 197804ed95f4SEric Blake local_flags &= ~BDRV_REQ_FUA; 197904ed95f4SEric Blake } 198004ed95f4SEric Blake 198104ed95f4SEric Blake ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining, 1982134b7decSMax Reitz num, qiov, 1983134b7decSMax Reitz qiov_offset + bytes - bytes_remaining, 198428c4da28SVladimir Sementsov-Ogievskiy local_flags); 198504ed95f4SEric Blake if (ret < 0) { 198604ed95f4SEric Blake break; 198704ed95f4SEric Blake } 198804ed95f4SEric Blake bytes_remaining -= num; 198904ed95f4SEric Blake } 199061007b31SStefan Hajnoczi } 19919a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE); 199261007b31SStefan Hajnoczi 199361007b31SStefan Hajnoczi if (ret >= 0) { 199404ed95f4SEric Blake ret = 0; 199561007b31SStefan Hajnoczi } 199685fe2479SFam Zheng bdrv_co_write_req_finish(child, offset, bytes, req, ret); 199761007b31SStefan Hajnoczi 199861007b31SStefan Hajnoczi return ret; 199961007b31SStefan Hajnoczi } 200061007b31SStefan Hajnoczi 200185c97ca7SKevin Wolf static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child, 20029eeb6dd1SFam Zheng int64_t offset, 20039eeb6dd1SFam Zheng unsigned int bytes, 20049eeb6dd1SFam Zheng BdrvRequestFlags flags, 20059eeb6dd1SFam Zheng BdrvTrackedRequest *req) 20069eeb6dd1SFam Zheng { 200785c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 20089eeb6dd1SFam Zheng QEMUIOVector local_qiov; 2009a5b8dd2cSEric Blake uint64_t align = bs->bl.request_alignment; 20109eeb6dd1SFam Zheng int ret = 0; 20117a3f542fSVladimir Sementsov-Ogievskiy bool padding; 20127a3f542fSVladimir Sementsov-Ogievskiy BdrvRequestPadding pad; 20139eeb6dd1SFam Zheng 20147a3f542fSVladimir Sementsov-Ogievskiy padding = bdrv_init_padding(bs, offset, bytes, &pad); 20157a3f542fSVladimir Sementsov-Ogievskiy if (padding) { 2016304d9d7fSMax Reitz bdrv_mark_request_serialising(req, align); 20179eeb6dd1SFam Zheng 20187a3f542fSVladimir Sementsov-Ogievskiy bdrv_padding_rmw_read(child, req, &pad, true); 20197a3f542fSVladimir Sementsov-Ogievskiy 20207a3f542fSVladimir Sementsov-Ogievskiy if (pad.head || pad.merge_reads) { 20217a3f542fSVladimir Sementsov-Ogievskiy int64_t aligned_offset = offset & ~(align - 1); 20227a3f542fSVladimir Sementsov-Ogievskiy int64_t write_bytes = pad.merge_reads ? pad.buf_len : align; 20237a3f542fSVladimir Sementsov-Ogievskiy 20247a3f542fSVladimir Sementsov-Ogievskiy qemu_iovec_init_buf(&local_qiov, pad.buf, write_bytes); 20257a3f542fSVladimir Sementsov-Ogievskiy ret = bdrv_aligned_pwritev(child, req, aligned_offset, write_bytes, 202628c4da28SVladimir Sementsov-Ogievskiy align, &local_qiov, 0, 20279eeb6dd1SFam Zheng flags & ~BDRV_REQ_ZERO_WRITE); 20287a3f542fSVladimir Sementsov-Ogievskiy if (ret < 0 || pad.merge_reads) { 20297a3f542fSVladimir Sementsov-Ogievskiy /* Error or all work is done */ 20307a3f542fSVladimir Sementsov-Ogievskiy goto out; 20319eeb6dd1SFam Zheng } 20327a3f542fSVladimir Sementsov-Ogievskiy offset += write_bytes - pad.head; 20337a3f542fSVladimir Sementsov-Ogievskiy bytes -= write_bytes - pad.head; 20347a3f542fSVladimir Sementsov-Ogievskiy } 20359eeb6dd1SFam Zheng } 20369eeb6dd1SFam Zheng 20379eeb6dd1SFam Zheng assert(!bytes || (offset & (align - 1)) == 0); 20389eeb6dd1SFam Zheng if (bytes >= align) { 20399eeb6dd1SFam Zheng /* Write the aligned part in the middle. */ 20409eeb6dd1SFam Zheng uint64_t aligned_bytes = bytes & ~(align - 1); 204185c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align, 204228c4da28SVladimir Sementsov-Ogievskiy NULL, 0, flags); 20439eeb6dd1SFam Zheng if (ret < 0) { 20447a3f542fSVladimir Sementsov-Ogievskiy goto out; 20459eeb6dd1SFam Zheng } 20469eeb6dd1SFam Zheng bytes -= aligned_bytes; 20479eeb6dd1SFam Zheng offset += aligned_bytes; 20489eeb6dd1SFam Zheng } 20499eeb6dd1SFam Zheng 20509eeb6dd1SFam Zheng assert(!bytes || (offset & (align - 1)) == 0); 20519eeb6dd1SFam Zheng if (bytes) { 20527a3f542fSVladimir Sementsov-Ogievskiy assert(align == pad.tail + bytes); 20539eeb6dd1SFam Zheng 20547a3f542fSVladimir Sementsov-Ogievskiy qemu_iovec_init_buf(&local_qiov, pad.tail_buf, align); 205585c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, req, offset, align, align, 205628c4da28SVladimir Sementsov-Ogievskiy &local_qiov, 0, 205728c4da28SVladimir Sementsov-Ogievskiy flags & ~BDRV_REQ_ZERO_WRITE); 20589eeb6dd1SFam Zheng } 20599eeb6dd1SFam Zheng 20607a3f542fSVladimir Sementsov-Ogievskiy out: 20617a3f542fSVladimir Sementsov-Ogievskiy bdrv_padding_destroy(&pad); 20627a3f542fSVladimir Sementsov-Ogievskiy 20637a3f542fSVladimir Sementsov-Ogievskiy return ret; 20649eeb6dd1SFam Zheng } 20659eeb6dd1SFam Zheng 206661007b31SStefan Hajnoczi /* 206761007b31SStefan Hajnoczi * Handle a write request in coroutine context 206861007b31SStefan Hajnoczi */ 2069a03ef88fSKevin Wolf int coroutine_fn bdrv_co_pwritev(BdrvChild *child, 207061007b31SStefan Hajnoczi int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 207161007b31SStefan Hajnoczi BdrvRequestFlags flags) 207261007b31SStefan Hajnoczi { 20731acc3466SVladimir Sementsov-Ogievskiy return bdrv_co_pwritev_part(child, offset, bytes, qiov, 0, flags); 20741acc3466SVladimir Sementsov-Ogievskiy } 20751acc3466SVladimir Sementsov-Ogievskiy 20761acc3466SVladimir Sementsov-Ogievskiy int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child, 20771acc3466SVladimir Sementsov-Ogievskiy int64_t offset, unsigned int bytes, QEMUIOVector *qiov, size_t qiov_offset, 20781acc3466SVladimir Sementsov-Ogievskiy BdrvRequestFlags flags) 20791acc3466SVladimir Sementsov-Ogievskiy { 2080a03ef88fSKevin Wolf BlockDriverState *bs = child->bs; 208161007b31SStefan Hajnoczi BdrvTrackedRequest req; 2082a5b8dd2cSEric Blake uint64_t align = bs->bl.request_alignment; 20837a3f542fSVladimir Sementsov-Ogievskiy BdrvRequestPadding pad; 208461007b31SStefan Hajnoczi int ret; 208561007b31SStefan Hajnoczi 2086f42cf447SDaniel P. Berrange trace_bdrv_co_pwritev(child->bs, offset, bytes, flags); 2087f42cf447SDaniel P. Berrange 2088f4dad307SVladimir Sementsov-Ogievskiy if (!bdrv_is_inserted(bs)) { 208961007b31SStefan Hajnoczi return -ENOMEDIUM; 209061007b31SStefan Hajnoczi } 209161007b31SStefan Hajnoczi 20928b117001SVladimir Sementsov-Ogievskiy ret = bdrv_check_request32(offset, bytes); 209361007b31SStefan Hajnoczi if (ret < 0) { 209461007b31SStefan Hajnoczi return ret; 209561007b31SStefan Hajnoczi } 209661007b31SStefan Hajnoczi 2097f2208fdcSAlberto Garcia /* If the request is misaligned then we can't make it efficient */ 2098f2208fdcSAlberto Garcia if ((flags & BDRV_REQ_NO_FALLBACK) && 2099f2208fdcSAlberto Garcia !QEMU_IS_ALIGNED(offset | bytes, align)) 2100f2208fdcSAlberto Garcia { 2101f2208fdcSAlberto Garcia return -ENOTSUP; 2102f2208fdcSAlberto Garcia } 2103f2208fdcSAlberto Garcia 2104ac9d00bfSVladimir Sementsov-Ogievskiy if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) { 2105ac9d00bfSVladimir Sementsov-Ogievskiy /* 2106ac9d00bfSVladimir Sementsov-Ogievskiy * Aligning zero request is nonsense. Even if driver has special meaning 2107ac9d00bfSVladimir Sementsov-Ogievskiy * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass 2108ac9d00bfSVladimir Sementsov-Ogievskiy * it to driver due to request_alignment. 2109ac9d00bfSVladimir Sementsov-Ogievskiy * 2110ac9d00bfSVladimir Sementsov-Ogievskiy * Still, no reason to return an error if someone do unaligned 2111ac9d00bfSVladimir Sementsov-Ogievskiy * zero-length write occasionally. 2112ac9d00bfSVladimir Sementsov-Ogievskiy */ 2113ac9d00bfSVladimir Sementsov-Ogievskiy return 0; 2114ac9d00bfSVladimir Sementsov-Ogievskiy } 2115ac9d00bfSVladimir Sementsov-Ogievskiy 211699723548SPaolo Bonzini bdrv_inc_in_flight(bs); 211761007b31SStefan Hajnoczi /* 211861007b31SStefan Hajnoczi * Align write if necessary by performing a read-modify-write cycle. 211961007b31SStefan Hajnoczi * Pad qiov with the read parts and be sure to have a tracked request not 212061007b31SStefan Hajnoczi * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle. 212161007b31SStefan Hajnoczi */ 2122ebde595cSFam Zheng tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE); 212361007b31SStefan Hajnoczi 212418a59f03SAnton Nefedov if (flags & BDRV_REQ_ZERO_WRITE) { 212585c97ca7SKevin Wolf ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req); 21269eeb6dd1SFam Zheng goto out; 21279eeb6dd1SFam Zheng } 21289eeb6dd1SFam Zheng 21291acc3466SVladimir Sementsov-Ogievskiy if (bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad)) { 2130304d9d7fSMax Reitz bdrv_mark_request_serialising(&req, align); 21317a3f542fSVladimir Sementsov-Ogievskiy bdrv_padding_rmw_read(child, &req, &pad, false); 213261007b31SStefan Hajnoczi } 213361007b31SStefan Hajnoczi 213485c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align, 21351acc3466SVladimir Sementsov-Ogievskiy qiov, qiov_offset, flags); 213661007b31SStefan Hajnoczi 21377a3f542fSVladimir Sementsov-Ogievskiy bdrv_padding_destroy(&pad); 213861007b31SStefan Hajnoczi 21399eeb6dd1SFam Zheng out: 21409eeb6dd1SFam Zheng tracked_request_end(&req); 214199723548SPaolo Bonzini bdrv_dec_in_flight(bs); 21427a3f542fSVladimir Sementsov-Ogievskiy 214361007b31SStefan Hajnoczi return ret; 214461007b31SStefan Hajnoczi } 214561007b31SStefan Hajnoczi 2146a03ef88fSKevin Wolf int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset, 2147f5a5ca79SManos Pitsidianakis int bytes, BdrvRequestFlags flags) 214861007b31SStefan Hajnoczi { 2149f5a5ca79SManos Pitsidianakis trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags); 215061007b31SStefan Hajnoczi 2151a03ef88fSKevin Wolf if (!(child->bs->open_flags & BDRV_O_UNMAP)) { 215261007b31SStefan Hajnoczi flags &= ~BDRV_REQ_MAY_UNMAP; 215361007b31SStefan Hajnoczi } 215461007b31SStefan Hajnoczi 2155f5a5ca79SManos Pitsidianakis return bdrv_co_pwritev(child, offset, bytes, NULL, 215661007b31SStefan Hajnoczi BDRV_REQ_ZERO_WRITE | flags); 215761007b31SStefan Hajnoczi } 215861007b31SStefan Hajnoczi 21594085f5c7SJohn Snow /* 21604085f5c7SJohn Snow * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not. 21614085f5c7SJohn Snow */ 21624085f5c7SJohn Snow int bdrv_flush_all(void) 21634085f5c7SJohn Snow { 21644085f5c7SJohn Snow BdrvNextIterator it; 21654085f5c7SJohn Snow BlockDriverState *bs = NULL; 21664085f5c7SJohn Snow int result = 0; 21674085f5c7SJohn Snow 2168c8aa7895SPavel Dovgalyuk /* 2169c8aa7895SPavel Dovgalyuk * bdrv queue is managed by record/replay, 2170c8aa7895SPavel Dovgalyuk * creating new flush request for stopping 2171c8aa7895SPavel Dovgalyuk * the VM may break the determinism 2172c8aa7895SPavel Dovgalyuk */ 2173c8aa7895SPavel Dovgalyuk if (replay_events_enabled()) { 2174c8aa7895SPavel Dovgalyuk return result; 2175c8aa7895SPavel Dovgalyuk } 2176c8aa7895SPavel Dovgalyuk 21774085f5c7SJohn Snow for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 21784085f5c7SJohn Snow AioContext *aio_context = bdrv_get_aio_context(bs); 21794085f5c7SJohn Snow int ret; 21804085f5c7SJohn Snow 21814085f5c7SJohn Snow aio_context_acquire(aio_context); 21824085f5c7SJohn Snow ret = bdrv_flush(bs); 21834085f5c7SJohn Snow if (ret < 0 && !result) { 21844085f5c7SJohn Snow result = ret; 21854085f5c7SJohn Snow } 21864085f5c7SJohn Snow aio_context_release(aio_context); 21874085f5c7SJohn Snow } 21884085f5c7SJohn Snow 21894085f5c7SJohn Snow return result; 21904085f5c7SJohn Snow } 21914085f5c7SJohn Snow 219261007b31SStefan Hajnoczi /* 219361007b31SStefan Hajnoczi * Returns the allocation status of the specified sectors. 219461007b31SStefan Hajnoczi * Drivers not implementing the functionality are assumed to not support 219561007b31SStefan Hajnoczi * backing files, hence all their sectors are reported as allocated. 219661007b31SStefan Hajnoczi * 219786a3d5c6SEric Blake * If 'want_zero' is true, the caller is querying for mapping 219886a3d5c6SEric Blake * purposes, with a focus on valid BDRV_BLOCK_OFFSET_VALID, _DATA, and 219986a3d5c6SEric Blake * _ZERO where possible; otherwise, the result favors larger 'pnum', 220086a3d5c6SEric Blake * with a focus on accurate BDRV_BLOCK_ALLOCATED. 2201c9ce8c4dSEric Blake * 22022e8bc787SEric Blake * If 'offset' is beyond the end of the disk image the return value is 2203fb0d8654SEric Blake * BDRV_BLOCK_EOF and 'pnum' is set to 0. 220461007b31SStefan Hajnoczi * 22052e8bc787SEric Blake * 'bytes' is the max value 'pnum' should be set to. If bytes goes 2206fb0d8654SEric Blake * beyond the end of the disk image it will be clamped; if 'pnum' is set to 2207fb0d8654SEric Blake * the end of the image, then the returned value will include BDRV_BLOCK_EOF. 220867a0fd2aSFam Zheng * 22092e8bc787SEric Blake * 'pnum' is set to the number of bytes (including and immediately 22102e8bc787SEric Blake * following the specified offset) that are easily known to be in the 22112e8bc787SEric Blake * same allocated/unallocated state. Note that a second call starting 22122e8bc787SEric Blake * at the original offset plus returned pnum may have the same status. 22132e8bc787SEric Blake * The returned value is non-zero on success except at end-of-file. 22142e8bc787SEric Blake * 22152e8bc787SEric Blake * Returns negative errno on failure. Otherwise, if the 22162e8bc787SEric Blake * BDRV_BLOCK_OFFSET_VALID bit is set, 'map' and 'file' (if non-NULL) are 22172e8bc787SEric Blake * set to the host mapping and BDS corresponding to the guest offset. 221861007b31SStefan Hajnoczi */ 22192e8bc787SEric Blake static int coroutine_fn bdrv_co_block_status(BlockDriverState *bs, 2220c9ce8c4dSEric Blake bool want_zero, 22212e8bc787SEric Blake int64_t offset, int64_t bytes, 22222e8bc787SEric Blake int64_t *pnum, int64_t *map, 222367a0fd2aSFam Zheng BlockDriverState **file) 222461007b31SStefan Hajnoczi { 22252e8bc787SEric Blake int64_t total_size; 22262e8bc787SEric Blake int64_t n; /* bytes */ 2227efa6e2edSEric Blake int ret; 22282e8bc787SEric Blake int64_t local_map = 0; 2229298a1665SEric Blake BlockDriverState *local_file = NULL; 2230efa6e2edSEric Blake int64_t aligned_offset, aligned_bytes; 2231efa6e2edSEric Blake uint32_t align; 2232549ec0d9SMax Reitz bool has_filtered_child; 223361007b31SStefan Hajnoczi 2234298a1665SEric Blake assert(pnum); 2235298a1665SEric Blake *pnum = 0; 22362e8bc787SEric Blake total_size = bdrv_getlength(bs); 22372e8bc787SEric Blake if (total_size < 0) { 22382e8bc787SEric Blake ret = total_size; 2239298a1665SEric Blake goto early_out; 224061007b31SStefan Hajnoczi } 224161007b31SStefan Hajnoczi 22422e8bc787SEric Blake if (offset >= total_size) { 2243298a1665SEric Blake ret = BDRV_BLOCK_EOF; 2244298a1665SEric Blake goto early_out; 224561007b31SStefan Hajnoczi } 22462e8bc787SEric Blake if (!bytes) { 2247298a1665SEric Blake ret = 0; 2248298a1665SEric Blake goto early_out; 22499cdcfd9fSEric Blake } 225061007b31SStefan Hajnoczi 22512e8bc787SEric Blake n = total_size - offset; 22522e8bc787SEric Blake if (n < bytes) { 22532e8bc787SEric Blake bytes = n; 225461007b31SStefan Hajnoczi } 225561007b31SStefan Hajnoczi 2256d470ad42SMax Reitz /* Must be non-NULL or bdrv_getlength() would have failed */ 2257d470ad42SMax Reitz assert(bs->drv); 2258549ec0d9SMax Reitz has_filtered_child = bdrv_filter_child(bs); 2259549ec0d9SMax Reitz if (!bs->drv->bdrv_co_block_status && !has_filtered_child) { 22602e8bc787SEric Blake *pnum = bytes; 226161007b31SStefan Hajnoczi ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED; 22622e8bc787SEric Blake if (offset + bytes == total_size) { 2263fb0d8654SEric Blake ret |= BDRV_BLOCK_EOF; 2264fb0d8654SEric Blake } 226561007b31SStefan Hajnoczi if (bs->drv->protocol_name) { 22662e8bc787SEric Blake ret |= BDRV_BLOCK_OFFSET_VALID; 22672e8bc787SEric Blake local_map = offset; 2268298a1665SEric Blake local_file = bs; 226961007b31SStefan Hajnoczi } 2270298a1665SEric Blake goto early_out; 227161007b31SStefan Hajnoczi } 227261007b31SStefan Hajnoczi 227399723548SPaolo Bonzini bdrv_inc_in_flight(bs); 2274efa6e2edSEric Blake 2275efa6e2edSEric Blake /* Round out to request_alignment boundaries */ 227686a3d5c6SEric Blake align = bs->bl.request_alignment; 2277efa6e2edSEric Blake aligned_offset = QEMU_ALIGN_DOWN(offset, align); 2278efa6e2edSEric Blake aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset; 2279efa6e2edSEric Blake 2280549ec0d9SMax Reitz if (bs->drv->bdrv_co_block_status) { 228186a3d5c6SEric Blake ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset, 228286a3d5c6SEric Blake aligned_bytes, pnum, &local_map, 228386a3d5c6SEric Blake &local_file); 2284549ec0d9SMax Reitz } else { 2285549ec0d9SMax Reitz /* Default code for filters */ 2286549ec0d9SMax Reitz 2287549ec0d9SMax Reitz local_file = bdrv_filter_bs(bs); 2288549ec0d9SMax Reitz assert(local_file); 2289549ec0d9SMax Reitz 2290549ec0d9SMax Reitz *pnum = aligned_bytes; 2291549ec0d9SMax Reitz local_map = aligned_offset; 2292549ec0d9SMax Reitz ret = BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID; 2293549ec0d9SMax Reitz } 229486a3d5c6SEric Blake if (ret < 0) { 229586a3d5c6SEric Blake *pnum = 0; 229686a3d5c6SEric Blake goto out; 229786a3d5c6SEric Blake } 2298efa6e2edSEric Blake 2299efa6e2edSEric Blake /* 2300636cb512SEric Blake * The driver's result must be a non-zero multiple of request_alignment. 2301efa6e2edSEric Blake * Clamp pnum and adjust map to original request. 2302efa6e2edSEric Blake */ 2303636cb512SEric Blake assert(*pnum && QEMU_IS_ALIGNED(*pnum, align) && 2304636cb512SEric Blake align > offset - aligned_offset); 230569f47505SVladimir Sementsov-Ogievskiy if (ret & BDRV_BLOCK_RECURSE) { 230669f47505SVladimir Sementsov-Ogievskiy assert(ret & BDRV_BLOCK_DATA); 230769f47505SVladimir Sementsov-Ogievskiy assert(ret & BDRV_BLOCK_OFFSET_VALID); 230869f47505SVladimir Sementsov-Ogievskiy assert(!(ret & BDRV_BLOCK_ZERO)); 230969f47505SVladimir Sementsov-Ogievskiy } 231069f47505SVladimir Sementsov-Ogievskiy 2311efa6e2edSEric Blake *pnum -= offset - aligned_offset; 2312efa6e2edSEric Blake if (*pnum > bytes) { 2313efa6e2edSEric Blake *pnum = bytes; 2314efa6e2edSEric Blake } 2315efa6e2edSEric Blake if (ret & BDRV_BLOCK_OFFSET_VALID) { 2316efa6e2edSEric Blake local_map += offset - aligned_offset; 2317efa6e2edSEric Blake } 231861007b31SStefan Hajnoczi 231961007b31SStefan Hajnoczi if (ret & BDRV_BLOCK_RAW) { 2320298a1665SEric Blake assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file); 23212e8bc787SEric Blake ret = bdrv_co_block_status(local_file, want_zero, local_map, 23222e8bc787SEric Blake *pnum, pnum, &local_map, &local_file); 232399723548SPaolo Bonzini goto out; 232461007b31SStefan Hajnoczi } 232561007b31SStefan Hajnoczi 232661007b31SStefan Hajnoczi if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) { 232761007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ALLOCATED; 2328d40f4a56SAlberto Garcia } else if (bs->drv->supports_backing) { 2329cb850315SMax Reitz BlockDriverState *cow_bs = bdrv_cow_bs(bs); 2330cb850315SMax Reitz 2331d40f4a56SAlberto Garcia if (!cow_bs) { 2332d40f4a56SAlberto Garcia ret |= BDRV_BLOCK_ZERO; 2333d40f4a56SAlberto Garcia } else if (want_zero) { 2334cb850315SMax Reitz int64_t size2 = bdrv_getlength(cow_bs); 2335c9ce8c4dSEric Blake 23362e8bc787SEric Blake if (size2 >= 0 && offset >= size2) { 233761007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 233861007b31SStefan Hajnoczi } 23397b1efe99SVladimir Sementsov-Ogievskiy } 234061007b31SStefan Hajnoczi } 234161007b31SStefan Hajnoczi 234269f47505SVladimir Sementsov-Ogievskiy if (want_zero && ret & BDRV_BLOCK_RECURSE && 234369f47505SVladimir Sementsov-Ogievskiy local_file && local_file != bs && 234461007b31SStefan Hajnoczi (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) && 234561007b31SStefan Hajnoczi (ret & BDRV_BLOCK_OFFSET_VALID)) { 23462e8bc787SEric Blake int64_t file_pnum; 23472e8bc787SEric Blake int ret2; 234861007b31SStefan Hajnoczi 23492e8bc787SEric Blake ret2 = bdrv_co_block_status(local_file, want_zero, local_map, 23502e8bc787SEric Blake *pnum, &file_pnum, NULL, NULL); 235161007b31SStefan Hajnoczi if (ret2 >= 0) { 235261007b31SStefan Hajnoczi /* Ignore errors. This is just providing extra information, it 235361007b31SStefan Hajnoczi * is useful but not necessary. 235461007b31SStefan Hajnoczi */ 2355c61e684eSEric Blake if (ret2 & BDRV_BLOCK_EOF && 2356c61e684eSEric Blake (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) { 2357c61e684eSEric Blake /* 2358c61e684eSEric Blake * It is valid for the format block driver to read 2359c61e684eSEric Blake * beyond the end of the underlying file's current 2360c61e684eSEric Blake * size; such areas read as zero. 2361c61e684eSEric Blake */ 236261007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 236361007b31SStefan Hajnoczi } else { 236461007b31SStefan Hajnoczi /* Limit request to the range reported by the protocol driver */ 236561007b31SStefan Hajnoczi *pnum = file_pnum; 236661007b31SStefan Hajnoczi ret |= (ret2 & BDRV_BLOCK_ZERO); 236761007b31SStefan Hajnoczi } 236861007b31SStefan Hajnoczi } 236961007b31SStefan Hajnoczi } 237061007b31SStefan Hajnoczi 237199723548SPaolo Bonzini out: 237299723548SPaolo Bonzini bdrv_dec_in_flight(bs); 23732e8bc787SEric Blake if (ret >= 0 && offset + *pnum == total_size) { 2374fb0d8654SEric Blake ret |= BDRV_BLOCK_EOF; 2375fb0d8654SEric Blake } 2376298a1665SEric Blake early_out: 2377298a1665SEric Blake if (file) { 2378298a1665SEric Blake *file = local_file; 2379298a1665SEric Blake } 23802e8bc787SEric Blake if (map) { 23812e8bc787SEric Blake *map = local_map; 23822e8bc787SEric Blake } 238361007b31SStefan Hajnoczi return ret; 238461007b31SStefan Hajnoczi } 238561007b31SStefan Hajnoczi 238621c2283eSVladimir Sementsov-Ogievskiy int coroutine_fn 2387f9e694cbSVladimir Sementsov-Ogievskiy bdrv_co_common_block_status_above(BlockDriverState *bs, 2388ba3f0e25SFam Zheng BlockDriverState *base, 23893555a432SVladimir Sementsov-Ogievskiy bool include_base, 2390c9ce8c4dSEric Blake bool want_zero, 23915b648c67SEric Blake int64_t offset, 23925b648c67SEric Blake int64_t bytes, 23935b648c67SEric Blake int64_t *pnum, 23945b648c67SEric Blake int64_t *map, 2395a92b1b06SEric Blake BlockDriverState **file, 2396a92b1b06SEric Blake int *depth) 2397ba3f0e25SFam Zheng { 239867c095c8SVladimir Sementsov-Ogievskiy int ret; 2399ba3f0e25SFam Zheng BlockDriverState *p; 240067c095c8SVladimir Sementsov-Ogievskiy int64_t eof = 0; 2401a92b1b06SEric Blake int dummy; 2402ba3f0e25SFam Zheng 24033555a432SVladimir Sementsov-Ogievskiy assert(!include_base || base); /* Can't include NULL base */ 240467c095c8SVladimir Sementsov-Ogievskiy 2405a92b1b06SEric Blake if (!depth) { 2406a92b1b06SEric Blake depth = &dummy; 2407a92b1b06SEric Blake } 2408a92b1b06SEric Blake *depth = 0; 2409a92b1b06SEric Blake 2410624f27bbSVladimir Sementsov-Ogievskiy if (!include_base && bs == base) { 2411624f27bbSVladimir Sementsov-Ogievskiy *pnum = bytes; 2412624f27bbSVladimir Sementsov-Ogievskiy return 0; 2413624f27bbSVladimir Sementsov-Ogievskiy } 2414624f27bbSVladimir Sementsov-Ogievskiy 241567c095c8SVladimir Sementsov-Ogievskiy ret = bdrv_co_block_status(bs, want_zero, offset, bytes, pnum, map, file); 2416a92b1b06SEric Blake ++*depth; 24173555a432SVladimir Sementsov-Ogievskiy if (ret < 0 || *pnum == 0 || ret & BDRV_BLOCK_ALLOCATED || bs == base) { 241867c095c8SVladimir Sementsov-Ogievskiy return ret; 241967c095c8SVladimir Sementsov-Ogievskiy } 242067c095c8SVladimir Sementsov-Ogievskiy 242167c095c8SVladimir Sementsov-Ogievskiy if (ret & BDRV_BLOCK_EOF) { 242267c095c8SVladimir Sementsov-Ogievskiy eof = offset + *pnum; 242367c095c8SVladimir Sementsov-Ogievskiy } 242467c095c8SVladimir Sementsov-Ogievskiy 242567c095c8SVladimir Sementsov-Ogievskiy assert(*pnum <= bytes); 242667c095c8SVladimir Sementsov-Ogievskiy bytes = *pnum; 242767c095c8SVladimir Sementsov-Ogievskiy 24283555a432SVladimir Sementsov-Ogievskiy for (p = bdrv_filter_or_cow_bs(bs); include_base || p != base; 242967c095c8SVladimir Sementsov-Ogievskiy p = bdrv_filter_or_cow_bs(p)) 243067c095c8SVladimir Sementsov-Ogievskiy { 24315b648c67SEric Blake ret = bdrv_co_block_status(p, want_zero, offset, bytes, pnum, map, 24325b648c67SEric Blake file); 2433a92b1b06SEric Blake ++*depth; 2434c61e684eSEric Blake if (ret < 0) { 243567c095c8SVladimir Sementsov-Ogievskiy return ret; 2436c61e684eSEric Blake } 243767c095c8SVladimir Sementsov-Ogievskiy if (*pnum == 0) { 2438c61e684eSEric Blake /* 243967c095c8SVladimir Sementsov-Ogievskiy * The top layer deferred to this layer, and because this layer is 244067c095c8SVladimir Sementsov-Ogievskiy * short, any zeroes that we synthesize beyond EOF behave as if they 244167c095c8SVladimir Sementsov-Ogievskiy * were allocated at this layer. 244267c095c8SVladimir Sementsov-Ogievskiy * 244367c095c8SVladimir Sementsov-Ogievskiy * We don't include BDRV_BLOCK_EOF into ret, as upper layer may be 244467c095c8SVladimir Sementsov-Ogievskiy * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see 244567c095c8SVladimir Sementsov-Ogievskiy * below. 2446c61e684eSEric Blake */ 244767c095c8SVladimir Sementsov-Ogievskiy assert(ret & BDRV_BLOCK_EOF); 24485b648c67SEric Blake *pnum = bytes; 244967c095c8SVladimir Sementsov-Ogievskiy if (file) { 245067c095c8SVladimir Sementsov-Ogievskiy *file = p; 2451c61e684eSEric Blake } 245267c095c8SVladimir Sementsov-Ogievskiy ret = BDRV_BLOCK_ZERO | BDRV_BLOCK_ALLOCATED; 2453ba3f0e25SFam Zheng break; 2454ba3f0e25SFam Zheng } 245567c095c8SVladimir Sementsov-Ogievskiy if (ret & BDRV_BLOCK_ALLOCATED) { 245667c095c8SVladimir Sementsov-Ogievskiy /* 245767c095c8SVladimir Sementsov-Ogievskiy * We've found the node and the status, we must break. 245867c095c8SVladimir Sementsov-Ogievskiy * 245967c095c8SVladimir Sementsov-Ogievskiy * Drop BDRV_BLOCK_EOF, as it's not for upper layer, which may be 246067c095c8SVladimir Sementsov-Ogievskiy * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see 246167c095c8SVladimir Sementsov-Ogievskiy * below. 246267c095c8SVladimir Sementsov-Ogievskiy */ 246367c095c8SVladimir Sementsov-Ogievskiy ret &= ~BDRV_BLOCK_EOF; 246467c095c8SVladimir Sementsov-Ogievskiy break; 2465ba3f0e25SFam Zheng } 246667c095c8SVladimir Sementsov-Ogievskiy 24673555a432SVladimir Sementsov-Ogievskiy if (p == base) { 24683555a432SVladimir Sementsov-Ogievskiy assert(include_base); 24693555a432SVladimir Sementsov-Ogievskiy break; 24703555a432SVladimir Sementsov-Ogievskiy } 24713555a432SVladimir Sementsov-Ogievskiy 247267c095c8SVladimir Sementsov-Ogievskiy /* 247367c095c8SVladimir Sementsov-Ogievskiy * OK, [offset, offset + *pnum) region is unallocated on this layer, 247467c095c8SVladimir Sementsov-Ogievskiy * let's continue the diving. 247567c095c8SVladimir Sementsov-Ogievskiy */ 247667c095c8SVladimir Sementsov-Ogievskiy assert(*pnum <= bytes); 247767c095c8SVladimir Sementsov-Ogievskiy bytes = *pnum; 247867c095c8SVladimir Sementsov-Ogievskiy } 247967c095c8SVladimir Sementsov-Ogievskiy 248067c095c8SVladimir Sementsov-Ogievskiy if (offset + *pnum == eof) { 248167c095c8SVladimir Sementsov-Ogievskiy ret |= BDRV_BLOCK_EOF; 248267c095c8SVladimir Sementsov-Ogievskiy } 248367c095c8SVladimir Sementsov-Ogievskiy 2484ba3f0e25SFam Zheng return ret; 2485ba3f0e25SFam Zheng } 2486ba3f0e25SFam Zheng 248731826642SEric Blake int bdrv_block_status_above(BlockDriverState *bs, BlockDriverState *base, 248831826642SEric Blake int64_t offset, int64_t bytes, int64_t *pnum, 248931826642SEric Blake int64_t *map, BlockDriverState **file) 2490c9ce8c4dSEric Blake { 24913555a432SVladimir Sementsov-Ogievskiy return bdrv_common_block_status_above(bs, base, false, true, offset, bytes, 2492a92b1b06SEric Blake pnum, map, file, NULL); 2493c9ce8c4dSEric Blake } 2494c9ce8c4dSEric Blake 2495237d78f8SEric Blake int bdrv_block_status(BlockDriverState *bs, int64_t offset, int64_t bytes, 2496237d78f8SEric Blake int64_t *pnum, int64_t *map, BlockDriverState **file) 2497ba3f0e25SFam Zheng { 2498cb850315SMax Reitz return bdrv_block_status_above(bs, bdrv_filter_or_cow_bs(bs), 249931826642SEric Blake offset, bytes, pnum, map, file); 2500ba3f0e25SFam Zheng } 2501ba3f0e25SFam Zheng 250246cd1e8aSAlberto Garcia /* 250346cd1e8aSAlberto Garcia * Check @bs (and its backing chain) to see if the range defined 250446cd1e8aSAlberto Garcia * by @offset and @bytes is known to read as zeroes. 250546cd1e8aSAlberto Garcia * Return 1 if that is the case, 0 otherwise and -errno on error. 250646cd1e8aSAlberto Garcia * This test is meant to be fast rather than accurate so returning 0 250746cd1e8aSAlberto Garcia * does not guarantee non-zero data. 250846cd1e8aSAlberto Garcia */ 250946cd1e8aSAlberto Garcia int coroutine_fn bdrv_co_is_zero_fast(BlockDriverState *bs, int64_t offset, 251046cd1e8aSAlberto Garcia int64_t bytes) 251146cd1e8aSAlberto Garcia { 251246cd1e8aSAlberto Garcia int ret; 251346cd1e8aSAlberto Garcia int64_t pnum = bytes; 251446cd1e8aSAlberto Garcia 251546cd1e8aSAlberto Garcia if (!bytes) { 251646cd1e8aSAlberto Garcia return 1; 251746cd1e8aSAlberto Garcia } 251846cd1e8aSAlberto Garcia 251946cd1e8aSAlberto Garcia ret = bdrv_common_block_status_above(bs, NULL, false, false, offset, 2520a92b1b06SEric Blake bytes, &pnum, NULL, NULL, NULL); 252146cd1e8aSAlberto Garcia 252246cd1e8aSAlberto Garcia if (ret < 0) { 252346cd1e8aSAlberto Garcia return ret; 252446cd1e8aSAlberto Garcia } 252546cd1e8aSAlberto Garcia 252646cd1e8aSAlberto Garcia return (pnum == bytes) && (ret & BDRV_BLOCK_ZERO); 252746cd1e8aSAlberto Garcia } 252846cd1e8aSAlberto Garcia 2529d6a644bbSEric Blake int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t offset, 2530d6a644bbSEric Blake int64_t bytes, int64_t *pnum) 253161007b31SStefan Hajnoczi { 25327ddb99b9SEric Blake int ret; 25337ddb99b9SEric Blake int64_t dummy; 2534d6a644bbSEric Blake 25353555a432SVladimir Sementsov-Ogievskiy ret = bdrv_common_block_status_above(bs, bs, true, false, offset, 25363555a432SVladimir Sementsov-Ogievskiy bytes, pnum ? pnum : &dummy, NULL, 2537a92b1b06SEric Blake NULL, NULL); 253861007b31SStefan Hajnoczi if (ret < 0) { 253961007b31SStefan Hajnoczi return ret; 254061007b31SStefan Hajnoczi } 254161007b31SStefan Hajnoczi return !!(ret & BDRV_BLOCK_ALLOCATED); 254261007b31SStefan Hajnoczi } 254361007b31SStefan Hajnoczi 254461007b31SStefan Hajnoczi /* 254561007b31SStefan Hajnoczi * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP] 254661007b31SStefan Hajnoczi * 2547a92b1b06SEric Blake * Return a positive depth if (a prefix of) the given range is allocated 2548a92b1b06SEric Blake * in any image between BASE and TOP (BASE is only included if include_base 2549a92b1b06SEric Blake * is set). Depth 1 is TOP, 2 is the first backing layer, and so forth. 2550170d3bd3SAndrey Shinkevich * BASE can be NULL to check if the given offset is allocated in any 2551170d3bd3SAndrey Shinkevich * image of the chain. Return 0 otherwise, or negative errno on 2552170d3bd3SAndrey Shinkevich * failure. 255361007b31SStefan Hajnoczi * 255451b0a488SEric Blake * 'pnum' is set to the number of bytes (including and immediately 255551b0a488SEric Blake * following the specified offset) that are known to be in the same 255651b0a488SEric Blake * allocated/unallocated state. Note that a subsequent call starting 255751b0a488SEric Blake * at 'offset + *pnum' may return the same allocation status (in other 255851b0a488SEric Blake * words, the result is not necessarily the maximum possible range); 255951b0a488SEric Blake * but 'pnum' will only be 0 when end of file is reached. 256061007b31SStefan Hajnoczi */ 256161007b31SStefan Hajnoczi int bdrv_is_allocated_above(BlockDriverState *top, 256261007b31SStefan Hajnoczi BlockDriverState *base, 2563170d3bd3SAndrey Shinkevich bool include_base, int64_t offset, 2564170d3bd3SAndrey Shinkevich int64_t bytes, int64_t *pnum) 256561007b31SStefan Hajnoczi { 2566a92b1b06SEric Blake int depth; 25677e7e5100SVladimir Sementsov-Ogievskiy int ret = bdrv_common_block_status_above(top, base, include_base, false, 2568a92b1b06SEric Blake offset, bytes, pnum, NULL, NULL, 2569a92b1b06SEric Blake &depth); 257061007b31SStefan Hajnoczi if (ret < 0) { 257161007b31SStefan Hajnoczi return ret; 2572d6a644bbSEric Blake } 257361007b31SStefan Hajnoczi 2574a92b1b06SEric Blake if (ret & BDRV_BLOCK_ALLOCATED) { 2575a92b1b06SEric Blake return depth; 2576a92b1b06SEric Blake } 2577a92b1b06SEric Blake return 0; 257861007b31SStefan Hajnoczi } 257961007b31SStefan Hajnoczi 258021c2283eSVladimir Sementsov-Ogievskiy int coroutine_fn 2581b33b354fSVladimir Sementsov-Ogievskiy bdrv_co_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 25821a8ae822SKevin Wolf { 25831a8ae822SKevin Wolf BlockDriver *drv = bs->drv; 2584c4db2e25SMax Reitz BlockDriverState *child_bs = bdrv_primary_bs(bs); 2585dc88a467SStefan Hajnoczi int ret = -ENOTSUP; 2586dc88a467SStefan Hajnoczi 2587b33b354fSVladimir Sementsov-Ogievskiy if (!drv) { 2588b33b354fSVladimir Sementsov-Ogievskiy return -ENOMEDIUM; 2589b33b354fSVladimir Sementsov-Ogievskiy } 2590b33b354fSVladimir Sementsov-Ogievskiy 2591dc88a467SStefan Hajnoczi bdrv_inc_in_flight(bs); 25921a8ae822SKevin Wolf 2593b33b354fSVladimir Sementsov-Ogievskiy if (drv->bdrv_load_vmstate) { 2594dc88a467SStefan Hajnoczi ret = drv->bdrv_load_vmstate(bs, qiov, pos); 2595c4db2e25SMax Reitz } else if (child_bs) { 2596b33b354fSVladimir Sementsov-Ogievskiy ret = bdrv_co_readv_vmstate(child_bs, qiov, pos); 25971a8ae822SKevin Wolf } 25981a8ae822SKevin Wolf 2599dc88a467SStefan Hajnoczi bdrv_dec_in_flight(bs); 2600b33b354fSVladimir Sementsov-Ogievskiy 2601b33b354fSVladimir Sementsov-Ogievskiy return ret; 2602b33b354fSVladimir Sementsov-Ogievskiy } 2603b33b354fSVladimir Sementsov-Ogievskiy 2604b33b354fSVladimir Sementsov-Ogievskiy int coroutine_fn 2605b33b354fSVladimir Sementsov-Ogievskiy bdrv_co_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 2606b33b354fSVladimir Sementsov-Ogievskiy { 2607b33b354fSVladimir Sementsov-Ogievskiy BlockDriver *drv = bs->drv; 2608b33b354fSVladimir Sementsov-Ogievskiy BlockDriverState *child_bs = bdrv_primary_bs(bs); 2609b33b354fSVladimir Sementsov-Ogievskiy int ret = -ENOTSUP; 2610b33b354fSVladimir Sementsov-Ogievskiy 2611b33b354fSVladimir Sementsov-Ogievskiy if (!drv) { 2612b33b354fSVladimir Sementsov-Ogievskiy return -ENOMEDIUM; 2613b33b354fSVladimir Sementsov-Ogievskiy } 2614b33b354fSVladimir Sementsov-Ogievskiy 2615b33b354fSVladimir Sementsov-Ogievskiy bdrv_inc_in_flight(bs); 2616b33b354fSVladimir Sementsov-Ogievskiy 2617b33b354fSVladimir Sementsov-Ogievskiy if (drv->bdrv_save_vmstate) { 2618b33b354fSVladimir Sementsov-Ogievskiy ret = drv->bdrv_save_vmstate(bs, qiov, pos); 2619b33b354fSVladimir Sementsov-Ogievskiy } else if (child_bs) { 2620b33b354fSVladimir Sementsov-Ogievskiy ret = bdrv_co_writev_vmstate(child_bs, qiov, pos); 2621b33b354fSVladimir Sementsov-Ogievskiy } 2622b33b354fSVladimir Sementsov-Ogievskiy 2623b33b354fSVladimir Sementsov-Ogievskiy bdrv_dec_in_flight(bs); 2624b33b354fSVladimir Sementsov-Ogievskiy 2625dc88a467SStefan Hajnoczi return ret; 26261a8ae822SKevin Wolf } 26271a8ae822SKevin Wolf 262861007b31SStefan Hajnoczi int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf, 262961007b31SStefan Hajnoczi int64_t pos, int size) 263061007b31SStefan Hajnoczi { 26310d93ed08SVladimir Sementsov-Ogievskiy QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size); 2632b33b354fSVladimir Sementsov-Ogievskiy int ret = bdrv_writev_vmstate(bs, &qiov, pos); 263361007b31SStefan Hajnoczi 2634b33b354fSVladimir Sementsov-Ogievskiy return ret < 0 ? ret : size; 263561007b31SStefan Hajnoczi } 263661007b31SStefan Hajnoczi 263761007b31SStefan Hajnoczi int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf, 263861007b31SStefan Hajnoczi int64_t pos, int size) 263961007b31SStefan Hajnoczi { 26400d93ed08SVladimir Sementsov-Ogievskiy QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size); 2641b33b354fSVladimir Sementsov-Ogievskiy int ret = bdrv_readv_vmstate(bs, &qiov, pos); 26425ddda0b8SKevin Wolf 2643b33b354fSVladimir Sementsov-Ogievskiy return ret < 0 ? ret : size; 264461007b31SStefan Hajnoczi } 264561007b31SStefan Hajnoczi 264661007b31SStefan Hajnoczi /**************************************************************/ 264761007b31SStefan Hajnoczi /* async I/Os */ 264861007b31SStefan Hajnoczi 264961007b31SStefan Hajnoczi void bdrv_aio_cancel(BlockAIOCB *acb) 265061007b31SStefan Hajnoczi { 265161007b31SStefan Hajnoczi qemu_aio_ref(acb); 265261007b31SStefan Hajnoczi bdrv_aio_cancel_async(acb); 265361007b31SStefan Hajnoczi while (acb->refcnt > 1) { 265461007b31SStefan Hajnoczi if (acb->aiocb_info->get_aio_context) { 265561007b31SStefan Hajnoczi aio_poll(acb->aiocb_info->get_aio_context(acb), true); 265661007b31SStefan Hajnoczi } else if (acb->bs) { 26572f47da5fSPaolo Bonzini /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so 26582f47da5fSPaolo Bonzini * assert that we're not using an I/O thread. Thread-safe 26592f47da5fSPaolo Bonzini * code should use bdrv_aio_cancel_async exclusively. 26602f47da5fSPaolo Bonzini */ 26612f47da5fSPaolo Bonzini assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context()); 266261007b31SStefan Hajnoczi aio_poll(bdrv_get_aio_context(acb->bs), true); 266361007b31SStefan Hajnoczi } else { 266461007b31SStefan Hajnoczi abort(); 266561007b31SStefan Hajnoczi } 266661007b31SStefan Hajnoczi } 266761007b31SStefan Hajnoczi qemu_aio_unref(acb); 266861007b31SStefan Hajnoczi } 266961007b31SStefan Hajnoczi 267061007b31SStefan Hajnoczi /* Async version of aio cancel. The caller is not blocked if the acb implements 267161007b31SStefan Hajnoczi * cancel_async, otherwise we do nothing and let the request normally complete. 267261007b31SStefan Hajnoczi * In either case the completion callback must be called. */ 267361007b31SStefan Hajnoczi void bdrv_aio_cancel_async(BlockAIOCB *acb) 267461007b31SStefan Hajnoczi { 267561007b31SStefan Hajnoczi if (acb->aiocb_info->cancel_async) { 267661007b31SStefan Hajnoczi acb->aiocb_info->cancel_async(acb); 267761007b31SStefan Hajnoczi } 267861007b31SStefan Hajnoczi } 267961007b31SStefan Hajnoczi 268061007b31SStefan Hajnoczi /**************************************************************/ 268161007b31SStefan Hajnoczi /* Coroutine block device emulation */ 268261007b31SStefan Hajnoczi 268361007b31SStefan Hajnoczi int coroutine_fn bdrv_co_flush(BlockDriverState *bs) 268461007b31SStefan Hajnoczi { 2685883833e2SMax Reitz BdrvChild *primary_child = bdrv_primary_child(bs); 2686883833e2SMax Reitz BdrvChild *child; 268749ca6259SFam Zheng int current_gen; 268849ca6259SFam Zheng int ret = 0; 268961007b31SStefan Hajnoczi 269099723548SPaolo Bonzini bdrv_inc_in_flight(bs); 2691c32b82afSPavel Dovgalyuk 2692e914404eSFam Zheng if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs) || 269349ca6259SFam Zheng bdrv_is_sg(bs)) { 269449ca6259SFam Zheng goto early_exit; 269549ca6259SFam Zheng } 269649ca6259SFam Zheng 26973783fa3dSPaolo Bonzini qemu_co_mutex_lock(&bs->reqs_lock); 2698d73415a3SStefan Hajnoczi current_gen = qatomic_read(&bs->write_gen); 26993ff2f67aSEvgeny Yakovlev 27003ff2f67aSEvgeny Yakovlev /* Wait until any previous flushes are completed */ 270199723548SPaolo Bonzini while (bs->active_flush_req) { 27023783fa3dSPaolo Bonzini qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock); 27033ff2f67aSEvgeny Yakovlev } 27043ff2f67aSEvgeny Yakovlev 27053783fa3dSPaolo Bonzini /* Flushes reach this point in nondecreasing current_gen order. */ 270699723548SPaolo Bonzini bs->active_flush_req = true; 27073783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&bs->reqs_lock); 27083ff2f67aSEvgeny Yakovlev 2709c32b82afSPavel Dovgalyuk /* Write back all layers by calling one driver function */ 2710c32b82afSPavel Dovgalyuk if (bs->drv->bdrv_co_flush) { 2711c32b82afSPavel Dovgalyuk ret = bs->drv->bdrv_co_flush(bs); 2712c32b82afSPavel Dovgalyuk goto out; 2713c32b82afSPavel Dovgalyuk } 2714c32b82afSPavel Dovgalyuk 271561007b31SStefan Hajnoczi /* Write back cached data to the OS even with cache=unsafe */ 2716883833e2SMax Reitz BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_OS); 271761007b31SStefan Hajnoczi if (bs->drv->bdrv_co_flush_to_os) { 271861007b31SStefan Hajnoczi ret = bs->drv->bdrv_co_flush_to_os(bs); 271961007b31SStefan Hajnoczi if (ret < 0) { 2720cdb5e315SFam Zheng goto out; 272161007b31SStefan Hajnoczi } 272261007b31SStefan Hajnoczi } 272361007b31SStefan Hajnoczi 272461007b31SStefan Hajnoczi /* But don't actually force it to the disk with cache=unsafe */ 272561007b31SStefan Hajnoczi if (bs->open_flags & BDRV_O_NO_FLUSH) { 2726883833e2SMax Reitz goto flush_children; 272761007b31SStefan Hajnoczi } 272861007b31SStefan Hajnoczi 27293ff2f67aSEvgeny Yakovlev /* Check if we really need to flush anything */ 27303ff2f67aSEvgeny Yakovlev if (bs->flushed_gen == current_gen) { 2731883833e2SMax Reitz goto flush_children; 27323ff2f67aSEvgeny Yakovlev } 27333ff2f67aSEvgeny Yakovlev 2734883833e2SMax Reitz BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_DISK); 2735d470ad42SMax Reitz if (!bs->drv) { 2736d470ad42SMax Reitz /* bs->drv->bdrv_co_flush() might have ejected the BDS 2737d470ad42SMax Reitz * (even in case of apparent success) */ 2738d470ad42SMax Reitz ret = -ENOMEDIUM; 2739d470ad42SMax Reitz goto out; 2740d470ad42SMax Reitz } 274161007b31SStefan Hajnoczi if (bs->drv->bdrv_co_flush_to_disk) { 274261007b31SStefan Hajnoczi ret = bs->drv->bdrv_co_flush_to_disk(bs); 274361007b31SStefan Hajnoczi } else if (bs->drv->bdrv_aio_flush) { 274461007b31SStefan Hajnoczi BlockAIOCB *acb; 274561007b31SStefan Hajnoczi CoroutineIOCompletion co = { 274661007b31SStefan Hajnoczi .coroutine = qemu_coroutine_self(), 274761007b31SStefan Hajnoczi }; 274861007b31SStefan Hajnoczi 274961007b31SStefan Hajnoczi acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co); 275061007b31SStefan Hajnoczi if (acb == NULL) { 275161007b31SStefan Hajnoczi ret = -EIO; 275261007b31SStefan Hajnoczi } else { 275361007b31SStefan Hajnoczi qemu_coroutine_yield(); 275461007b31SStefan Hajnoczi ret = co.ret; 275561007b31SStefan Hajnoczi } 275661007b31SStefan Hajnoczi } else { 275761007b31SStefan Hajnoczi /* 275861007b31SStefan Hajnoczi * Some block drivers always operate in either writethrough or unsafe 275961007b31SStefan Hajnoczi * mode and don't support bdrv_flush therefore. Usually qemu doesn't 276061007b31SStefan Hajnoczi * know how the server works (because the behaviour is hardcoded or 276161007b31SStefan Hajnoczi * depends on server-side configuration), so we can't ensure that 276261007b31SStefan Hajnoczi * everything is safe on disk. Returning an error doesn't work because 276361007b31SStefan Hajnoczi * that would break guests even if the server operates in writethrough 276461007b31SStefan Hajnoczi * mode. 276561007b31SStefan Hajnoczi * 276661007b31SStefan Hajnoczi * Let's hope the user knows what he's doing. 276761007b31SStefan Hajnoczi */ 276861007b31SStefan Hajnoczi ret = 0; 276961007b31SStefan Hajnoczi } 27703ff2f67aSEvgeny Yakovlev 277161007b31SStefan Hajnoczi if (ret < 0) { 2772cdb5e315SFam Zheng goto out; 277361007b31SStefan Hajnoczi } 277461007b31SStefan Hajnoczi 277561007b31SStefan Hajnoczi /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH 277661007b31SStefan Hajnoczi * in the case of cache=unsafe, so there are no useless flushes. 277761007b31SStefan Hajnoczi */ 2778883833e2SMax Reitz flush_children: 2779883833e2SMax Reitz ret = 0; 2780883833e2SMax Reitz QLIST_FOREACH(child, &bs->children, next) { 2781883833e2SMax Reitz if (child->perm & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED)) { 2782883833e2SMax Reitz int this_child_ret = bdrv_co_flush(child->bs); 2783883833e2SMax Reitz if (!ret) { 2784883833e2SMax Reitz ret = this_child_ret; 2785883833e2SMax Reitz } 2786883833e2SMax Reitz } 2787883833e2SMax Reitz } 2788883833e2SMax Reitz 2789cdb5e315SFam Zheng out: 27903ff2f67aSEvgeny Yakovlev /* Notify any pending flushes that we have completed */ 2791e6af1e08SKevin Wolf if (ret == 0) { 27923ff2f67aSEvgeny Yakovlev bs->flushed_gen = current_gen; 2793e6af1e08SKevin Wolf } 27943783fa3dSPaolo Bonzini 27953783fa3dSPaolo Bonzini qemu_co_mutex_lock(&bs->reqs_lock); 279699723548SPaolo Bonzini bs->active_flush_req = false; 2797156af3acSDenis V. Lunev /* Return value is ignored - it's ok if wait queue is empty */ 2798156af3acSDenis V. Lunev qemu_co_queue_next(&bs->flush_queue); 27993783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&bs->reqs_lock); 28003ff2f67aSEvgeny Yakovlev 280149ca6259SFam Zheng early_exit: 280299723548SPaolo Bonzini bdrv_dec_in_flight(bs); 2803cdb5e315SFam Zheng return ret; 280461007b31SStefan Hajnoczi } 280561007b31SStefan Hajnoczi 2806d93e5726SVladimir Sementsov-Ogievskiy int coroutine_fn bdrv_co_pdiscard(BdrvChild *child, int64_t offset, 2807d93e5726SVladimir Sementsov-Ogievskiy int64_t bytes) 280861007b31SStefan Hajnoczi { 2809b1066c87SFam Zheng BdrvTrackedRequest req; 28109f1963b3SEric Blake int max_pdiscard, ret; 28113482b9bcSEric Blake int head, tail, align; 28120b9fd3f4SFam Zheng BlockDriverState *bs = child->bs; 281361007b31SStefan Hajnoczi 2814d93e5726SVladimir Sementsov-Ogievskiy if (!bs || !bs->drv || !bdrv_is_inserted(bs)) { 281561007b31SStefan Hajnoczi return -ENOMEDIUM; 281661007b31SStefan Hajnoczi } 281761007b31SStefan Hajnoczi 2818d6883bc9SVladimir Sementsov-Ogievskiy if (bdrv_has_readonly_bitmaps(bs)) { 2819d6883bc9SVladimir Sementsov-Ogievskiy return -EPERM; 2820d6883bc9SVladimir Sementsov-Ogievskiy } 2821d6883bc9SVladimir Sementsov-Ogievskiy 28228b117001SVladimir Sementsov-Ogievskiy ret = bdrv_check_request(offset, bytes); 28238b117001SVladimir Sementsov-Ogievskiy if (ret < 0) { 28248b117001SVladimir Sementsov-Ogievskiy return ret; 282561007b31SStefan Hajnoczi } 282661007b31SStefan Hajnoczi 282761007b31SStefan Hajnoczi /* Do nothing if disabled. */ 282861007b31SStefan Hajnoczi if (!(bs->open_flags & BDRV_O_UNMAP)) { 282961007b31SStefan Hajnoczi return 0; 283061007b31SStefan Hajnoczi } 283161007b31SStefan Hajnoczi 283202aefe43SEric Blake if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) { 283361007b31SStefan Hajnoczi return 0; 283461007b31SStefan Hajnoczi } 283561007b31SStefan Hajnoczi 28363482b9bcSEric Blake /* Discard is advisory, but some devices track and coalesce 28373482b9bcSEric Blake * unaligned requests, so we must pass everything down rather than 28383482b9bcSEric Blake * round here. Still, most devices will just silently ignore 28393482b9bcSEric Blake * unaligned requests (by returning -ENOTSUP), so we must fragment 28403482b9bcSEric Blake * the request accordingly. */ 284102aefe43SEric Blake align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment); 2842b8d0a980SEric Blake assert(align % bs->bl.request_alignment == 0); 2843b8d0a980SEric Blake head = offset % align; 2844f5a5ca79SManos Pitsidianakis tail = (offset + bytes) % align; 28459f1963b3SEric Blake 284699723548SPaolo Bonzini bdrv_inc_in_flight(bs); 2847f5a5ca79SManos Pitsidianakis tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD); 284850824995SFam Zheng 284900695c27SFam Zheng ret = bdrv_co_write_req_prepare(child, offset, bytes, &req, 0); 2850ec050f77SDenis V. Lunev if (ret < 0) { 2851ec050f77SDenis V. Lunev goto out; 2852ec050f77SDenis V. Lunev } 2853ec050f77SDenis V. Lunev 28549f1963b3SEric Blake max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT_MAX), 28559f1963b3SEric Blake align); 28563482b9bcSEric Blake assert(max_pdiscard >= bs->bl.request_alignment); 28579f1963b3SEric Blake 2858f5a5ca79SManos Pitsidianakis while (bytes > 0) { 2859d93e5726SVladimir Sementsov-Ogievskiy int64_t num = bytes; 28603482b9bcSEric Blake 28613482b9bcSEric Blake if (head) { 28623482b9bcSEric Blake /* Make small requests to get to alignment boundaries. */ 2863f5a5ca79SManos Pitsidianakis num = MIN(bytes, align - head); 28643482b9bcSEric Blake if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) { 28653482b9bcSEric Blake num %= bs->bl.request_alignment; 28663482b9bcSEric Blake } 28673482b9bcSEric Blake head = (head + num) % align; 28683482b9bcSEric Blake assert(num < max_pdiscard); 28693482b9bcSEric Blake } else if (tail) { 28703482b9bcSEric Blake if (num > align) { 28713482b9bcSEric Blake /* Shorten the request to the last aligned cluster. */ 28723482b9bcSEric Blake num -= tail; 28733482b9bcSEric Blake } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) && 28743482b9bcSEric Blake tail > bs->bl.request_alignment) { 28753482b9bcSEric Blake tail %= bs->bl.request_alignment; 28763482b9bcSEric Blake num -= tail; 28773482b9bcSEric Blake } 28783482b9bcSEric Blake } 28793482b9bcSEric Blake /* limit request size */ 28803482b9bcSEric Blake if (num > max_pdiscard) { 28813482b9bcSEric Blake num = max_pdiscard; 28823482b9bcSEric Blake } 288361007b31SStefan Hajnoczi 2884d470ad42SMax Reitz if (!bs->drv) { 2885d470ad42SMax Reitz ret = -ENOMEDIUM; 2886d470ad42SMax Reitz goto out; 2887d470ad42SMax Reitz } 288847a5486dSEric Blake if (bs->drv->bdrv_co_pdiscard) { 288947a5486dSEric Blake ret = bs->drv->bdrv_co_pdiscard(bs, offset, num); 289061007b31SStefan Hajnoczi } else { 289161007b31SStefan Hajnoczi BlockAIOCB *acb; 289261007b31SStefan Hajnoczi CoroutineIOCompletion co = { 289361007b31SStefan Hajnoczi .coroutine = qemu_coroutine_self(), 289461007b31SStefan Hajnoczi }; 289561007b31SStefan Hajnoczi 28964da444a0SEric Blake acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num, 289761007b31SStefan Hajnoczi bdrv_co_io_em_complete, &co); 289861007b31SStefan Hajnoczi if (acb == NULL) { 2899b1066c87SFam Zheng ret = -EIO; 2900b1066c87SFam Zheng goto out; 290161007b31SStefan Hajnoczi } else { 290261007b31SStefan Hajnoczi qemu_coroutine_yield(); 290361007b31SStefan Hajnoczi ret = co.ret; 290461007b31SStefan Hajnoczi } 290561007b31SStefan Hajnoczi } 290661007b31SStefan Hajnoczi if (ret && ret != -ENOTSUP) { 2907b1066c87SFam Zheng goto out; 290861007b31SStefan Hajnoczi } 290961007b31SStefan Hajnoczi 29109f1963b3SEric Blake offset += num; 2911f5a5ca79SManos Pitsidianakis bytes -= num; 291261007b31SStefan Hajnoczi } 2913b1066c87SFam Zheng ret = 0; 2914b1066c87SFam Zheng out: 291500695c27SFam Zheng bdrv_co_write_req_finish(child, req.offset, req.bytes, &req, ret); 2916b1066c87SFam Zheng tracked_request_end(&req); 291799723548SPaolo Bonzini bdrv_dec_in_flight(bs); 2918b1066c87SFam Zheng return ret; 291961007b31SStefan Hajnoczi } 292061007b31SStefan Hajnoczi 292148af776aSKevin Wolf int bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf) 292261007b31SStefan Hajnoczi { 292361007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 29245c5ae76aSFam Zheng CoroutineIOCompletion co = { 29255c5ae76aSFam Zheng .coroutine = qemu_coroutine_self(), 29265c5ae76aSFam Zheng }; 29275c5ae76aSFam Zheng BlockAIOCB *acb; 292861007b31SStefan Hajnoczi 292999723548SPaolo Bonzini bdrv_inc_in_flight(bs); 293016a389dcSKevin Wolf if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) { 29315c5ae76aSFam Zheng co.ret = -ENOTSUP; 29325c5ae76aSFam Zheng goto out; 29335c5ae76aSFam Zheng } 29345c5ae76aSFam Zheng 293516a389dcSKevin Wolf if (drv->bdrv_co_ioctl) { 293616a389dcSKevin Wolf co.ret = drv->bdrv_co_ioctl(bs, req, buf); 293716a389dcSKevin Wolf } else { 29385c5ae76aSFam Zheng acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co); 29395c5ae76aSFam Zheng if (!acb) { 2940c8a9fd80SFam Zheng co.ret = -ENOTSUP; 2941c8a9fd80SFam Zheng goto out; 29425c5ae76aSFam Zheng } 29435c5ae76aSFam Zheng qemu_coroutine_yield(); 294416a389dcSKevin Wolf } 29455c5ae76aSFam Zheng out: 294699723548SPaolo Bonzini bdrv_dec_in_flight(bs); 29475c5ae76aSFam Zheng return co.ret; 29485c5ae76aSFam Zheng } 29495c5ae76aSFam Zheng 295061007b31SStefan Hajnoczi void *qemu_blockalign(BlockDriverState *bs, size_t size) 295161007b31SStefan Hajnoczi { 295261007b31SStefan Hajnoczi return qemu_memalign(bdrv_opt_mem_align(bs), size); 295361007b31SStefan Hajnoczi } 295461007b31SStefan Hajnoczi 295561007b31SStefan Hajnoczi void *qemu_blockalign0(BlockDriverState *bs, size_t size) 295661007b31SStefan Hajnoczi { 295761007b31SStefan Hajnoczi return memset(qemu_blockalign(bs, size), 0, size); 295861007b31SStefan Hajnoczi } 295961007b31SStefan Hajnoczi 296061007b31SStefan Hajnoczi void *qemu_try_blockalign(BlockDriverState *bs, size_t size) 296161007b31SStefan Hajnoczi { 296261007b31SStefan Hajnoczi size_t align = bdrv_opt_mem_align(bs); 296361007b31SStefan Hajnoczi 296461007b31SStefan Hajnoczi /* Ensure that NULL is never returned on success */ 296561007b31SStefan Hajnoczi assert(align > 0); 296661007b31SStefan Hajnoczi if (size == 0) { 296761007b31SStefan Hajnoczi size = align; 296861007b31SStefan Hajnoczi } 296961007b31SStefan Hajnoczi 297061007b31SStefan Hajnoczi return qemu_try_memalign(align, size); 297161007b31SStefan Hajnoczi } 297261007b31SStefan Hajnoczi 297361007b31SStefan Hajnoczi void *qemu_try_blockalign0(BlockDriverState *bs, size_t size) 297461007b31SStefan Hajnoczi { 297561007b31SStefan Hajnoczi void *mem = qemu_try_blockalign(bs, size); 297661007b31SStefan Hajnoczi 297761007b31SStefan Hajnoczi if (mem) { 297861007b31SStefan Hajnoczi memset(mem, 0, size); 297961007b31SStefan Hajnoczi } 298061007b31SStefan Hajnoczi 298161007b31SStefan Hajnoczi return mem; 298261007b31SStefan Hajnoczi } 298361007b31SStefan Hajnoczi 298461007b31SStefan Hajnoczi /* 298561007b31SStefan Hajnoczi * Check if all memory in this vector is sector aligned. 298661007b31SStefan Hajnoczi */ 298761007b31SStefan Hajnoczi bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov) 298861007b31SStefan Hajnoczi { 298961007b31SStefan Hajnoczi int i; 29904196d2f0SDenis V. Lunev size_t alignment = bdrv_min_mem_align(bs); 299161007b31SStefan Hajnoczi 299261007b31SStefan Hajnoczi for (i = 0; i < qiov->niov; i++) { 299361007b31SStefan Hajnoczi if ((uintptr_t) qiov->iov[i].iov_base % alignment) { 299461007b31SStefan Hajnoczi return false; 299561007b31SStefan Hajnoczi } 299661007b31SStefan Hajnoczi if (qiov->iov[i].iov_len % alignment) { 299761007b31SStefan Hajnoczi return false; 299861007b31SStefan Hajnoczi } 299961007b31SStefan Hajnoczi } 300061007b31SStefan Hajnoczi 300161007b31SStefan Hajnoczi return true; 300261007b31SStefan Hajnoczi } 300361007b31SStefan Hajnoczi 300461007b31SStefan Hajnoczi void bdrv_add_before_write_notifier(BlockDriverState *bs, 300561007b31SStefan Hajnoczi NotifierWithReturn *notifier) 300661007b31SStefan Hajnoczi { 300761007b31SStefan Hajnoczi notifier_with_return_list_add(&bs->before_write_notifiers, notifier); 300861007b31SStefan Hajnoczi } 300961007b31SStefan Hajnoczi 301061007b31SStefan Hajnoczi void bdrv_io_plug(BlockDriverState *bs) 301161007b31SStefan Hajnoczi { 30126b98bd64SPaolo Bonzini BdrvChild *child; 30136b98bd64SPaolo Bonzini 30146b98bd64SPaolo Bonzini QLIST_FOREACH(child, &bs->children, next) { 30156b98bd64SPaolo Bonzini bdrv_io_plug(child->bs); 30166b98bd64SPaolo Bonzini } 30176b98bd64SPaolo Bonzini 3018d73415a3SStefan Hajnoczi if (qatomic_fetch_inc(&bs->io_plugged) == 0) { 301961007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 302061007b31SStefan Hajnoczi if (drv && drv->bdrv_io_plug) { 302161007b31SStefan Hajnoczi drv->bdrv_io_plug(bs); 30226b98bd64SPaolo Bonzini } 302361007b31SStefan Hajnoczi } 302461007b31SStefan Hajnoczi } 302561007b31SStefan Hajnoczi 302661007b31SStefan Hajnoczi void bdrv_io_unplug(BlockDriverState *bs) 302761007b31SStefan Hajnoczi { 30286b98bd64SPaolo Bonzini BdrvChild *child; 30296b98bd64SPaolo Bonzini 30306b98bd64SPaolo Bonzini assert(bs->io_plugged); 3031d73415a3SStefan Hajnoczi if (qatomic_fetch_dec(&bs->io_plugged) == 1) { 303261007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 303361007b31SStefan Hajnoczi if (drv && drv->bdrv_io_unplug) { 303461007b31SStefan Hajnoczi drv->bdrv_io_unplug(bs); 303561007b31SStefan Hajnoczi } 303661007b31SStefan Hajnoczi } 303761007b31SStefan Hajnoczi 30386b98bd64SPaolo Bonzini QLIST_FOREACH(child, &bs->children, next) { 30396b98bd64SPaolo Bonzini bdrv_io_unplug(child->bs); 30406b98bd64SPaolo Bonzini } 30416b98bd64SPaolo Bonzini } 304223d0ba93SFam Zheng 304323d0ba93SFam Zheng void bdrv_register_buf(BlockDriverState *bs, void *host, size_t size) 304423d0ba93SFam Zheng { 304523d0ba93SFam Zheng BdrvChild *child; 304623d0ba93SFam Zheng 304723d0ba93SFam Zheng if (bs->drv && bs->drv->bdrv_register_buf) { 304823d0ba93SFam Zheng bs->drv->bdrv_register_buf(bs, host, size); 304923d0ba93SFam Zheng } 305023d0ba93SFam Zheng QLIST_FOREACH(child, &bs->children, next) { 305123d0ba93SFam Zheng bdrv_register_buf(child->bs, host, size); 305223d0ba93SFam Zheng } 305323d0ba93SFam Zheng } 305423d0ba93SFam Zheng 305523d0ba93SFam Zheng void bdrv_unregister_buf(BlockDriverState *bs, void *host) 305623d0ba93SFam Zheng { 305723d0ba93SFam Zheng BdrvChild *child; 305823d0ba93SFam Zheng 305923d0ba93SFam Zheng if (bs->drv && bs->drv->bdrv_unregister_buf) { 306023d0ba93SFam Zheng bs->drv->bdrv_unregister_buf(bs, host); 306123d0ba93SFam Zheng } 306223d0ba93SFam Zheng QLIST_FOREACH(child, &bs->children, next) { 306323d0ba93SFam Zheng bdrv_unregister_buf(child->bs, host); 306423d0ba93SFam Zheng } 306523d0ba93SFam Zheng } 3066fcc67678SFam Zheng 306767b51fb9SVladimir Sementsov-Ogievskiy static int coroutine_fn bdrv_co_copy_range_internal( 306867b51fb9SVladimir Sementsov-Ogievskiy BdrvChild *src, uint64_t src_offset, BdrvChild *dst, 306967b51fb9SVladimir Sementsov-Ogievskiy uint64_t dst_offset, uint64_t bytes, 307067b51fb9SVladimir Sementsov-Ogievskiy BdrvRequestFlags read_flags, BdrvRequestFlags write_flags, 3071fcc67678SFam Zheng bool recurse_src) 3072fcc67678SFam Zheng { 3073999658a0SVladimir Sementsov-Ogievskiy BdrvTrackedRequest req; 3074fcc67678SFam Zheng int ret; 3075fcc67678SFam Zheng 3076fe0480d6SKevin Wolf /* TODO We can support BDRV_REQ_NO_FALLBACK here */ 3077fe0480d6SKevin Wolf assert(!(read_flags & BDRV_REQ_NO_FALLBACK)); 3078fe0480d6SKevin Wolf assert(!(write_flags & BDRV_REQ_NO_FALLBACK)); 3079fe0480d6SKevin Wolf 3080f4dad307SVladimir Sementsov-Ogievskiy if (!dst || !dst->bs || !bdrv_is_inserted(dst->bs)) { 3081fcc67678SFam Zheng return -ENOMEDIUM; 3082fcc67678SFam Zheng } 30838b117001SVladimir Sementsov-Ogievskiy ret = bdrv_check_request32(dst_offset, bytes); 3084fcc67678SFam Zheng if (ret) { 3085fcc67678SFam Zheng return ret; 3086fcc67678SFam Zheng } 308767b51fb9SVladimir Sementsov-Ogievskiy if (write_flags & BDRV_REQ_ZERO_WRITE) { 308867b51fb9SVladimir Sementsov-Ogievskiy return bdrv_co_pwrite_zeroes(dst, dst_offset, bytes, write_flags); 3089fcc67678SFam Zheng } 3090fcc67678SFam Zheng 3091f4dad307SVladimir Sementsov-Ogievskiy if (!src || !src->bs || !bdrv_is_inserted(src->bs)) { 3092d4d3e5a0SFam Zheng return -ENOMEDIUM; 3093d4d3e5a0SFam Zheng } 30948b117001SVladimir Sementsov-Ogievskiy ret = bdrv_check_request32(src_offset, bytes); 3095d4d3e5a0SFam Zheng if (ret) { 3096d4d3e5a0SFam Zheng return ret; 3097d4d3e5a0SFam Zheng } 3098d4d3e5a0SFam Zheng 3099fcc67678SFam Zheng if (!src->bs->drv->bdrv_co_copy_range_from 3100fcc67678SFam Zheng || !dst->bs->drv->bdrv_co_copy_range_to 3101fcc67678SFam Zheng || src->bs->encrypted || dst->bs->encrypted) { 3102fcc67678SFam Zheng return -ENOTSUP; 3103fcc67678SFam Zheng } 3104999658a0SVladimir Sementsov-Ogievskiy 3105999658a0SVladimir Sementsov-Ogievskiy if (recurse_src) { 3106d4d3e5a0SFam Zheng bdrv_inc_in_flight(src->bs); 3107999658a0SVladimir Sementsov-Ogievskiy tracked_request_begin(&req, src->bs, src_offset, bytes, 3108999658a0SVladimir Sementsov-Ogievskiy BDRV_TRACKED_READ); 310937aec7d7SFam Zheng 311009d2f948SVladimir Sementsov-Ogievskiy /* BDRV_REQ_SERIALISING is only for write operation */ 311109d2f948SVladimir Sementsov-Ogievskiy assert(!(read_flags & BDRV_REQ_SERIALISING)); 3112304d9d7fSMax Reitz bdrv_wait_serialising_requests(&req); 3113999658a0SVladimir Sementsov-Ogievskiy 311437aec7d7SFam Zheng ret = src->bs->drv->bdrv_co_copy_range_from(src->bs, 3115fcc67678SFam Zheng src, src_offset, 3116fcc67678SFam Zheng dst, dst_offset, 311767b51fb9SVladimir Sementsov-Ogievskiy bytes, 311867b51fb9SVladimir Sementsov-Ogievskiy read_flags, write_flags); 3119999658a0SVladimir Sementsov-Ogievskiy 3120999658a0SVladimir Sementsov-Ogievskiy tracked_request_end(&req); 3121999658a0SVladimir Sementsov-Ogievskiy bdrv_dec_in_flight(src->bs); 3122fcc67678SFam Zheng } else { 3123999658a0SVladimir Sementsov-Ogievskiy bdrv_inc_in_flight(dst->bs); 3124999658a0SVladimir Sementsov-Ogievskiy tracked_request_begin(&req, dst->bs, dst_offset, bytes, 3125999658a0SVladimir Sementsov-Ogievskiy BDRV_TRACKED_WRITE); 31260eb1e891SFam Zheng ret = bdrv_co_write_req_prepare(dst, dst_offset, bytes, &req, 31270eb1e891SFam Zheng write_flags); 31280eb1e891SFam Zheng if (!ret) { 312937aec7d7SFam Zheng ret = dst->bs->drv->bdrv_co_copy_range_to(dst->bs, 3130fcc67678SFam Zheng src, src_offset, 3131fcc67678SFam Zheng dst, dst_offset, 313267b51fb9SVladimir Sementsov-Ogievskiy bytes, 313367b51fb9SVladimir Sementsov-Ogievskiy read_flags, write_flags); 31340eb1e891SFam Zheng } 31350eb1e891SFam Zheng bdrv_co_write_req_finish(dst, dst_offset, bytes, &req, ret); 3136999658a0SVladimir Sementsov-Ogievskiy tracked_request_end(&req); 3137d4d3e5a0SFam Zheng bdrv_dec_in_flight(dst->bs); 3138999658a0SVladimir Sementsov-Ogievskiy } 3139999658a0SVladimir Sementsov-Ogievskiy 314037aec7d7SFam Zheng return ret; 3141fcc67678SFam Zheng } 3142fcc67678SFam Zheng 3143fcc67678SFam Zheng /* Copy range from @src to @dst. 3144fcc67678SFam Zheng * 3145fcc67678SFam Zheng * See the comment of bdrv_co_copy_range for the parameter and return value 3146fcc67678SFam Zheng * semantics. */ 3147fcc67678SFam Zheng int coroutine_fn bdrv_co_copy_range_from(BdrvChild *src, uint64_t src_offset, 3148fcc67678SFam Zheng BdrvChild *dst, uint64_t dst_offset, 314967b51fb9SVladimir Sementsov-Ogievskiy uint64_t bytes, 315067b51fb9SVladimir Sementsov-Ogievskiy BdrvRequestFlags read_flags, 315167b51fb9SVladimir Sementsov-Ogievskiy BdrvRequestFlags write_flags) 3152fcc67678SFam Zheng { 3153ecc983a5SFam Zheng trace_bdrv_co_copy_range_from(src, src_offset, dst, dst_offset, bytes, 3154ecc983a5SFam Zheng read_flags, write_flags); 3155fcc67678SFam Zheng return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset, 315667b51fb9SVladimir Sementsov-Ogievskiy bytes, read_flags, write_flags, true); 3157fcc67678SFam Zheng } 3158fcc67678SFam Zheng 3159fcc67678SFam Zheng /* Copy range from @src to @dst. 3160fcc67678SFam Zheng * 3161fcc67678SFam Zheng * See the comment of bdrv_co_copy_range for the parameter and return value 3162fcc67678SFam Zheng * semantics. */ 3163fcc67678SFam Zheng int coroutine_fn bdrv_co_copy_range_to(BdrvChild *src, uint64_t src_offset, 3164fcc67678SFam Zheng BdrvChild *dst, uint64_t dst_offset, 316567b51fb9SVladimir Sementsov-Ogievskiy uint64_t bytes, 316667b51fb9SVladimir Sementsov-Ogievskiy BdrvRequestFlags read_flags, 316767b51fb9SVladimir Sementsov-Ogievskiy BdrvRequestFlags write_flags) 3168fcc67678SFam Zheng { 3169ecc983a5SFam Zheng trace_bdrv_co_copy_range_to(src, src_offset, dst, dst_offset, bytes, 3170ecc983a5SFam Zheng read_flags, write_flags); 3171fcc67678SFam Zheng return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset, 317267b51fb9SVladimir Sementsov-Ogievskiy bytes, read_flags, write_flags, false); 3173fcc67678SFam Zheng } 3174fcc67678SFam Zheng 3175fcc67678SFam Zheng int coroutine_fn bdrv_co_copy_range(BdrvChild *src, uint64_t src_offset, 3176fcc67678SFam Zheng BdrvChild *dst, uint64_t dst_offset, 317767b51fb9SVladimir Sementsov-Ogievskiy uint64_t bytes, BdrvRequestFlags read_flags, 317867b51fb9SVladimir Sementsov-Ogievskiy BdrvRequestFlags write_flags) 3179fcc67678SFam Zheng { 318037aec7d7SFam Zheng return bdrv_co_copy_range_from(src, src_offset, 3181fcc67678SFam Zheng dst, dst_offset, 318267b51fb9SVladimir Sementsov-Ogievskiy bytes, read_flags, write_flags); 3183fcc67678SFam Zheng } 31843d9f2d2aSKevin Wolf 31853d9f2d2aSKevin Wolf static void bdrv_parent_cb_resize(BlockDriverState *bs) 31863d9f2d2aSKevin Wolf { 31873d9f2d2aSKevin Wolf BdrvChild *c; 31883d9f2d2aSKevin Wolf QLIST_FOREACH(c, &bs->parents, next_parent) { 3189bd86fb99SMax Reitz if (c->klass->resize) { 3190bd86fb99SMax Reitz c->klass->resize(c); 31913d9f2d2aSKevin Wolf } 31923d9f2d2aSKevin Wolf } 31933d9f2d2aSKevin Wolf } 31943d9f2d2aSKevin Wolf 31953d9f2d2aSKevin Wolf /** 31963d9f2d2aSKevin Wolf * Truncate file to 'offset' bytes (needed only for file protocols) 3197c80d8b06SMax Reitz * 3198c80d8b06SMax Reitz * If 'exact' is true, the file must be resized to exactly the given 3199c80d8b06SMax Reitz * 'offset'. Otherwise, it is sufficient for the node to be at least 3200c80d8b06SMax Reitz * 'offset' bytes in length. 32013d9f2d2aSKevin Wolf */ 3202c80d8b06SMax Reitz int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset, bool exact, 32037b8e4857SKevin Wolf PreallocMode prealloc, BdrvRequestFlags flags, 32047b8e4857SKevin Wolf Error **errp) 32053d9f2d2aSKevin Wolf { 32063d9f2d2aSKevin Wolf BlockDriverState *bs = child->bs; 320723b93525SMax Reitz BdrvChild *filtered, *backing; 32083d9f2d2aSKevin Wolf BlockDriver *drv = bs->drv; 32091bc5f09fSKevin Wolf BdrvTrackedRequest req; 32101bc5f09fSKevin Wolf int64_t old_size, new_bytes; 32113d9f2d2aSKevin Wolf int ret; 32123d9f2d2aSKevin Wolf 32133d9f2d2aSKevin Wolf 32143d9f2d2aSKevin Wolf /* if bs->drv == NULL, bs is closed, so there's nothing to do here */ 32153d9f2d2aSKevin Wolf if (!drv) { 32163d9f2d2aSKevin Wolf error_setg(errp, "No medium inserted"); 32173d9f2d2aSKevin Wolf return -ENOMEDIUM; 32183d9f2d2aSKevin Wolf } 32193d9f2d2aSKevin Wolf if (offset < 0) { 32203d9f2d2aSKevin Wolf error_setg(errp, "Image size cannot be negative"); 32213d9f2d2aSKevin Wolf return -EINVAL; 32223d9f2d2aSKevin Wolf } 32233d9f2d2aSKevin Wolf 32248b117001SVladimir Sementsov-Ogievskiy ret = bdrv_check_request(offset, 0); 32258b117001SVladimir Sementsov-Ogievskiy if (ret < 0) { 32268b117001SVladimir Sementsov-Ogievskiy error_setg(errp, "Required too big image size, it must be not greater " 32278b117001SVladimir Sementsov-Ogievskiy "than %" PRId64, BDRV_MAX_LENGTH); 32288b117001SVladimir Sementsov-Ogievskiy return ret; 32298b117001SVladimir Sementsov-Ogievskiy } 32308b117001SVladimir Sementsov-Ogievskiy 32311bc5f09fSKevin Wolf old_size = bdrv_getlength(bs); 32321bc5f09fSKevin Wolf if (old_size < 0) { 32331bc5f09fSKevin Wolf error_setg_errno(errp, -old_size, "Failed to get old image size"); 32341bc5f09fSKevin Wolf return old_size; 32351bc5f09fSKevin Wolf } 32361bc5f09fSKevin Wolf 32371bc5f09fSKevin Wolf if (offset > old_size) { 32381bc5f09fSKevin Wolf new_bytes = offset - old_size; 32391bc5f09fSKevin Wolf } else { 32401bc5f09fSKevin Wolf new_bytes = 0; 32411bc5f09fSKevin Wolf } 32421bc5f09fSKevin Wolf 32433d9f2d2aSKevin Wolf bdrv_inc_in_flight(bs); 32445416a11eSFam Zheng tracked_request_begin(&req, bs, offset - new_bytes, new_bytes, 32455416a11eSFam Zheng BDRV_TRACKED_TRUNCATE); 32461bc5f09fSKevin Wolf 32471bc5f09fSKevin Wolf /* If we are growing the image and potentially using preallocation for the 32481bc5f09fSKevin Wolf * new area, we need to make sure that no write requests are made to it 32491bc5f09fSKevin Wolf * concurrently or they might be overwritten by preallocation. */ 32501bc5f09fSKevin Wolf if (new_bytes) { 3251304d9d7fSMax Reitz bdrv_mark_request_serialising(&req, 1); 3252cd47d792SFam Zheng } 3253cd47d792SFam Zheng if (bs->read_only) { 3254cd47d792SFam Zheng error_setg(errp, "Image is read-only"); 3255cd47d792SFam Zheng ret = -EACCES; 3256cd47d792SFam Zheng goto out; 3257cd47d792SFam Zheng } 3258cd47d792SFam Zheng ret = bdrv_co_write_req_prepare(child, offset - new_bytes, new_bytes, &req, 3259cd47d792SFam Zheng 0); 3260cd47d792SFam Zheng if (ret < 0) { 3261cd47d792SFam Zheng error_setg_errno(errp, -ret, 3262cd47d792SFam Zheng "Failed to prepare request for truncation"); 3263cd47d792SFam Zheng goto out; 32641bc5f09fSKevin Wolf } 32653d9f2d2aSKevin Wolf 326693393e69SMax Reitz filtered = bdrv_filter_child(bs); 326723b93525SMax Reitz backing = bdrv_cow_child(bs); 326893393e69SMax Reitz 3269955c7d66SKevin Wolf /* 3270955c7d66SKevin Wolf * If the image has a backing file that is large enough that it would 3271955c7d66SKevin Wolf * provide data for the new area, we cannot leave it unallocated because 3272955c7d66SKevin Wolf * then the backing file content would become visible. Instead, zero-fill 3273955c7d66SKevin Wolf * the new area. 3274955c7d66SKevin Wolf * 3275955c7d66SKevin Wolf * Note that if the image has a backing file, but was opened without the 3276955c7d66SKevin Wolf * backing file, taking care of keeping things consistent with that backing 3277955c7d66SKevin Wolf * file is the user's responsibility. 3278955c7d66SKevin Wolf */ 327923b93525SMax Reitz if (new_bytes && backing) { 3280955c7d66SKevin Wolf int64_t backing_len; 3281955c7d66SKevin Wolf 328223b93525SMax Reitz backing_len = bdrv_getlength(backing->bs); 3283955c7d66SKevin Wolf if (backing_len < 0) { 3284955c7d66SKevin Wolf ret = backing_len; 3285955c7d66SKevin Wolf error_setg_errno(errp, -ret, "Could not get backing file size"); 3286955c7d66SKevin Wolf goto out; 3287955c7d66SKevin Wolf } 3288955c7d66SKevin Wolf 3289955c7d66SKevin Wolf if (backing_len > old_size) { 3290955c7d66SKevin Wolf flags |= BDRV_REQ_ZERO_WRITE; 3291955c7d66SKevin Wolf } 3292955c7d66SKevin Wolf } 3293955c7d66SKevin Wolf 32946b7e8f8bSMax Reitz if (drv->bdrv_co_truncate) { 329592b92799SKevin Wolf if (flags & ~bs->supported_truncate_flags) { 329692b92799SKevin Wolf error_setg(errp, "Block driver does not support requested flags"); 329792b92799SKevin Wolf ret = -ENOTSUP; 329892b92799SKevin Wolf goto out; 329992b92799SKevin Wolf } 330092b92799SKevin Wolf ret = drv->bdrv_co_truncate(bs, offset, exact, prealloc, flags, errp); 330193393e69SMax Reitz } else if (filtered) { 330293393e69SMax Reitz ret = bdrv_co_truncate(filtered, offset, exact, prealloc, flags, errp); 33036b7e8f8bSMax Reitz } else { 33043d9f2d2aSKevin Wolf error_setg(errp, "Image format driver does not support resize"); 33053d9f2d2aSKevin Wolf ret = -ENOTSUP; 33063d9f2d2aSKevin Wolf goto out; 33073d9f2d2aSKevin Wolf } 33083d9f2d2aSKevin Wolf if (ret < 0) { 33093d9f2d2aSKevin Wolf goto out; 33103d9f2d2aSKevin Wolf } 33116b7e8f8bSMax Reitz 33123d9f2d2aSKevin Wolf ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS); 33133d9f2d2aSKevin Wolf if (ret < 0) { 33143d9f2d2aSKevin Wolf error_setg_errno(errp, -ret, "Could not refresh total sector count"); 33153d9f2d2aSKevin Wolf } else { 33163d9f2d2aSKevin Wolf offset = bs->total_sectors * BDRV_SECTOR_SIZE; 33173d9f2d2aSKevin Wolf } 3318cd47d792SFam Zheng /* It's possible that truncation succeeded but refresh_total_sectors 3319cd47d792SFam Zheng * failed, but the latter doesn't affect how we should finish the request. 3320cd47d792SFam Zheng * Pass 0 as the last parameter so that dirty bitmaps etc. are handled. */ 3321cd47d792SFam Zheng bdrv_co_write_req_finish(child, offset - new_bytes, new_bytes, &req, 0); 33223d9f2d2aSKevin Wolf 33233d9f2d2aSKevin Wolf out: 33241bc5f09fSKevin Wolf tracked_request_end(&req); 33253d9f2d2aSKevin Wolf bdrv_dec_in_flight(bs); 33261bc5f09fSKevin Wolf 33273d9f2d2aSKevin Wolf return ret; 33283d9f2d2aSKevin Wolf } 3329