161007b31SStefan Hajnoczi /* 261007b31SStefan Hajnoczi * Block layer I/O functions 361007b31SStefan Hajnoczi * 461007b31SStefan Hajnoczi * Copyright (c) 2003 Fabrice Bellard 561007b31SStefan Hajnoczi * 661007b31SStefan Hajnoczi * Permission is hereby granted, free of charge, to any person obtaining a copy 761007b31SStefan Hajnoczi * of this software and associated documentation files (the "Software"), to deal 861007b31SStefan Hajnoczi * in the Software without restriction, including without limitation the rights 961007b31SStefan Hajnoczi * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 1061007b31SStefan Hajnoczi * copies of the Software, and to permit persons to whom the Software is 1161007b31SStefan Hajnoczi * furnished to do so, subject to the following conditions: 1261007b31SStefan Hajnoczi * 1361007b31SStefan Hajnoczi * The above copyright notice and this permission notice shall be included in 1461007b31SStefan Hajnoczi * all copies or substantial portions of the Software. 1561007b31SStefan Hajnoczi * 1661007b31SStefan Hajnoczi * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 1761007b31SStefan Hajnoczi * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 1861007b31SStefan Hajnoczi * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 1961007b31SStefan Hajnoczi * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 2061007b31SStefan Hajnoczi * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 2161007b31SStefan Hajnoczi * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 2261007b31SStefan Hajnoczi * THE SOFTWARE. 2361007b31SStefan Hajnoczi */ 2461007b31SStefan Hajnoczi 2580c71a24SPeter Maydell #include "qemu/osdep.h" 2661007b31SStefan Hajnoczi #include "trace.h" 277f0e9da6SMax Reitz #include "sysemu/block-backend.h" 2861007b31SStefan Hajnoczi #include "block/blockjob.h" 29f321dcb5SPaolo Bonzini #include "block/blockjob_int.h" 3061007b31SStefan Hajnoczi #include "block/block_int.h" 31f348b6d1SVeronia Bahaa #include "qemu/cutils.h" 32da34e65cSMarkus Armbruster #include "qapi/error.h" 33d49b6836SMarkus Armbruster #include "qemu/error-report.h" 3461007b31SStefan Hajnoczi 3561007b31SStefan Hajnoczi #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */ 3661007b31SStefan Hajnoczi 37cb2e2878SEric Blake /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */ 38cb2e2878SEric Blake #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS) 39cb2e2878SEric Blake 40d05aa8bbSEric Blake static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 41f5a5ca79SManos Pitsidianakis int64_t offset, int bytes, BdrvRequestFlags flags); 4261007b31SStefan Hajnoczi 430152bf40SKevin Wolf void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore) 4461007b31SStefan Hajnoczi { 4502d21300SKevin Wolf BdrvChild *c, *next; 4627ccdd52SKevin Wolf 4702d21300SKevin Wolf QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) { 480152bf40SKevin Wolf if (c == ignore) { 490152bf40SKevin Wolf continue; 500152bf40SKevin Wolf } 51c2066af0SKevin Wolf if (c->role->drained_begin) { 52c2066af0SKevin Wolf c->role->drained_begin(c); 53c2066af0SKevin Wolf } 54ce0f1412SPaolo Bonzini } 55ce0f1412SPaolo Bonzini } 56ce0f1412SPaolo Bonzini 570152bf40SKevin Wolf void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore) 58ce0f1412SPaolo Bonzini { 5902d21300SKevin Wolf BdrvChild *c, *next; 6027ccdd52SKevin Wolf 6102d21300SKevin Wolf QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) { 620152bf40SKevin Wolf if (c == ignore) { 630152bf40SKevin Wolf continue; 640152bf40SKevin Wolf } 65c2066af0SKevin Wolf if (c->role->drained_end) { 66c2066af0SKevin Wolf c->role->drained_end(c); 6727ccdd52SKevin Wolf } 68c2066af0SKevin Wolf } 6961007b31SStefan Hajnoczi } 7061007b31SStefan Hajnoczi 71d9e0dfa2SEric Blake static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src) 72d9e0dfa2SEric Blake { 73d9e0dfa2SEric Blake dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer); 74d9e0dfa2SEric Blake dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer); 75d9e0dfa2SEric Blake dst->opt_mem_alignment = MAX(dst->opt_mem_alignment, 76d9e0dfa2SEric Blake src->opt_mem_alignment); 77d9e0dfa2SEric Blake dst->min_mem_alignment = MAX(dst->min_mem_alignment, 78d9e0dfa2SEric Blake src->min_mem_alignment); 79d9e0dfa2SEric Blake dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov); 80d9e0dfa2SEric Blake } 81d9e0dfa2SEric Blake 8261007b31SStefan Hajnoczi void bdrv_refresh_limits(BlockDriverState *bs, Error **errp) 8361007b31SStefan Hajnoczi { 8461007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 8561007b31SStefan Hajnoczi Error *local_err = NULL; 8661007b31SStefan Hajnoczi 8761007b31SStefan Hajnoczi memset(&bs->bl, 0, sizeof(bs->bl)); 8861007b31SStefan Hajnoczi 8961007b31SStefan Hajnoczi if (!drv) { 9061007b31SStefan Hajnoczi return; 9161007b31SStefan Hajnoczi } 9261007b31SStefan Hajnoczi 9379ba8c98SEric Blake /* Default alignment based on whether driver has byte interface */ 94a5b8dd2cSEric Blake bs->bl.request_alignment = drv->bdrv_co_preadv ? 1 : 512; 9579ba8c98SEric Blake 9661007b31SStefan Hajnoczi /* Take some limits from the children as a default */ 9761007b31SStefan Hajnoczi if (bs->file) { 989a4f4c31SKevin Wolf bdrv_refresh_limits(bs->file->bs, &local_err); 9961007b31SStefan Hajnoczi if (local_err) { 10061007b31SStefan Hajnoczi error_propagate(errp, local_err); 10161007b31SStefan Hajnoczi return; 10261007b31SStefan Hajnoczi } 103d9e0dfa2SEric Blake bdrv_merge_limits(&bs->bl, &bs->file->bs->bl); 10461007b31SStefan Hajnoczi } else { 1054196d2f0SDenis V. Lunev bs->bl.min_mem_alignment = 512; 106459b4e66SDenis V. Lunev bs->bl.opt_mem_alignment = getpagesize(); 107bd44feb7SStefan Hajnoczi 108bd44feb7SStefan Hajnoczi /* Safe default since most protocols use readv()/writev()/etc */ 109bd44feb7SStefan Hajnoczi bs->bl.max_iov = IOV_MAX; 11061007b31SStefan Hajnoczi } 11161007b31SStefan Hajnoczi 112760e0063SKevin Wolf if (bs->backing) { 113760e0063SKevin Wolf bdrv_refresh_limits(bs->backing->bs, &local_err); 11461007b31SStefan Hajnoczi if (local_err) { 11561007b31SStefan Hajnoczi error_propagate(errp, local_err); 11661007b31SStefan Hajnoczi return; 11761007b31SStefan Hajnoczi } 118d9e0dfa2SEric Blake bdrv_merge_limits(&bs->bl, &bs->backing->bs->bl); 11961007b31SStefan Hajnoczi } 12061007b31SStefan Hajnoczi 12161007b31SStefan Hajnoczi /* Then let the driver override it */ 12261007b31SStefan Hajnoczi if (drv->bdrv_refresh_limits) { 12361007b31SStefan Hajnoczi drv->bdrv_refresh_limits(bs, errp); 12461007b31SStefan Hajnoczi } 12561007b31SStefan Hajnoczi } 12661007b31SStefan Hajnoczi 12761007b31SStefan Hajnoczi /** 12861007b31SStefan Hajnoczi * The copy-on-read flag is actually a reference count so multiple users may 12961007b31SStefan Hajnoczi * use the feature without worrying about clobbering its previous state. 13061007b31SStefan Hajnoczi * Copy-on-read stays enabled until all users have called to disable it. 13161007b31SStefan Hajnoczi */ 13261007b31SStefan Hajnoczi void bdrv_enable_copy_on_read(BlockDriverState *bs) 13361007b31SStefan Hajnoczi { 134d3faa13eSPaolo Bonzini atomic_inc(&bs->copy_on_read); 13561007b31SStefan Hajnoczi } 13661007b31SStefan Hajnoczi 13761007b31SStefan Hajnoczi void bdrv_disable_copy_on_read(BlockDriverState *bs) 13861007b31SStefan Hajnoczi { 139d3faa13eSPaolo Bonzini int old = atomic_fetch_dec(&bs->copy_on_read); 140d3faa13eSPaolo Bonzini assert(old >= 1); 14161007b31SStefan Hajnoczi } 14261007b31SStefan Hajnoczi 14361124f03SPaolo Bonzini typedef struct { 14461124f03SPaolo Bonzini Coroutine *co; 14561124f03SPaolo Bonzini BlockDriverState *bs; 14661124f03SPaolo Bonzini bool done; 147481cad48SManos Pitsidianakis bool begin; 148b0165585SKevin Wolf bool recursive; 1490152bf40SKevin Wolf BdrvChild *parent; 15061124f03SPaolo Bonzini } BdrvCoDrainData; 15161124f03SPaolo Bonzini 15261124f03SPaolo Bonzini static void coroutine_fn bdrv_drain_invoke_entry(void *opaque) 15361124f03SPaolo Bonzini { 15461124f03SPaolo Bonzini BdrvCoDrainData *data = opaque; 15561124f03SPaolo Bonzini BlockDriverState *bs = data->bs; 15661124f03SPaolo Bonzini 157481cad48SManos Pitsidianakis if (data->begin) { 158f8ea8dacSManos Pitsidianakis bs->drv->bdrv_co_drain_begin(bs); 159481cad48SManos Pitsidianakis } else { 160481cad48SManos Pitsidianakis bs->drv->bdrv_co_drain_end(bs); 161481cad48SManos Pitsidianakis } 16261124f03SPaolo Bonzini 16361124f03SPaolo Bonzini /* Set data->done before reading bs->wakeup. */ 16461124f03SPaolo Bonzini atomic_mb_set(&data->done, true); 16561124f03SPaolo Bonzini bdrv_wakeup(bs); 16661124f03SPaolo Bonzini } 16761124f03SPaolo Bonzini 168db0289b9SKevin Wolf /* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */ 1697b6a3d35SKevin Wolf static void bdrv_drain_invoke(BlockDriverState *bs, bool begin, bool recursive) 17061124f03SPaolo Bonzini { 171db0289b9SKevin Wolf BdrvChild *child, *tmp; 172481cad48SManos Pitsidianakis BdrvCoDrainData data = { .bs = bs, .done = false, .begin = begin}; 17361124f03SPaolo Bonzini 174f8ea8dacSManos Pitsidianakis if (!bs->drv || (begin && !bs->drv->bdrv_co_drain_begin) || 175481cad48SManos Pitsidianakis (!begin && !bs->drv->bdrv_co_drain_end)) { 17661124f03SPaolo Bonzini return; 17761124f03SPaolo Bonzini } 17861124f03SPaolo Bonzini 17961124f03SPaolo Bonzini data.co = qemu_coroutine_create(bdrv_drain_invoke_entry, &data); 18061124f03SPaolo Bonzini bdrv_coroutine_enter(bs, data.co); 18161124f03SPaolo Bonzini BDRV_POLL_WHILE(bs, !data.done); 182db0289b9SKevin Wolf 1837b6a3d35SKevin Wolf if (recursive) { 184db0289b9SKevin Wolf QLIST_FOREACH_SAFE(child, &bs->children, next, tmp) { 1857b6a3d35SKevin Wolf bdrv_drain_invoke(child->bs, begin, true); 1867b6a3d35SKevin Wolf } 187db0289b9SKevin Wolf } 18861124f03SPaolo Bonzini } 18961124f03SPaolo Bonzini 19099c05de9SKevin Wolf static bool bdrv_drain_recurse(BlockDriverState *bs) 19167da1dc5SFam Zheng { 192178bd438SFam Zheng BdrvChild *child, *tmp; 193d42cf288SPaolo Bonzini bool waited; 194d42cf288SPaolo Bonzini 195481cad48SManos Pitsidianakis /* Wait for drained requests to finish */ 196481cad48SManos Pitsidianakis waited = BDRV_POLL_WHILE(bs, atomic_read(&bs->in_flight) > 0); 197d42cf288SPaolo Bonzini 198178bd438SFam Zheng QLIST_FOREACH_SAFE(child, &bs->children, next, tmp) { 199178bd438SFam Zheng BlockDriverState *bs = child->bs; 200178bd438SFam Zheng bool in_main_loop = 201178bd438SFam Zheng qemu_get_current_aio_context() == qemu_get_aio_context(); 202178bd438SFam Zheng assert(bs->refcnt > 0); 203178bd438SFam Zheng if (in_main_loop) { 204178bd438SFam Zheng /* In case the recursive bdrv_drain_recurse processes a 205178bd438SFam Zheng * block_job_defer_to_main_loop BH and modifies the graph, 206178bd438SFam Zheng * let's hold a reference to bs until we are done. 207178bd438SFam Zheng * 208178bd438SFam Zheng * IOThread doesn't have such a BH, and it is not safe to call 209178bd438SFam Zheng * bdrv_unref without BQL, so skip doing it there. 210178bd438SFam Zheng */ 211178bd438SFam Zheng bdrv_ref(bs); 212178bd438SFam Zheng } 21399c05de9SKevin Wolf waited |= bdrv_drain_recurse(bs); 214178bd438SFam Zheng if (in_main_loop) { 215178bd438SFam Zheng bdrv_unref(bs); 216178bd438SFam Zheng } 21767da1dc5SFam Zheng } 218d42cf288SPaolo Bonzini 219d42cf288SPaolo Bonzini return waited; 22067da1dc5SFam Zheng } 22167da1dc5SFam Zheng 222b0165585SKevin Wolf static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive, 223b0165585SKevin Wolf BdrvChild *parent); 224b0165585SKevin Wolf static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive, 225b0165585SKevin Wolf BdrvChild *parent); 2260152bf40SKevin Wolf 227a77fd4bbSFam Zheng static void bdrv_co_drain_bh_cb(void *opaque) 228a77fd4bbSFam Zheng { 229a77fd4bbSFam Zheng BdrvCoDrainData *data = opaque; 230a77fd4bbSFam Zheng Coroutine *co = data->co; 23199723548SPaolo Bonzini BlockDriverState *bs = data->bs; 232a77fd4bbSFam Zheng 23399723548SPaolo Bonzini bdrv_dec_in_flight(bs); 234481cad48SManos Pitsidianakis if (data->begin) { 235b0165585SKevin Wolf bdrv_do_drained_begin(bs, data->recursive, data->parent); 236481cad48SManos Pitsidianakis } else { 237b0165585SKevin Wolf bdrv_do_drained_end(bs, data->recursive, data->parent); 238481cad48SManos Pitsidianakis } 239481cad48SManos Pitsidianakis 240a77fd4bbSFam Zheng data->done = true; 2411919631eSPaolo Bonzini aio_co_wake(co); 242a77fd4bbSFam Zheng } 243a77fd4bbSFam Zheng 244481cad48SManos Pitsidianakis static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs, 245b0165585SKevin Wolf bool begin, bool recursive, 246b0165585SKevin Wolf BdrvChild *parent) 247a77fd4bbSFam Zheng { 248a77fd4bbSFam Zheng BdrvCoDrainData data; 249a77fd4bbSFam Zheng 250a77fd4bbSFam Zheng /* Calling bdrv_drain() from a BH ensures the current coroutine yields and 251a77fd4bbSFam Zheng * other coroutines run if they were queued from 252a77fd4bbSFam Zheng * qemu_co_queue_run_restart(). */ 253a77fd4bbSFam Zheng 254a77fd4bbSFam Zheng assert(qemu_in_coroutine()); 255a77fd4bbSFam Zheng data = (BdrvCoDrainData) { 256a77fd4bbSFam Zheng .co = qemu_coroutine_self(), 257a77fd4bbSFam Zheng .bs = bs, 258a77fd4bbSFam Zheng .done = false, 259481cad48SManos Pitsidianakis .begin = begin, 260b0165585SKevin Wolf .recursive = recursive, 2610152bf40SKevin Wolf .parent = parent, 262a77fd4bbSFam Zheng }; 26399723548SPaolo Bonzini bdrv_inc_in_flight(bs); 264fffb6e12SPaolo Bonzini aio_bh_schedule_oneshot(bdrv_get_aio_context(bs), 265fffb6e12SPaolo Bonzini bdrv_co_drain_bh_cb, &data); 266a77fd4bbSFam Zheng 267a77fd4bbSFam Zheng qemu_coroutine_yield(); 268a77fd4bbSFam Zheng /* If we are resumed from some other event (such as an aio completion or a 269a77fd4bbSFam Zheng * timer callback), it is a bug in the caller that should be fixed. */ 270a77fd4bbSFam Zheng assert(data.done); 271a77fd4bbSFam Zheng } 272a77fd4bbSFam Zheng 273d736f119SKevin Wolf void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive, 274b0165585SKevin Wolf BdrvChild *parent) 2756820643fSKevin Wolf { 276b0165585SKevin Wolf BdrvChild *child, *next; 277b0165585SKevin Wolf 278d42cf288SPaolo Bonzini if (qemu_in_coroutine()) { 279b0165585SKevin Wolf bdrv_co_yield_to_drain(bs, true, recursive, parent); 280d42cf288SPaolo Bonzini return; 281d42cf288SPaolo Bonzini } 282d42cf288SPaolo Bonzini 28360369b86SKevin Wolf /* Stop things in parent-to-child order */ 284414c2ec3SPaolo Bonzini if (atomic_fetch_inc(&bs->quiesce_counter) == 0) { 2856820643fSKevin Wolf aio_disable_external(bdrv_get_aio_context(bs)); 2866820643fSKevin Wolf } 2876820643fSKevin Wolf 2880152bf40SKevin Wolf bdrv_parent_drained_begin(bs, parent); 2897b6a3d35SKevin Wolf bdrv_drain_invoke(bs, true, false); 29099c05de9SKevin Wolf bdrv_drain_recurse(bs); 291b0165585SKevin Wolf 292b0165585SKevin Wolf if (recursive) { 293d736f119SKevin Wolf bs->recursive_quiesce_counter++; 294b0165585SKevin Wolf QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 295b0165585SKevin Wolf bdrv_do_drained_begin(child->bs, true, child); 296b0165585SKevin Wolf } 297b0165585SKevin Wolf } 2986820643fSKevin Wolf } 2996820643fSKevin Wolf 3000152bf40SKevin Wolf void bdrv_drained_begin(BlockDriverState *bs) 3010152bf40SKevin Wolf { 302b0165585SKevin Wolf bdrv_do_drained_begin(bs, false, NULL); 3030152bf40SKevin Wolf } 3040152bf40SKevin Wolf 305b0165585SKevin Wolf void bdrv_subtree_drained_begin(BlockDriverState *bs) 3066820643fSKevin Wolf { 307b0165585SKevin Wolf bdrv_do_drained_begin(bs, true, NULL); 308b0165585SKevin Wolf } 309b0165585SKevin Wolf 310d736f119SKevin Wolf void bdrv_do_drained_end(BlockDriverState *bs, bool recursive, 311b0165585SKevin Wolf BdrvChild *parent) 312b0165585SKevin Wolf { 313b0165585SKevin Wolf BdrvChild *child, *next; 3140f115168SKevin Wolf int old_quiesce_counter; 3150f115168SKevin Wolf 316481cad48SManos Pitsidianakis if (qemu_in_coroutine()) { 317b0165585SKevin Wolf bdrv_co_yield_to_drain(bs, false, recursive, parent); 318481cad48SManos Pitsidianakis return; 319481cad48SManos Pitsidianakis } 3206820643fSKevin Wolf assert(bs->quiesce_counter > 0); 3210f115168SKevin Wolf old_quiesce_counter = atomic_fetch_dec(&bs->quiesce_counter); 3226820643fSKevin Wolf 32360369b86SKevin Wolf /* Re-enable things in child-to-parent order */ 3247b6a3d35SKevin Wolf bdrv_drain_invoke(bs, false, false); 3250152bf40SKevin Wolf bdrv_parent_drained_end(bs, parent); 3260f115168SKevin Wolf if (old_quiesce_counter == 1) { 3276820643fSKevin Wolf aio_enable_external(bdrv_get_aio_context(bs)); 3286820643fSKevin Wolf } 329b0165585SKevin Wolf 330b0165585SKevin Wolf if (recursive) { 331d736f119SKevin Wolf bs->recursive_quiesce_counter--; 332b0165585SKevin Wolf QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 333b0165585SKevin Wolf bdrv_do_drained_end(child->bs, true, child); 334b0165585SKevin Wolf } 335b0165585SKevin Wolf } 3360f115168SKevin Wolf } 3376820643fSKevin Wolf 3380152bf40SKevin Wolf void bdrv_drained_end(BlockDriverState *bs) 3390152bf40SKevin Wolf { 340b0165585SKevin Wolf bdrv_do_drained_end(bs, false, NULL); 341b0165585SKevin Wolf } 342b0165585SKevin Wolf 343b0165585SKevin Wolf void bdrv_subtree_drained_end(BlockDriverState *bs) 344b0165585SKevin Wolf { 345b0165585SKevin Wolf bdrv_do_drained_end(bs, true, NULL); 3460152bf40SKevin Wolf } 3470152bf40SKevin Wolf 348d736f119SKevin Wolf void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent) 349d736f119SKevin Wolf { 350d736f119SKevin Wolf int i; 351d736f119SKevin Wolf 352d736f119SKevin Wolf for (i = 0; i < new_parent->recursive_quiesce_counter; i++) { 353d736f119SKevin Wolf bdrv_do_drained_begin(child->bs, true, child); 354d736f119SKevin Wolf } 355d736f119SKevin Wolf } 356d736f119SKevin Wolf 357d736f119SKevin Wolf void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent) 358d736f119SKevin Wolf { 359d736f119SKevin Wolf int i; 360d736f119SKevin Wolf 361d736f119SKevin Wolf for (i = 0; i < old_parent->recursive_quiesce_counter; i++) { 362d736f119SKevin Wolf bdrv_do_drained_end(child->bs, true, child); 363d736f119SKevin Wolf } 364d736f119SKevin Wolf } 365d736f119SKevin Wolf 36661007b31SStefan Hajnoczi /* 36767da1dc5SFam Zheng * Wait for pending requests to complete on a single BlockDriverState subtree, 36867da1dc5SFam Zheng * and suspend block driver's internal I/O until next request arrives. 36961007b31SStefan Hajnoczi * 37061007b31SStefan Hajnoczi * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState 37161007b31SStefan Hajnoczi * AioContext. 3727a63f3cdSStefan Hajnoczi * 3737a63f3cdSStefan Hajnoczi * Only this BlockDriverState's AioContext is run, so in-flight requests must 3747a63f3cdSStefan Hajnoczi * not depend on events in other AioContexts. In that case, use 3757a63f3cdSStefan Hajnoczi * bdrv_drain_all() instead. 37661007b31SStefan Hajnoczi */ 377b6e84c97SPaolo Bonzini void coroutine_fn bdrv_co_drain(BlockDriverState *bs) 378b6e84c97SPaolo Bonzini { 3796820643fSKevin Wolf assert(qemu_in_coroutine()); 3806820643fSKevin Wolf bdrv_drained_begin(bs); 3816820643fSKevin Wolf bdrv_drained_end(bs); 382b6e84c97SPaolo Bonzini } 383b6e84c97SPaolo Bonzini 38461007b31SStefan Hajnoczi void bdrv_drain(BlockDriverState *bs) 38561007b31SStefan Hajnoczi { 3866820643fSKevin Wolf bdrv_drained_begin(bs); 3876820643fSKevin Wolf bdrv_drained_end(bs); 38861007b31SStefan Hajnoczi } 38961007b31SStefan Hajnoczi 39061007b31SStefan Hajnoczi /* 39161007b31SStefan Hajnoczi * Wait for pending requests to complete across all BlockDriverStates 39261007b31SStefan Hajnoczi * 39361007b31SStefan Hajnoczi * This function does not flush data to disk, use bdrv_flush_all() for that 39461007b31SStefan Hajnoczi * after calling this function. 395c0778f66SAlberto Garcia * 396c0778f66SAlberto Garcia * This pauses all block jobs and disables external clients. It must 397c0778f66SAlberto Garcia * be paired with bdrv_drain_all_end(). 398c0778f66SAlberto Garcia * 399c0778f66SAlberto Garcia * NOTE: no new block jobs or BlockDriverStates can be created between 400c0778f66SAlberto Garcia * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls. 40161007b31SStefan Hajnoczi */ 402c0778f66SAlberto Garcia void bdrv_drain_all_begin(void) 40361007b31SStefan Hajnoczi { 40461007b31SStefan Hajnoczi /* Always run first iteration so any pending completion BHs run */ 40599723548SPaolo Bonzini bool waited = true; 4067c8eece4SKevin Wolf BlockDriverState *bs; 40788be7b4bSKevin Wolf BdrvNextIterator it; 408f406c03cSAlexander Yarygin GSList *aio_ctxs = NULL, *ctx; 40961007b31SStefan Hajnoczi 4109a7e86c8SKevin Wolf /* BDRV_POLL_WHILE() for a node can only be called from its own I/O thread 4119a7e86c8SKevin Wolf * or the main loop AioContext. We potentially use BDRV_POLL_WHILE() on 4129a7e86c8SKevin Wolf * nodes in several different AioContexts, so make sure we're in the main 4139a7e86c8SKevin Wolf * context. */ 4149a7e86c8SKevin Wolf assert(qemu_get_current_aio_context() == qemu_get_aio_context()); 4159a7e86c8SKevin Wolf 41688be7b4bSKevin Wolf for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 41761007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 41861007b31SStefan Hajnoczi 41960369b86SKevin Wolf /* Stop things in parent-to-child order */ 42061007b31SStefan Hajnoczi aio_context_acquire(aio_context); 421c0778f66SAlberto Garcia aio_disable_external(aio_context); 4220152bf40SKevin Wolf bdrv_parent_drained_begin(bs, NULL); 4237b6a3d35SKevin Wolf bdrv_drain_invoke(bs, true, true); 42461007b31SStefan Hajnoczi aio_context_release(aio_context); 425f406c03cSAlexander Yarygin 426764ba3aeSAlberto Garcia if (!g_slist_find(aio_ctxs, aio_context)) { 427f406c03cSAlexander Yarygin aio_ctxs = g_slist_prepend(aio_ctxs, aio_context); 428f406c03cSAlexander Yarygin } 42961007b31SStefan Hajnoczi } 43061007b31SStefan Hajnoczi 4317a63f3cdSStefan Hajnoczi /* Note that completion of an asynchronous I/O operation can trigger any 4327a63f3cdSStefan Hajnoczi * number of other I/O operations on other devices---for example a 4337a63f3cdSStefan Hajnoczi * coroutine can submit an I/O request to another device in response to 4347a63f3cdSStefan Hajnoczi * request completion. Therefore we must keep looping until there was no 4357a63f3cdSStefan Hajnoczi * more activity rather than simply draining each device independently. 4367a63f3cdSStefan Hajnoczi */ 43799723548SPaolo Bonzini while (waited) { 43899723548SPaolo Bonzini waited = false; 439f406c03cSAlexander Yarygin 440f406c03cSAlexander Yarygin for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) { 441f406c03cSAlexander Yarygin AioContext *aio_context = ctx->data; 44261007b31SStefan Hajnoczi 44361007b31SStefan Hajnoczi aio_context_acquire(aio_context); 44488be7b4bSKevin Wolf for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 445f406c03cSAlexander Yarygin if (aio_context == bdrv_get_aio_context(bs)) { 44699c05de9SKevin Wolf waited |= bdrv_drain_recurse(bs); 447f406c03cSAlexander Yarygin } 448f406c03cSAlexander Yarygin } 44961007b31SStefan Hajnoczi aio_context_release(aio_context); 45061007b31SStefan Hajnoczi } 45161007b31SStefan Hajnoczi } 45261007b31SStefan Hajnoczi 453c0778f66SAlberto Garcia g_slist_free(aio_ctxs); 454c0778f66SAlberto Garcia } 455c0778f66SAlberto Garcia 456c0778f66SAlberto Garcia void bdrv_drain_all_end(void) 457c0778f66SAlberto Garcia { 458c0778f66SAlberto Garcia BlockDriverState *bs; 459c0778f66SAlberto Garcia BdrvNextIterator it; 460c0778f66SAlberto Garcia 46188be7b4bSKevin Wolf for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 46261007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 46361007b31SStefan Hajnoczi 46460369b86SKevin Wolf /* Re-enable things in child-to-parent order */ 46561007b31SStefan Hajnoczi aio_context_acquire(aio_context); 4667b6a3d35SKevin Wolf bdrv_drain_invoke(bs, false, true); 4670152bf40SKevin Wolf bdrv_parent_drained_end(bs, NULL); 46860369b86SKevin Wolf aio_enable_external(aio_context); 46961007b31SStefan Hajnoczi aio_context_release(aio_context); 47061007b31SStefan Hajnoczi } 47161007b31SStefan Hajnoczi } 47261007b31SStefan Hajnoczi 473c0778f66SAlberto Garcia void bdrv_drain_all(void) 474c0778f66SAlberto Garcia { 475c0778f66SAlberto Garcia bdrv_drain_all_begin(); 476c0778f66SAlberto Garcia bdrv_drain_all_end(); 477c0778f66SAlberto Garcia } 478c0778f66SAlberto Garcia 47961007b31SStefan Hajnoczi /** 48061007b31SStefan Hajnoczi * Remove an active request from the tracked requests list 48161007b31SStefan Hajnoczi * 48261007b31SStefan Hajnoczi * This function should be called when a tracked request is completing. 48361007b31SStefan Hajnoczi */ 48461007b31SStefan Hajnoczi static void tracked_request_end(BdrvTrackedRequest *req) 48561007b31SStefan Hajnoczi { 48661007b31SStefan Hajnoczi if (req->serialising) { 48720fc71b2SPaolo Bonzini atomic_dec(&req->bs->serialising_in_flight); 48861007b31SStefan Hajnoczi } 48961007b31SStefan Hajnoczi 4903783fa3dSPaolo Bonzini qemu_co_mutex_lock(&req->bs->reqs_lock); 49161007b31SStefan Hajnoczi QLIST_REMOVE(req, list); 49261007b31SStefan Hajnoczi qemu_co_queue_restart_all(&req->wait_queue); 4933783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&req->bs->reqs_lock); 49461007b31SStefan Hajnoczi } 49561007b31SStefan Hajnoczi 49661007b31SStefan Hajnoczi /** 49761007b31SStefan Hajnoczi * Add an active request to the tracked requests list 49861007b31SStefan Hajnoczi */ 49961007b31SStefan Hajnoczi static void tracked_request_begin(BdrvTrackedRequest *req, 50061007b31SStefan Hajnoczi BlockDriverState *bs, 50161007b31SStefan Hajnoczi int64_t offset, 502ebde595cSFam Zheng unsigned int bytes, 503ebde595cSFam Zheng enum BdrvTrackedRequestType type) 50461007b31SStefan Hajnoczi { 50561007b31SStefan Hajnoczi *req = (BdrvTrackedRequest){ 50661007b31SStefan Hajnoczi .bs = bs, 50761007b31SStefan Hajnoczi .offset = offset, 50861007b31SStefan Hajnoczi .bytes = bytes, 509ebde595cSFam Zheng .type = type, 51061007b31SStefan Hajnoczi .co = qemu_coroutine_self(), 51161007b31SStefan Hajnoczi .serialising = false, 51261007b31SStefan Hajnoczi .overlap_offset = offset, 51361007b31SStefan Hajnoczi .overlap_bytes = bytes, 51461007b31SStefan Hajnoczi }; 51561007b31SStefan Hajnoczi 51661007b31SStefan Hajnoczi qemu_co_queue_init(&req->wait_queue); 51761007b31SStefan Hajnoczi 5183783fa3dSPaolo Bonzini qemu_co_mutex_lock(&bs->reqs_lock); 51961007b31SStefan Hajnoczi QLIST_INSERT_HEAD(&bs->tracked_requests, req, list); 5203783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&bs->reqs_lock); 52161007b31SStefan Hajnoczi } 52261007b31SStefan Hajnoczi 52361007b31SStefan Hajnoczi static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align) 52461007b31SStefan Hajnoczi { 52561007b31SStefan Hajnoczi int64_t overlap_offset = req->offset & ~(align - 1); 52661007b31SStefan Hajnoczi unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align) 52761007b31SStefan Hajnoczi - overlap_offset; 52861007b31SStefan Hajnoczi 52961007b31SStefan Hajnoczi if (!req->serialising) { 53020fc71b2SPaolo Bonzini atomic_inc(&req->bs->serialising_in_flight); 53161007b31SStefan Hajnoczi req->serialising = true; 53261007b31SStefan Hajnoczi } 53361007b31SStefan Hajnoczi 53461007b31SStefan Hajnoczi req->overlap_offset = MIN(req->overlap_offset, overlap_offset); 53561007b31SStefan Hajnoczi req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes); 53661007b31SStefan Hajnoczi } 53761007b31SStefan Hajnoczi 53861007b31SStefan Hajnoczi /** 539244483e6SKevin Wolf * Round a region to cluster boundaries 540244483e6SKevin Wolf */ 541244483e6SKevin Wolf void bdrv_round_to_clusters(BlockDriverState *bs, 5427cfd5275SEric Blake int64_t offset, int64_t bytes, 543244483e6SKevin Wolf int64_t *cluster_offset, 5447cfd5275SEric Blake int64_t *cluster_bytes) 545244483e6SKevin Wolf { 546244483e6SKevin Wolf BlockDriverInfo bdi; 547244483e6SKevin Wolf 548244483e6SKevin Wolf if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 549244483e6SKevin Wolf *cluster_offset = offset; 550244483e6SKevin Wolf *cluster_bytes = bytes; 551244483e6SKevin Wolf } else { 552244483e6SKevin Wolf int64_t c = bdi.cluster_size; 553244483e6SKevin Wolf *cluster_offset = QEMU_ALIGN_DOWN(offset, c); 554244483e6SKevin Wolf *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c); 555244483e6SKevin Wolf } 556244483e6SKevin Wolf } 557244483e6SKevin Wolf 55861007b31SStefan Hajnoczi static int bdrv_get_cluster_size(BlockDriverState *bs) 55961007b31SStefan Hajnoczi { 56061007b31SStefan Hajnoczi BlockDriverInfo bdi; 56161007b31SStefan Hajnoczi int ret; 56261007b31SStefan Hajnoczi 56361007b31SStefan Hajnoczi ret = bdrv_get_info(bs, &bdi); 56461007b31SStefan Hajnoczi if (ret < 0 || bdi.cluster_size == 0) { 565a5b8dd2cSEric Blake return bs->bl.request_alignment; 56661007b31SStefan Hajnoczi } else { 56761007b31SStefan Hajnoczi return bdi.cluster_size; 56861007b31SStefan Hajnoczi } 56961007b31SStefan Hajnoczi } 57061007b31SStefan Hajnoczi 57161007b31SStefan Hajnoczi static bool tracked_request_overlaps(BdrvTrackedRequest *req, 57261007b31SStefan Hajnoczi int64_t offset, unsigned int bytes) 57361007b31SStefan Hajnoczi { 57461007b31SStefan Hajnoczi /* aaaa bbbb */ 57561007b31SStefan Hajnoczi if (offset >= req->overlap_offset + req->overlap_bytes) { 57661007b31SStefan Hajnoczi return false; 57761007b31SStefan Hajnoczi } 57861007b31SStefan Hajnoczi /* bbbb aaaa */ 57961007b31SStefan Hajnoczi if (req->overlap_offset >= offset + bytes) { 58061007b31SStefan Hajnoczi return false; 58161007b31SStefan Hajnoczi } 58261007b31SStefan Hajnoczi return true; 58361007b31SStefan Hajnoczi } 58461007b31SStefan Hajnoczi 58599723548SPaolo Bonzini void bdrv_inc_in_flight(BlockDriverState *bs) 58699723548SPaolo Bonzini { 58799723548SPaolo Bonzini atomic_inc(&bs->in_flight); 58899723548SPaolo Bonzini } 58999723548SPaolo Bonzini 590c9d1a561SPaolo Bonzini static void dummy_bh_cb(void *opaque) 591c9d1a561SPaolo Bonzini { 592c9d1a561SPaolo Bonzini } 593c9d1a561SPaolo Bonzini 594c9d1a561SPaolo Bonzini void bdrv_wakeup(BlockDriverState *bs) 595c9d1a561SPaolo Bonzini { 596e2a6ae7fSPaolo Bonzini /* The barrier (or an atomic op) is in the caller. */ 597e2a6ae7fSPaolo Bonzini if (atomic_read(&bs->wakeup)) { 598c9d1a561SPaolo Bonzini aio_bh_schedule_oneshot(qemu_get_aio_context(), dummy_bh_cb, NULL); 599c9d1a561SPaolo Bonzini } 600c9d1a561SPaolo Bonzini } 601c9d1a561SPaolo Bonzini 60299723548SPaolo Bonzini void bdrv_dec_in_flight(BlockDriverState *bs) 60399723548SPaolo Bonzini { 60499723548SPaolo Bonzini atomic_dec(&bs->in_flight); 605c9d1a561SPaolo Bonzini bdrv_wakeup(bs); 60699723548SPaolo Bonzini } 60799723548SPaolo Bonzini 60861007b31SStefan Hajnoczi static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self) 60961007b31SStefan Hajnoczi { 61061007b31SStefan Hajnoczi BlockDriverState *bs = self->bs; 61161007b31SStefan Hajnoczi BdrvTrackedRequest *req; 61261007b31SStefan Hajnoczi bool retry; 61361007b31SStefan Hajnoczi bool waited = false; 61461007b31SStefan Hajnoczi 61520fc71b2SPaolo Bonzini if (!atomic_read(&bs->serialising_in_flight)) { 61661007b31SStefan Hajnoczi return false; 61761007b31SStefan Hajnoczi } 61861007b31SStefan Hajnoczi 61961007b31SStefan Hajnoczi do { 62061007b31SStefan Hajnoczi retry = false; 6213783fa3dSPaolo Bonzini qemu_co_mutex_lock(&bs->reqs_lock); 62261007b31SStefan Hajnoczi QLIST_FOREACH(req, &bs->tracked_requests, list) { 62361007b31SStefan Hajnoczi if (req == self || (!req->serialising && !self->serialising)) { 62461007b31SStefan Hajnoczi continue; 62561007b31SStefan Hajnoczi } 62661007b31SStefan Hajnoczi if (tracked_request_overlaps(req, self->overlap_offset, 62761007b31SStefan Hajnoczi self->overlap_bytes)) 62861007b31SStefan Hajnoczi { 62961007b31SStefan Hajnoczi /* Hitting this means there was a reentrant request, for 63061007b31SStefan Hajnoczi * example, a block driver issuing nested requests. This must 63161007b31SStefan Hajnoczi * never happen since it means deadlock. 63261007b31SStefan Hajnoczi */ 63361007b31SStefan Hajnoczi assert(qemu_coroutine_self() != req->co); 63461007b31SStefan Hajnoczi 63561007b31SStefan Hajnoczi /* If the request is already (indirectly) waiting for us, or 63661007b31SStefan Hajnoczi * will wait for us as soon as it wakes up, then just go on 63761007b31SStefan Hajnoczi * (instead of producing a deadlock in the former case). */ 63861007b31SStefan Hajnoczi if (!req->waiting_for) { 63961007b31SStefan Hajnoczi self->waiting_for = req; 6403783fa3dSPaolo Bonzini qemu_co_queue_wait(&req->wait_queue, &bs->reqs_lock); 64161007b31SStefan Hajnoczi self->waiting_for = NULL; 64261007b31SStefan Hajnoczi retry = true; 64361007b31SStefan Hajnoczi waited = true; 64461007b31SStefan Hajnoczi break; 64561007b31SStefan Hajnoczi } 64661007b31SStefan Hajnoczi } 64761007b31SStefan Hajnoczi } 6483783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&bs->reqs_lock); 64961007b31SStefan Hajnoczi } while (retry); 65061007b31SStefan Hajnoczi 65161007b31SStefan Hajnoczi return waited; 65261007b31SStefan Hajnoczi } 65361007b31SStefan Hajnoczi 65461007b31SStefan Hajnoczi static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset, 65561007b31SStefan Hajnoczi size_t size) 65661007b31SStefan Hajnoczi { 65761007b31SStefan Hajnoczi if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) { 65861007b31SStefan Hajnoczi return -EIO; 65961007b31SStefan Hajnoczi } 66061007b31SStefan Hajnoczi 66161007b31SStefan Hajnoczi if (!bdrv_is_inserted(bs)) { 66261007b31SStefan Hajnoczi return -ENOMEDIUM; 66361007b31SStefan Hajnoczi } 66461007b31SStefan Hajnoczi 66561007b31SStefan Hajnoczi if (offset < 0) { 66661007b31SStefan Hajnoczi return -EIO; 66761007b31SStefan Hajnoczi } 66861007b31SStefan Hajnoczi 66961007b31SStefan Hajnoczi return 0; 67061007b31SStefan Hajnoczi } 67161007b31SStefan Hajnoczi 67261007b31SStefan Hajnoczi typedef struct RwCo { 673e293b7a3SKevin Wolf BdrvChild *child; 67461007b31SStefan Hajnoczi int64_t offset; 67561007b31SStefan Hajnoczi QEMUIOVector *qiov; 67661007b31SStefan Hajnoczi bool is_write; 67761007b31SStefan Hajnoczi int ret; 67861007b31SStefan Hajnoczi BdrvRequestFlags flags; 67961007b31SStefan Hajnoczi } RwCo; 68061007b31SStefan Hajnoczi 68161007b31SStefan Hajnoczi static void coroutine_fn bdrv_rw_co_entry(void *opaque) 68261007b31SStefan Hajnoczi { 68361007b31SStefan Hajnoczi RwCo *rwco = opaque; 68461007b31SStefan Hajnoczi 68561007b31SStefan Hajnoczi if (!rwco->is_write) { 686a03ef88fSKevin Wolf rwco->ret = bdrv_co_preadv(rwco->child, rwco->offset, 68761007b31SStefan Hajnoczi rwco->qiov->size, rwco->qiov, 68861007b31SStefan Hajnoczi rwco->flags); 68961007b31SStefan Hajnoczi } else { 690a03ef88fSKevin Wolf rwco->ret = bdrv_co_pwritev(rwco->child, rwco->offset, 69161007b31SStefan Hajnoczi rwco->qiov->size, rwco->qiov, 69261007b31SStefan Hajnoczi rwco->flags); 69361007b31SStefan Hajnoczi } 69461007b31SStefan Hajnoczi } 69561007b31SStefan Hajnoczi 69661007b31SStefan Hajnoczi /* 69761007b31SStefan Hajnoczi * Process a vectored synchronous request using coroutines 69861007b31SStefan Hajnoczi */ 699e293b7a3SKevin Wolf static int bdrv_prwv_co(BdrvChild *child, int64_t offset, 70061007b31SStefan Hajnoczi QEMUIOVector *qiov, bool is_write, 70161007b31SStefan Hajnoczi BdrvRequestFlags flags) 70261007b31SStefan Hajnoczi { 70361007b31SStefan Hajnoczi Coroutine *co; 70461007b31SStefan Hajnoczi RwCo rwco = { 705e293b7a3SKevin Wolf .child = child, 70661007b31SStefan Hajnoczi .offset = offset, 70761007b31SStefan Hajnoczi .qiov = qiov, 70861007b31SStefan Hajnoczi .is_write = is_write, 70961007b31SStefan Hajnoczi .ret = NOT_DONE, 71061007b31SStefan Hajnoczi .flags = flags, 71161007b31SStefan Hajnoczi }; 71261007b31SStefan Hajnoczi 71361007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 71461007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 71561007b31SStefan Hajnoczi bdrv_rw_co_entry(&rwco); 71661007b31SStefan Hajnoczi } else { 7170b8b8753SPaolo Bonzini co = qemu_coroutine_create(bdrv_rw_co_entry, &rwco); 718e92f0e19SFam Zheng bdrv_coroutine_enter(child->bs, co); 71988b062c2SPaolo Bonzini BDRV_POLL_WHILE(child->bs, rwco.ret == NOT_DONE); 72061007b31SStefan Hajnoczi } 72161007b31SStefan Hajnoczi return rwco.ret; 72261007b31SStefan Hajnoczi } 72361007b31SStefan Hajnoczi 72461007b31SStefan Hajnoczi /* 72561007b31SStefan Hajnoczi * Process a synchronous request using coroutines 72661007b31SStefan Hajnoczi */ 727e293b7a3SKevin Wolf static int bdrv_rw_co(BdrvChild *child, int64_t sector_num, uint8_t *buf, 72861007b31SStefan Hajnoczi int nb_sectors, bool is_write, BdrvRequestFlags flags) 72961007b31SStefan Hajnoczi { 73061007b31SStefan Hajnoczi QEMUIOVector qiov; 73161007b31SStefan Hajnoczi struct iovec iov = { 73261007b31SStefan Hajnoczi .iov_base = (void *)buf, 73361007b31SStefan Hajnoczi .iov_len = nb_sectors * BDRV_SECTOR_SIZE, 73461007b31SStefan Hajnoczi }; 73561007b31SStefan Hajnoczi 73661007b31SStefan Hajnoczi if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 73761007b31SStefan Hajnoczi return -EINVAL; 73861007b31SStefan Hajnoczi } 73961007b31SStefan Hajnoczi 74061007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 741e293b7a3SKevin Wolf return bdrv_prwv_co(child, sector_num << BDRV_SECTOR_BITS, 74261007b31SStefan Hajnoczi &qiov, is_write, flags); 74361007b31SStefan Hajnoczi } 74461007b31SStefan Hajnoczi 74561007b31SStefan Hajnoczi /* return < 0 if error. See bdrv_write() for the return codes */ 746fbcbbf4eSKevin Wolf int bdrv_read(BdrvChild *child, int64_t sector_num, 74761007b31SStefan Hajnoczi uint8_t *buf, int nb_sectors) 74861007b31SStefan Hajnoczi { 749e293b7a3SKevin Wolf return bdrv_rw_co(child, sector_num, buf, nb_sectors, false, 0); 75061007b31SStefan Hajnoczi } 75161007b31SStefan Hajnoczi 75261007b31SStefan Hajnoczi /* Return < 0 if error. Important errors are: 75361007b31SStefan Hajnoczi -EIO generic I/O error (may happen for all errors) 75461007b31SStefan Hajnoczi -ENOMEDIUM No media inserted. 75561007b31SStefan Hajnoczi -EINVAL Invalid sector number or nb_sectors 75661007b31SStefan Hajnoczi -EACCES Trying to write a read-only device 75761007b31SStefan Hajnoczi */ 75818d51c4bSKevin Wolf int bdrv_write(BdrvChild *child, int64_t sector_num, 75961007b31SStefan Hajnoczi const uint8_t *buf, int nb_sectors) 76061007b31SStefan Hajnoczi { 761e293b7a3SKevin Wolf return bdrv_rw_co(child, sector_num, (uint8_t *)buf, nb_sectors, true, 0); 76261007b31SStefan Hajnoczi } 76361007b31SStefan Hajnoczi 764720ff280SKevin Wolf int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset, 765f5a5ca79SManos Pitsidianakis int bytes, BdrvRequestFlags flags) 76661007b31SStefan Hajnoczi { 76774021bc4SEric Blake QEMUIOVector qiov; 76874021bc4SEric Blake struct iovec iov = { 76974021bc4SEric Blake .iov_base = NULL, 770f5a5ca79SManos Pitsidianakis .iov_len = bytes, 77174021bc4SEric Blake }; 77274021bc4SEric Blake 77374021bc4SEric Blake qemu_iovec_init_external(&qiov, &iov, 1); 774e293b7a3SKevin Wolf return bdrv_prwv_co(child, offset, &qiov, true, 77561007b31SStefan Hajnoczi BDRV_REQ_ZERO_WRITE | flags); 77661007b31SStefan Hajnoczi } 77761007b31SStefan Hajnoczi 77861007b31SStefan Hajnoczi /* 77974021bc4SEric Blake * Completely zero out a block device with the help of bdrv_pwrite_zeroes. 78061007b31SStefan Hajnoczi * The operation is sped up by checking the block status and only writing 78161007b31SStefan Hajnoczi * zeroes to the device if they currently do not return zeroes. Optional 78274021bc4SEric Blake * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP, 783465fe887SEric Blake * BDRV_REQ_FUA). 78461007b31SStefan Hajnoczi * 78561007b31SStefan Hajnoczi * Returns < 0 on error, 0 on success. For error codes see bdrv_write(). 78661007b31SStefan Hajnoczi */ 787720ff280SKevin Wolf int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags) 78861007b31SStefan Hajnoczi { 789237d78f8SEric Blake int ret; 790237d78f8SEric Blake int64_t target_size, bytes, offset = 0; 791720ff280SKevin Wolf BlockDriverState *bs = child->bs; 79261007b31SStefan Hajnoczi 7937286d610SEric Blake target_size = bdrv_getlength(bs); 7947286d610SEric Blake if (target_size < 0) { 7957286d610SEric Blake return target_size; 79661007b31SStefan Hajnoczi } 79761007b31SStefan Hajnoczi 79861007b31SStefan Hajnoczi for (;;) { 7997286d610SEric Blake bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES); 8007286d610SEric Blake if (bytes <= 0) { 80161007b31SStefan Hajnoczi return 0; 80261007b31SStefan Hajnoczi } 803237d78f8SEric Blake ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL); 80461007b31SStefan Hajnoczi if (ret < 0) { 8057286d610SEric Blake error_report("error getting block status at offset %" PRId64 ": %s", 8067286d610SEric Blake offset, strerror(-ret)); 80761007b31SStefan Hajnoczi return ret; 80861007b31SStefan Hajnoczi } 80961007b31SStefan Hajnoczi if (ret & BDRV_BLOCK_ZERO) { 810237d78f8SEric Blake offset += bytes; 81161007b31SStefan Hajnoczi continue; 81261007b31SStefan Hajnoczi } 813237d78f8SEric Blake ret = bdrv_pwrite_zeroes(child, offset, bytes, flags); 81461007b31SStefan Hajnoczi if (ret < 0) { 8157286d610SEric Blake error_report("error writing zeroes at offset %" PRId64 ": %s", 8167286d610SEric Blake offset, strerror(-ret)); 81761007b31SStefan Hajnoczi return ret; 81861007b31SStefan Hajnoczi } 819237d78f8SEric Blake offset += bytes; 82061007b31SStefan Hajnoczi } 82161007b31SStefan Hajnoczi } 82261007b31SStefan Hajnoczi 823cf2ab8fcSKevin Wolf int bdrv_preadv(BdrvChild *child, int64_t offset, QEMUIOVector *qiov) 824f1e84741SKevin Wolf { 825f1e84741SKevin Wolf int ret; 826f1e84741SKevin Wolf 827e293b7a3SKevin Wolf ret = bdrv_prwv_co(child, offset, qiov, false, 0); 828f1e84741SKevin Wolf if (ret < 0) { 829f1e84741SKevin Wolf return ret; 830f1e84741SKevin Wolf } 831f1e84741SKevin Wolf 832f1e84741SKevin Wolf return qiov->size; 833f1e84741SKevin Wolf } 834f1e84741SKevin Wolf 835cf2ab8fcSKevin Wolf int bdrv_pread(BdrvChild *child, int64_t offset, void *buf, int bytes) 83661007b31SStefan Hajnoczi { 83761007b31SStefan Hajnoczi QEMUIOVector qiov; 83861007b31SStefan Hajnoczi struct iovec iov = { 83961007b31SStefan Hajnoczi .iov_base = (void *)buf, 84061007b31SStefan Hajnoczi .iov_len = bytes, 84161007b31SStefan Hajnoczi }; 84261007b31SStefan Hajnoczi 84361007b31SStefan Hajnoczi if (bytes < 0) { 84461007b31SStefan Hajnoczi return -EINVAL; 84561007b31SStefan Hajnoczi } 84661007b31SStefan Hajnoczi 84761007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 848cf2ab8fcSKevin Wolf return bdrv_preadv(child, offset, &qiov); 84961007b31SStefan Hajnoczi } 85061007b31SStefan Hajnoczi 851d9ca2ea2SKevin Wolf int bdrv_pwritev(BdrvChild *child, int64_t offset, QEMUIOVector *qiov) 85261007b31SStefan Hajnoczi { 85361007b31SStefan Hajnoczi int ret; 85461007b31SStefan Hajnoczi 855e293b7a3SKevin Wolf ret = bdrv_prwv_co(child, offset, qiov, true, 0); 85661007b31SStefan Hajnoczi if (ret < 0) { 85761007b31SStefan Hajnoczi return ret; 85861007b31SStefan Hajnoczi } 85961007b31SStefan Hajnoczi 86061007b31SStefan Hajnoczi return qiov->size; 86161007b31SStefan Hajnoczi } 86261007b31SStefan Hajnoczi 863d9ca2ea2SKevin Wolf int bdrv_pwrite(BdrvChild *child, int64_t offset, const void *buf, int bytes) 86461007b31SStefan Hajnoczi { 86561007b31SStefan Hajnoczi QEMUIOVector qiov; 86661007b31SStefan Hajnoczi struct iovec iov = { 86761007b31SStefan Hajnoczi .iov_base = (void *) buf, 86861007b31SStefan Hajnoczi .iov_len = bytes, 86961007b31SStefan Hajnoczi }; 87061007b31SStefan Hajnoczi 87161007b31SStefan Hajnoczi if (bytes < 0) { 87261007b31SStefan Hajnoczi return -EINVAL; 87361007b31SStefan Hajnoczi } 87461007b31SStefan Hajnoczi 87561007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 876d9ca2ea2SKevin Wolf return bdrv_pwritev(child, offset, &qiov); 87761007b31SStefan Hajnoczi } 87861007b31SStefan Hajnoczi 87961007b31SStefan Hajnoczi /* 88061007b31SStefan Hajnoczi * Writes to the file and ensures that no writes are reordered across this 88161007b31SStefan Hajnoczi * request (acts as a barrier) 88261007b31SStefan Hajnoczi * 88361007b31SStefan Hajnoczi * Returns 0 on success, -errno in error cases. 88461007b31SStefan Hajnoczi */ 885d9ca2ea2SKevin Wolf int bdrv_pwrite_sync(BdrvChild *child, int64_t offset, 88661007b31SStefan Hajnoczi const void *buf, int count) 88761007b31SStefan Hajnoczi { 88861007b31SStefan Hajnoczi int ret; 88961007b31SStefan Hajnoczi 890d9ca2ea2SKevin Wolf ret = bdrv_pwrite(child, offset, buf, count); 89161007b31SStefan Hajnoczi if (ret < 0) { 89261007b31SStefan Hajnoczi return ret; 89361007b31SStefan Hajnoczi } 89461007b31SStefan Hajnoczi 895d9ca2ea2SKevin Wolf ret = bdrv_flush(child->bs); 896855a6a93SKevin Wolf if (ret < 0) { 897855a6a93SKevin Wolf return ret; 89861007b31SStefan Hajnoczi } 89961007b31SStefan Hajnoczi 90061007b31SStefan Hajnoczi return 0; 90161007b31SStefan Hajnoczi } 90261007b31SStefan Hajnoczi 90308844473SKevin Wolf typedef struct CoroutineIOCompletion { 90408844473SKevin Wolf Coroutine *coroutine; 90508844473SKevin Wolf int ret; 90608844473SKevin Wolf } CoroutineIOCompletion; 90708844473SKevin Wolf 90808844473SKevin Wolf static void bdrv_co_io_em_complete(void *opaque, int ret) 90908844473SKevin Wolf { 91008844473SKevin Wolf CoroutineIOCompletion *co = opaque; 91108844473SKevin Wolf 91208844473SKevin Wolf co->ret = ret; 913b9e413ddSPaolo Bonzini aio_co_wake(co->coroutine); 91408844473SKevin Wolf } 91508844473SKevin Wolf 916166fe960SKevin Wolf static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs, 917166fe960SKevin Wolf uint64_t offset, uint64_t bytes, 918166fe960SKevin Wolf QEMUIOVector *qiov, int flags) 919166fe960SKevin Wolf { 920166fe960SKevin Wolf BlockDriver *drv = bs->drv; 9213fb06697SKevin Wolf int64_t sector_num; 9223fb06697SKevin Wolf unsigned int nb_sectors; 9233fb06697SKevin Wolf 924fa166538SEric Blake assert(!(flags & ~BDRV_REQ_MASK)); 925fa166538SEric Blake 926d470ad42SMax Reitz if (!drv) { 927d470ad42SMax Reitz return -ENOMEDIUM; 928d470ad42SMax Reitz } 929d470ad42SMax Reitz 9303fb06697SKevin Wolf if (drv->bdrv_co_preadv) { 9313fb06697SKevin Wolf return drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags); 9323fb06697SKevin Wolf } 9333fb06697SKevin Wolf 9343fb06697SKevin Wolf sector_num = offset >> BDRV_SECTOR_BITS; 9353fb06697SKevin Wolf nb_sectors = bytes >> BDRV_SECTOR_BITS; 936166fe960SKevin Wolf 937166fe960SKevin Wolf assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 938166fe960SKevin Wolf assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 939166fe960SKevin Wolf assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS); 940166fe960SKevin Wolf 94108844473SKevin Wolf if (drv->bdrv_co_readv) { 942166fe960SKevin Wolf return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov); 94308844473SKevin Wolf } else { 94408844473SKevin Wolf BlockAIOCB *acb; 94508844473SKevin Wolf CoroutineIOCompletion co = { 94608844473SKevin Wolf .coroutine = qemu_coroutine_self(), 94708844473SKevin Wolf }; 94808844473SKevin Wolf 94908844473SKevin Wolf acb = bs->drv->bdrv_aio_readv(bs, sector_num, qiov, nb_sectors, 95008844473SKevin Wolf bdrv_co_io_em_complete, &co); 95108844473SKevin Wolf if (acb == NULL) { 95208844473SKevin Wolf return -EIO; 95308844473SKevin Wolf } else { 95408844473SKevin Wolf qemu_coroutine_yield(); 95508844473SKevin Wolf return co.ret; 95608844473SKevin Wolf } 95708844473SKevin Wolf } 958166fe960SKevin Wolf } 959166fe960SKevin Wolf 96078a07294SKevin Wolf static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs, 96178a07294SKevin Wolf uint64_t offset, uint64_t bytes, 96278a07294SKevin Wolf QEMUIOVector *qiov, int flags) 96378a07294SKevin Wolf { 96478a07294SKevin Wolf BlockDriver *drv = bs->drv; 9653fb06697SKevin Wolf int64_t sector_num; 9663fb06697SKevin Wolf unsigned int nb_sectors; 96778a07294SKevin Wolf int ret; 96878a07294SKevin Wolf 969fa166538SEric Blake assert(!(flags & ~BDRV_REQ_MASK)); 970fa166538SEric Blake 971d470ad42SMax Reitz if (!drv) { 972d470ad42SMax Reitz return -ENOMEDIUM; 973d470ad42SMax Reitz } 974d470ad42SMax Reitz 9753fb06697SKevin Wolf if (drv->bdrv_co_pwritev) { 976515c2f43SKevin Wolf ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, 977515c2f43SKevin Wolf flags & bs->supported_write_flags); 978515c2f43SKevin Wolf flags &= ~bs->supported_write_flags; 9793fb06697SKevin Wolf goto emulate_flags; 9803fb06697SKevin Wolf } 9813fb06697SKevin Wolf 9823fb06697SKevin Wolf sector_num = offset >> BDRV_SECTOR_BITS; 9833fb06697SKevin Wolf nb_sectors = bytes >> BDRV_SECTOR_BITS; 9843fb06697SKevin Wolf 98578a07294SKevin Wolf assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 98678a07294SKevin Wolf assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 98778a07294SKevin Wolf assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS); 98878a07294SKevin Wolf 98978a07294SKevin Wolf if (drv->bdrv_co_writev_flags) { 99078a07294SKevin Wolf ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov, 9914df863f3SEric Blake flags & bs->supported_write_flags); 9924df863f3SEric Blake flags &= ~bs->supported_write_flags; 99308844473SKevin Wolf } else if (drv->bdrv_co_writev) { 9944df863f3SEric Blake assert(!bs->supported_write_flags); 99578a07294SKevin Wolf ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov); 99608844473SKevin Wolf } else { 99708844473SKevin Wolf BlockAIOCB *acb; 99808844473SKevin Wolf CoroutineIOCompletion co = { 99908844473SKevin Wolf .coroutine = qemu_coroutine_self(), 100008844473SKevin Wolf }; 100108844473SKevin Wolf 100208844473SKevin Wolf acb = bs->drv->bdrv_aio_writev(bs, sector_num, qiov, nb_sectors, 100308844473SKevin Wolf bdrv_co_io_em_complete, &co); 100408844473SKevin Wolf if (acb == NULL) { 10053fb06697SKevin Wolf ret = -EIO; 100608844473SKevin Wolf } else { 100708844473SKevin Wolf qemu_coroutine_yield(); 10083fb06697SKevin Wolf ret = co.ret; 100908844473SKevin Wolf } 101078a07294SKevin Wolf } 101178a07294SKevin Wolf 10123fb06697SKevin Wolf emulate_flags: 10134df863f3SEric Blake if (ret == 0 && (flags & BDRV_REQ_FUA)) { 101478a07294SKevin Wolf ret = bdrv_co_flush(bs); 101578a07294SKevin Wolf } 101678a07294SKevin Wolf 101778a07294SKevin Wolf return ret; 101878a07294SKevin Wolf } 101978a07294SKevin Wolf 102029a298afSPavel Butsykin static int coroutine_fn 102129a298afSPavel Butsykin bdrv_driver_pwritev_compressed(BlockDriverState *bs, uint64_t offset, 102229a298afSPavel Butsykin uint64_t bytes, QEMUIOVector *qiov) 102329a298afSPavel Butsykin { 102429a298afSPavel Butsykin BlockDriver *drv = bs->drv; 102529a298afSPavel Butsykin 1026d470ad42SMax Reitz if (!drv) { 1027d470ad42SMax Reitz return -ENOMEDIUM; 1028d470ad42SMax Reitz } 1029d470ad42SMax Reitz 103029a298afSPavel Butsykin if (!drv->bdrv_co_pwritev_compressed) { 103129a298afSPavel Butsykin return -ENOTSUP; 103229a298afSPavel Butsykin } 103329a298afSPavel Butsykin 103429a298afSPavel Butsykin return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov); 103529a298afSPavel Butsykin } 103629a298afSPavel Butsykin 103785c97ca7SKevin Wolf static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child, 1038244483e6SKevin Wolf int64_t offset, unsigned int bytes, QEMUIOVector *qiov) 103961007b31SStefan Hajnoczi { 104085c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 104185c97ca7SKevin Wolf 104261007b31SStefan Hajnoczi /* Perform I/O through a temporary buffer so that users who scribble over 104361007b31SStefan Hajnoczi * their read buffer while the operation is in progress do not end up 104461007b31SStefan Hajnoczi * modifying the image file. This is critical for zero-copy guest I/O 104561007b31SStefan Hajnoczi * where anything might happen inside guest memory. 104661007b31SStefan Hajnoczi */ 104761007b31SStefan Hajnoczi void *bounce_buffer; 104861007b31SStefan Hajnoczi 104961007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 105061007b31SStefan Hajnoczi struct iovec iov; 1051cb2e2878SEric Blake QEMUIOVector local_qiov; 1052244483e6SKevin Wolf int64_t cluster_offset; 10537cfd5275SEric Blake int64_t cluster_bytes; 105461007b31SStefan Hajnoczi size_t skip_bytes; 105561007b31SStefan Hajnoczi int ret; 1056cb2e2878SEric Blake int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, 1057cb2e2878SEric Blake BDRV_REQUEST_MAX_BYTES); 1058cb2e2878SEric Blake unsigned int progress = 0; 105961007b31SStefan Hajnoczi 1060d470ad42SMax Reitz if (!drv) { 1061d470ad42SMax Reitz return -ENOMEDIUM; 1062d470ad42SMax Reitz } 1063d470ad42SMax Reitz 10641bf03e66SKevin Wolf /* FIXME We cannot require callers to have write permissions when all they 10651bf03e66SKevin Wolf * are doing is a read request. If we did things right, write permissions 10661bf03e66SKevin Wolf * would be obtained anyway, but internally by the copy-on-read code. As 1067765d9df9SEric Blake * long as it is implemented here rather than in a separate filter driver, 10681bf03e66SKevin Wolf * the copy-on-read code doesn't have its own BdrvChild, however, for which 10691bf03e66SKevin Wolf * it could request permissions. Therefore we have to bypass the permission 10701bf03e66SKevin Wolf * system for the moment. */ 10711bf03e66SKevin Wolf // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE)); 1072afa4b293SKevin Wolf 107361007b31SStefan Hajnoczi /* Cover entire cluster so no additional backing file I/O is required when 1074cb2e2878SEric Blake * allocating cluster in the image file. Note that this value may exceed 1075cb2e2878SEric Blake * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which 1076cb2e2878SEric Blake * is one reason we loop rather than doing it all at once. 107761007b31SStefan Hajnoczi */ 1078244483e6SKevin Wolf bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes); 1079cb2e2878SEric Blake skip_bytes = offset - cluster_offset; 108061007b31SStefan Hajnoczi 1081244483e6SKevin Wolf trace_bdrv_co_do_copy_on_readv(bs, offset, bytes, 1082244483e6SKevin Wolf cluster_offset, cluster_bytes); 108361007b31SStefan Hajnoczi 1084cb2e2878SEric Blake bounce_buffer = qemu_try_blockalign(bs, 1085cb2e2878SEric Blake MIN(MIN(max_transfer, cluster_bytes), 1086cb2e2878SEric Blake MAX_BOUNCE_BUFFER)); 108761007b31SStefan Hajnoczi if (bounce_buffer == NULL) { 108861007b31SStefan Hajnoczi ret = -ENOMEM; 108961007b31SStefan Hajnoczi goto err; 109061007b31SStefan Hajnoczi } 109161007b31SStefan Hajnoczi 1092cb2e2878SEric Blake while (cluster_bytes) { 1093cb2e2878SEric Blake int64_t pnum; 109461007b31SStefan Hajnoczi 1095cb2e2878SEric Blake ret = bdrv_is_allocated(bs, cluster_offset, 1096cb2e2878SEric Blake MIN(cluster_bytes, max_transfer), &pnum); 1097cb2e2878SEric Blake if (ret < 0) { 1098cb2e2878SEric Blake /* Safe to treat errors in querying allocation as if 1099cb2e2878SEric Blake * unallocated; we'll probably fail again soon on the 1100cb2e2878SEric Blake * read, but at least that will set a decent errno. 1101cb2e2878SEric Blake */ 1102cb2e2878SEric Blake pnum = MIN(cluster_bytes, max_transfer); 1103cb2e2878SEric Blake } 1104cb2e2878SEric Blake 1105cb2e2878SEric Blake assert(skip_bytes < pnum); 1106cb2e2878SEric Blake 1107cb2e2878SEric Blake if (ret <= 0) { 1108cb2e2878SEric Blake /* Must copy-on-read; use the bounce buffer */ 1109cb2e2878SEric Blake iov.iov_base = bounce_buffer; 1110cb2e2878SEric Blake iov.iov_len = pnum = MIN(pnum, MAX_BOUNCE_BUFFER); 1111cb2e2878SEric Blake qemu_iovec_init_external(&local_qiov, &iov, 1); 1112cb2e2878SEric Blake 1113cb2e2878SEric Blake ret = bdrv_driver_preadv(bs, cluster_offset, pnum, 1114cb2e2878SEric Blake &local_qiov, 0); 111561007b31SStefan Hajnoczi if (ret < 0) { 111661007b31SStefan Hajnoczi goto err; 111761007b31SStefan Hajnoczi } 111861007b31SStefan Hajnoczi 1119d855ebcdSEric Blake bdrv_debug_event(bs, BLKDBG_COR_WRITE); 1120c1499a5eSEric Blake if (drv->bdrv_co_pwrite_zeroes && 1121cb2e2878SEric Blake buffer_is_zero(bounce_buffer, pnum)) { 1122a604fa2bSEric Blake /* FIXME: Should we (perhaps conditionally) be setting 1123a604fa2bSEric Blake * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy 1124a604fa2bSEric Blake * that still correctly reads as zero? */ 1125cb2e2878SEric Blake ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, pnum, 0); 112661007b31SStefan Hajnoczi } else { 1127cb2e2878SEric Blake /* This does not change the data on the disk, it is not 1128cb2e2878SEric Blake * necessary to flush even in cache=writethrough mode. 112961007b31SStefan Hajnoczi */ 1130cb2e2878SEric Blake ret = bdrv_driver_pwritev(bs, cluster_offset, pnum, 1131cb2e2878SEric Blake &local_qiov, 0); 113261007b31SStefan Hajnoczi } 113361007b31SStefan Hajnoczi 113461007b31SStefan Hajnoczi if (ret < 0) { 1135cb2e2878SEric Blake /* It might be okay to ignore write errors for guest 1136cb2e2878SEric Blake * requests. If this is a deliberate copy-on-read 1137cb2e2878SEric Blake * then we don't want to ignore the error. Simply 1138cb2e2878SEric Blake * report it in all cases. 113961007b31SStefan Hajnoczi */ 114061007b31SStefan Hajnoczi goto err; 114161007b31SStefan Hajnoczi } 114261007b31SStefan Hajnoczi 1143cb2e2878SEric Blake qemu_iovec_from_buf(qiov, progress, bounce_buffer + skip_bytes, 1144cb2e2878SEric Blake pnum - skip_bytes); 1145cb2e2878SEric Blake } else { 1146cb2e2878SEric Blake /* Read directly into the destination */ 1147cb2e2878SEric Blake qemu_iovec_init(&local_qiov, qiov->niov); 1148cb2e2878SEric Blake qemu_iovec_concat(&local_qiov, qiov, progress, pnum - skip_bytes); 1149cb2e2878SEric Blake ret = bdrv_driver_preadv(bs, offset + progress, local_qiov.size, 1150cb2e2878SEric Blake &local_qiov, 0); 1151cb2e2878SEric Blake qemu_iovec_destroy(&local_qiov); 1152cb2e2878SEric Blake if (ret < 0) { 1153cb2e2878SEric Blake goto err; 1154cb2e2878SEric Blake } 1155cb2e2878SEric Blake } 1156cb2e2878SEric Blake 1157cb2e2878SEric Blake cluster_offset += pnum; 1158cb2e2878SEric Blake cluster_bytes -= pnum; 1159cb2e2878SEric Blake progress += pnum - skip_bytes; 1160cb2e2878SEric Blake skip_bytes = 0; 1161cb2e2878SEric Blake } 1162cb2e2878SEric Blake ret = 0; 116361007b31SStefan Hajnoczi 116461007b31SStefan Hajnoczi err: 116561007b31SStefan Hajnoczi qemu_vfree(bounce_buffer); 116661007b31SStefan Hajnoczi return ret; 116761007b31SStefan Hajnoczi } 116861007b31SStefan Hajnoczi 116961007b31SStefan Hajnoczi /* 117061007b31SStefan Hajnoczi * Forwards an already correctly aligned request to the BlockDriver. This 11711a62d0acSEric Blake * handles copy on read, zeroing after EOF, and fragmentation of large 11721a62d0acSEric Blake * reads; any other features must be implemented by the caller. 117361007b31SStefan Hajnoczi */ 117485c97ca7SKevin Wolf static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child, 117561007b31SStefan Hajnoczi BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 117661007b31SStefan Hajnoczi int64_t align, QEMUIOVector *qiov, int flags) 117761007b31SStefan Hajnoczi { 117885c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 1179c9d20029SKevin Wolf int64_t total_bytes, max_bytes; 11801a62d0acSEric Blake int ret = 0; 11811a62d0acSEric Blake uint64_t bytes_remaining = bytes; 11821a62d0acSEric Blake int max_transfer; 118361007b31SStefan Hajnoczi 118449c07526SKevin Wolf assert(is_power_of_2(align)); 118549c07526SKevin Wolf assert((offset & (align - 1)) == 0); 118649c07526SKevin Wolf assert((bytes & (align - 1)) == 0); 118761007b31SStefan Hajnoczi assert(!qiov || bytes == qiov->size); 1188abb06c5aSDaniel P. Berrange assert((bs->open_flags & BDRV_O_NO_IO) == 0); 11891a62d0acSEric Blake max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 11901a62d0acSEric Blake align); 1191a604fa2bSEric Blake 1192a604fa2bSEric Blake /* TODO: We would need a per-BDS .supported_read_flags and 1193a604fa2bSEric Blake * potential fallback support, if we ever implement any read flags 1194a604fa2bSEric Blake * to pass through to drivers. For now, there aren't any 1195a604fa2bSEric Blake * passthrough flags. */ 1196a604fa2bSEric Blake assert(!(flags & ~(BDRV_REQ_NO_SERIALISING | BDRV_REQ_COPY_ON_READ))); 119761007b31SStefan Hajnoczi 119861007b31SStefan Hajnoczi /* Handle Copy on Read and associated serialisation */ 119961007b31SStefan Hajnoczi if (flags & BDRV_REQ_COPY_ON_READ) { 120061007b31SStefan Hajnoczi /* If we touch the same cluster it counts as an overlap. This 120161007b31SStefan Hajnoczi * guarantees that allocating writes will be serialized and not race 120261007b31SStefan Hajnoczi * with each other for the same cluster. For example, in copy-on-read 120361007b31SStefan Hajnoczi * it ensures that the CoR read and write operations are atomic and 120461007b31SStefan Hajnoczi * guest writes cannot interleave between them. */ 120561007b31SStefan Hajnoczi mark_request_serialising(req, bdrv_get_cluster_size(bs)); 120661007b31SStefan Hajnoczi } 120761007b31SStefan Hajnoczi 120861408b25SFam Zheng if (!(flags & BDRV_REQ_NO_SERIALISING)) { 120961007b31SStefan Hajnoczi wait_serialising_requests(req); 121061408b25SFam Zheng } 121161007b31SStefan Hajnoczi 121261007b31SStefan Hajnoczi if (flags & BDRV_REQ_COPY_ON_READ) { 1213d6a644bbSEric Blake int64_t pnum; 121461007b31SStefan Hajnoczi 121588e63df2SEric Blake ret = bdrv_is_allocated(bs, offset, bytes, &pnum); 121661007b31SStefan Hajnoczi if (ret < 0) { 121761007b31SStefan Hajnoczi goto out; 121861007b31SStefan Hajnoczi } 121961007b31SStefan Hajnoczi 122088e63df2SEric Blake if (!ret || pnum != bytes) { 122185c97ca7SKevin Wolf ret = bdrv_co_do_copy_on_readv(child, offset, bytes, qiov); 122261007b31SStefan Hajnoczi goto out; 122361007b31SStefan Hajnoczi } 122461007b31SStefan Hajnoczi } 122561007b31SStefan Hajnoczi 12261a62d0acSEric Blake /* Forward the request to the BlockDriver, possibly fragmenting it */ 122749c07526SKevin Wolf total_bytes = bdrv_getlength(bs); 122849c07526SKevin Wolf if (total_bytes < 0) { 122949c07526SKevin Wolf ret = total_bytes; 123061007b31SStefan Hajnoczi goto out; 123161007b31SStefan Hajnoczi } 123261007b31SStefan Hajnoczi 123349c07526SKevin Wolf max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align); 12341a62d0acSEric Blake if (bytes <= max_bytes && bytes <= max_transfer) { 1235166fe960SKevin Wolf ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0); 12361a62d0acSEric Blake goto out; 123761007b31SStefan Hajnoczi } 123861007b31SStefan Hajnoczi 12391a62d0acSEric Blake while (bytes_remaining) { 12401a62d0acSEric Blake int num; 12411a62d0acSEric Blake 12421a62d0acSEric Blake if (max_bytes) { 12431a62d0acSEric Blake QEMUIOVector local_qiov; 12441a62d0acSEric Blake 12451a62d0acSEric Blake num = MIN(bytes_remaining, MIN(max_bytes, max_transfer)); 12461a62d0acSEric Blake assert(num); 12471a62d0acSEric Blake qemu_iovec_init(&local_qiov, qiov->niov); 12481a62d0acSEric Blake qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num); 12491a62d0acSEric Blake 12501a62d0acSEric Blake ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining, 12511a62d0acSEric Blake num, &local_qiov, 0); 12521a62d0acSEric Blake max_bytes -= num; 12531a62d0acSEric Blake qemu_iovec_destroy(&local_qiov); 12541a62d0acSEric Blake } else { 12551a62d0acSEric Blake num = bytes_remaining; 12561a62d0acSEric Blake ret = qemu_iovec_memset(qiov, bytes - bytes_remaining, 0, 12571a62d0acSEric Blake bytes_remaining); 12581a62d0acSEric Blake } 12591a62d0acSEric Blake if (ret < 0) { 12601a62d0acSEric Blake goto out; 12611a62d0acSEric Blake } 12621a62d0acSEric Blake bytes_remaining -= num; 126361007b31SStefan Hajnoczi } 126461007b31SStefan Hajnoczi 126561007b31SStefan Hajnoczi out: 12661a62d0acSEric Blake return ret < 0 ? ret : 0; 126761007b31SStefan Hajnoczi } 126861007b31SStefan Hajnoczi 126961007b31SStefan Hajnoczi /* 127061007b31SStefan Hajnoczi * Handle a read request in coroutine context 127161007b31SStefan Hajnoczi */ 1272a03ef88fSKevin Wolf int coroutine_fn bdrv_co_preadv(BdrvChild *child, 127361007b31SStefan Hajnoczi int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 127461007b31SStefan Hajnoczi BdrvRequestFlags flags) 127561007b31SStefan Hajnoczi { 1276a03ef88fSKevin Wolf BlockDriverState *bs = child->bs; 127761007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 127861007b31SStefan Hajnoczi BdrvTrackedRequest req; 127961007b31SStefan Hajnoczi 1280a5b8dd2cSEric Blake uint64_t align = bs->bl.request_alignment; 128161007b31SStefan Hajnoczi uint8_t *head_buf = NULL; 128261007b31SStefan Hajnoczi uint8_t *tail_buf = NULL; 128361007b31SStefan Hajnoczi QEMUIOVector local_qiov; 128461007b31SStefan Hajnoczi bool use_local_qiov = false; 128561007b31SStefan Hajnoczi int ret; 128661007b31SStefan Hajnoczi 1287f42cf447SDaniel P. Berrange trace_bdrv_co_preadv(child->bs, offset, bytes, flags); 1288f42cf447SDaniel P. Berrange 128961007b31SStefan Hajnoczi if (!drv) { 129061007b31SStefan Hajnoczi return -ENOMEDIUM; 129161007b31SStefan Hajnoczi } 129261007b31SStefan Hajnoczi 129361007b31SStefan Hajnoczi ret = bdrv_check_byte_request(bs, offset, bytes); 129461007b31SStefan Hajnoczi if (ret < 0) { 129561007b31SStefan Hajnoczi return ret; 129661007b31SStefan Hajnoczi } 129761007b31SStefan Hajnoczi 129899723548SPaolo Bonzini bdrv_inc_in_flight(bs); 129999723548SPaolo Bonzini 13009568b511SWen Congyang /* Don't do copy-on-read if we read data before write operation */ 1301d3faa13eSPaolo Bonzini if (atomic_read(&bs->copy_on_read) && !(flags & BDRV_REQ_NO_SERIALISING)) { 130261007b31SStefan Hajnoczi flags |= BDRV_REQ_COPY_ON_READ; 130361007b31SStefan Hajnoczi } 130461007b31SStefan Hajnoczi 130561007b31SStefan Hajnoczi /* Align read if necessary by padding qiov */ 130661007b31SStefan Hajnoczi if (offset & (align - 1)) { 130761007b31SStefan Hajnoczi head_buf = qemu_blockalign(bs, align); 130861007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 2); 130961007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 131061007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 131161007b31SStefan Hajnoczi use_local_qiov = true; 131261007b31SStefan Hajnoczi 131361007b31SStefan Hajnoczi bytes += offset & (align - 1); 131461007b31SStefan Hajnoczi offset = offset & ~(align - 1); 131561007b31SStefan Hajnoczi } 131661007b31SStefan Hajnoczi 131761007b31SStefan Hajnoczi if ((offset + bytes) & (align - 1)) { 131861007b31SStefan Hajnoczi if (!use_local_qiov) { 131961007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 1); 132061007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 132161007b31SStefan Hajnoczi use_local_qiov = true; 132261007b31SStefan Hajnoczi } 132361007b31SStefan Hajnoczi tail_buf = qemu_blockalign(bs, align); 132461007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, tail_buf, 132561007b31SStefan Hajnoczi align - ((offset + bytes) & (align - 1))); 132661007b31SStefan Hajnoczi 132761007b31SStefan Hajnoczi bytes = ROUND_UP(bytes, align); 132861007b31SStefan Hajnoczi } 132961007b31SStefan Hajnoczi 1330ebde595cSFam Zheng tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ); 133185c97ca7SKevin Wolf ret = bdrv_aligned_preadv(child, &req, offset, bytes, align, 133261007b31SStefan Hajnoczi use_local_qiov ? &local_qiov : qiov, 133361007b31SStefan Hajnoczi flags); 133461007b31SStefan Hajnoczi tracked_request_end(&req); 133599723548SPaolo Bonzini bdrv_dec_in_flight(bs); 133661007b31SStefan Hajnoczi 133761007b31SStefan Hajnoczi if (use_local_qiov) { 133861007b31SStefan Hajnoczi qemu_iovec_destroy(&local_qiov); 133961007b31SStefan Hajnoczi qemu_vfree(head_buf); 134061007b31SStefan Hajnoczi qemu_vfree(tail_buf); 134161007b31SStefan Hajnoczi } 134261007b31SStefan Hajnoczi 134361007b31SStefan Hajnoczi return ret; 134461007b31SStefan Hajnoczi } 134561007b31SStefan Hajnoczi 1346adad6496SKevin Wolf static int coroutine_fn bdrv_co_do_readv(BdrvChild *child, 134761007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, 134861007b31SStefan Hajnoczi BdrvRequestFlags flags) 134961007b31SStefan Hajnoczi { 135061007b31SStefan Hajnoczi if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 135161007b31SStefan Hajnoczi return -EINVAL; 135261007b31SStefan Hajnoczi } 135361007b31SStefan Hajnoczi 1354a03ef88fSKevin Wolf return bdrv_co_preadv(child, sector_num << BDRV_SECTOR_BITS, 135561007b31SStefan Hajnoczi nb_sectors << BDRV_SECTOR_BITS, qiov, flags); 135661007b31SStefan Hajnoczi } 135761007b31SStefan Hajnoczi 135828b04a8fSKevin Wolf int coroutine_fn bdrv_co_readv(BdrvChild *child, int64_t sector_num, 135961007b31SStefan Hajnoczi int nb_sectors, QEMUIOVector *qiov) 136061007b31SStefan Hajnoczi { 1361adad6496SKevin Wolf return bdrv_co_do_readv(child, sector_num, nb_sectors, qiov, 0); 136261007b31SStefan Hajnoczi } 136361007b31SStefan Hajnoczi 1364d05aa8bbSEric Blake static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 1365f5a5ca79SManos Pitsidianakis int64_t offset, int bytes, BdrvRequestFlags flags) 136661007b31SStefan Hajnoczi { 136761007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 136861007b31SStefan Hajnoczi QEMUIOVector qiov; 136961007b31SStefan Hajnoczi struct iovec iov = {0}; 137061007b31SStefan Hajnoczi int ret = 0; 1371465fe887SEric Blake bool need_flush = false; 1372443668caSDenis V. Lunev int head = 0; 1373443668caSDenis V. Lunev int tail = 0; 137461007b31SStefan Hajnoczi 1375cf081fcaSEric Blake int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, INT_MAX); 1376a5b8dd2cSEric Blake int alignment = MAX(bs->bl.pwrite_zeroes_alignment, 1377a5b8dd2cSEric Blake bs->bl.request_alignment); 1378cb2e2878SEric Blake int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER); 1379cf081fcaSEric Blake 1380d470ad42SMax Reitz if (!drv) { 1381d470ad42SMax Reitz return -ENOMEDIUM; 1382d470ad42SMax Reitz } 1383d470ad42SMax Reitz 1384b8d0a980SEric Blake assert(alignment % bs->bl.request_alignment == 0); 1385b8d0a980SEric Blake head = offset % alignment; 1386f5a5ca79SManos Pitsidianakis tail = (offset + bytes) % alignment; 1387b8d0a980SEric Blake max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment); 1388b8d0a980SEric Blake assert(max_write_zeroes >= bs->bl.request_alignment); 138961007b31SStefan Hajnoczi 1390f5a5ca79SManos Pitsidianakis while (bytes > 0 && !ret) { 1391f5a5ca79SManos Pitsidianakis int num = bytes; 139261007b31SStefan Hajnoczi 139361007b31SStefan Hajnoczi /* Align request. Block drivers can expect the "bulk" of the request 1394443668caSDenis V. Lunev * to be aligned, and that unaligned requests do not cross cluster 1395443668caSDenis V. Lunev * boundaries. 139661007b31SStefan Hajnoczi */ 1397443668caSDenis V. Lunev if (head) { 1398b2f95feeSEric Blake /* Make a small request up to the first aligned sector. For 1399b2f95feeSEric Blake * convenience, limit this request to max_transfer even if 1400b2f95feeSEric Blake * we don't need to fall back to writes. */ 1401f5a5ca79SManos Pitsidianakis num = MIN(MIN(bytes, max_transfer), alignment - head); 1402b2f95feeSEric Blake head = (head + num) % alignment; 1403b2f95feeSEric Blake assert(num < max_write_zeroes); 1404d05aa8bbSEric Blake } else if (tail && num > alignment) { 1405443668caSDenis V. Lunev /* Shorten the request to the last aligned sector. */ 1406443668caSDenis V. Lunev num -= tail; 140761007b31SStefan Hajnoczi } 140861007b31SStefan Hajnoczi 140961007b31SStefan Hajnoczi /* limit request size */ 141061007b31SStefan Hajnoczi if (num > max_write_zeroes) { 141161007b31SStefan Hajnoczi num = max_write_zeroes; 141261007b31SStefan Hajnoczi } 141361007b31SStefan Hajnoczi 141461007b31SStefan Hajnoczi ret = -ENOTSUP; 141561007b31SStefan Hajnoczi /* First try the efficient write zeroes operation */ 1416d05aa8bbSEric Blake if (drv->bdrv_co_pwrite_zeroes) { 1417d05aa8bbSEric Blake ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num, 1418d05aa8bbSEric Blake flags & bs->supported_zero_flags); 1419d05aa8bbSEric Blake if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) && 1420d05aa8bbSEric Blake !(bs->supported_zero_flags & BDRV_REQ_FUA)) { 1421d05aa8bbSEric Blake need_flush = true; 1422d05aa8bbSEric Blake } 1423465fe887SEric Blake } else { 1424465fe887SEric Blake assert(!bs->supported_zero_flags); 142561007b31SStefan Hajnoczi } 142661007b31SStefan Hajnoczi 142761007b31SStefan Hajnoczi if (ret == -ENOTSUP) { 142861007b31SStefan Hajnoczi /* Fall back to bounce buffer if write zeroes is unsupported */ 1429465fe887SEric Blake BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE; 1430465fe887SEric Blake 1431465fe887SEric Blake if ((flags & BDRV_REQ_FUA) && 1432465fe887SEric Blake !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1433465fe887SEric Blake /* No need for bdrv_driver_pwrite() to do a fallback 1434465fe887SEric Blake * flush on each chunk; use just one at the end */ 1435465fe887SEric Blake write_flags &= ~BDRV_REQ_FUA; 1436465fe887SEric Blake need_flush = true; 1437465fe887SEric Blake } 14385def6b80SEric Blake num = MIN(num, max_transfer); 1439d05aa8bbSEric Blake iov.iov_len = num; 144061007b31SStefan Hajnoczi if (iov.iov_base == NULL) { 1441d05aa8bbSEric Blake iov.iov_base = qemu_try_blockalign(bs, num); 144261007b31SStefan Hajnoczi if (iov.iov_base == NULL) { 144361007b31SStefan Hajnoczi ret = -ENOMEM; 144461007b31SStefan Hajnoczi goto fail; 144561007b31SStefan Hajnoczi } 1446d05aa8bbSEric Blake memset(iov.iov_base, 0, num); 144761007b31SStefan Hajnoczi } 144861007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 144961007b31SStefan Hajnoczi 1450d05aa8bbSEric Blake ret = bdrv_driver_pwritev(bs, offset, num, &qiov, write_flags); 145161007b31SStefan Hajnoczi 145261007b31SStefan Hajnoczi /* Keep bounce buffer around if it is big enough for all 145361007b31SStefan Hajnoczi * all future requests. 145461007b31SStefan Hajnoczi */ 14555def6b80SEric Blake if (num < max_transfer) { 145661007b31SStefan Hajnoczi qemu_vfree(iov.iov_base); 145761007b31SStefan Hajnoczi iov.iov_base = NULL; 145861007b31SStefan Hajnoczi } 145961007b31SStefan Hajnoczi } 146061007b31SStefan Hajnoczi 1461d05aa8bbSEric Blake offset += num; 1462f5a5ca79SManos Pitsidianakis bytes -= num; 146361007b31SStefan Hajnoczi } 146461007b31SStefan Hajnoczi 146561007b31SStefan Hajnoczi fail: 1466465fe887SEric Blake if (ret == 0 && need_flush) { 1467465fe887SEric Blake ret = bdrv_co_flush(bs); 1468465fe887SEric Blake } 146961007b31SStefan Hajnoczi qemu_vfree(iov.iov_base); 147061007b31SStefan Hajnoczi return ret; 147161007b31SStefan Hajnoczi } 147261007b31SStefan Hajnoczi 147361007b31SStefan Hajnoczi /* 147404ed95f4SEric Blake * Forwards an already correctly aligned write request to the BlockDriver, 147504ed95f4SEric Blake * after possibly fragmenting it. 147661007b31SStefan Hajnoczi */ 147785c97ca7SKevin Wolf static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child, 147861007b31SStefan Hajnoczi BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 1479cff86b38SEric Blake int64_t align, QEMUIOVector *qiov, int flags) 148061007b31SStefan Hajnoczi { 148185c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 148261007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 148361007b31SStefan Hajnoczi bool waited; 148461007b31SStefan Hajnoczi int ret; 148561007b31SStefan Hajnoczi 14869896c876SKevin Wolf int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 148704ed95f4SEric Blake uint64_t bytes_remaining = bytes; 148804ed95f4SEric Blake int max_transfer; 148961007b31SStefan Hajnoczi 1490d470ad42SMax Reitz if (!drv) { 1491d470ad42SMax Reitz return -ENOMEDIUM; 1492d470ad42SMax Reitz } 1493d470ad42SMax Reitz 1494d6883bc9SVladimir Sementsov-Ogievskiy if (bdrv_has_readonly_bitmaps(bs)) { 1495d6883bc9SVladimir Sementsov-Ogievskiy return -EPERM; 1496d6883bc9SVladimir Sementsov-Ogievskiy } 1497d6883bc9SVladimir Sementsov-Ogievskiy 1498cff86b38SEric Blake assert(is_power_of_2(align)); 1499cff86b38SEric Blake assert((offset & (align - 1)) == 0); 1500cff86b38SEric Blake assert((bytes & (align - 1)) == 0); 150161007b31SStefan Hajnoczi assert(!qiov || bytes == qiov->size); 1502abb06c5aSDaniel P. Berrange assert((bs->open_flags & BDRV_O_NO_IO) == 0); 1503fa166538SEric Blake assert(!(flags & ~BDRV_REQ_MASK)); 150404ed95f4SEric Blake max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 150504ed95f4SEric Blake align); 150661007b31SStefan Hajnoczi 150761007b31SStefan Hajnoczi waited = wait_serialising_requests(req); 150861007b31SStefan Hajnoczi assert(!waited || !req->serialising); 150961007b31SStefan Hajnoczi assert(req->overlap_offset <= offset); 151061007b31SStefan Hajnoczi assert(offset + bytes <= req->overlap_offset + req->overlap_bytes); 1511362b3786SMax Reitz assert(child->perm & BLK_PERM_WRITE); 1512362b3786SMax Reitz assert(end_sector <= bs->total_sectors || child->perm & BLK_PERM_RESIZE); 151361007b31SStefan Hajnoczi 151461007b31SStefan Hajnoczi ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req); 151561007b31SStefan Hajnoczi 151661007b31SStefan Hajnoczi if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF && 1517c1499a5eSEric Blake !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes && 151861007b31SStefan Hajnoczi qemu_iovec_is_zero(qiov)) { 151961007b31SStefan Hajnoczi flags |= BDRV_REQ_ZERO_WRITE; 152061007b31SStefan Hajnoczi if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) { 152161007b31SStefan Hajnoczi flags |= BDRV_REQ_MAY_UNMAP; 152261007b31SStefan Hajnoczi } 152361007b31SStefan Hajnoczi } 152461007b31SStefan Hajnoczi 152561007b31SStefan Hajnoczi if (ret < 0) { 152661007b31SStefan Hajnoczi /* Do nothing, write notifier decided to fail this request */ 152761007b31SStefan Hajnoczi } else if (flags & BDRV_REQ_ZERO_WRITE) { 15289a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO); 15299896c876SKevin Wolf ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags); 15303ea1a091SPavel Butsykin } else if (flags & BDRV_REQ_WRITE_COMPRESSED) { 15313ea1a091SPavel Butsykin ret = bdrv_driver_pwritev_compressed(bs, offset, bytes, qiov); 153204ed95f4SEric Blake } else if (bytes <= max_transfer) { 15339a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV); 153478a07294SKevin Wolf ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, flags); 153504ed95f4SEric Blake } else { 153604ed95f4SEric Blake bdrv_debug_event(bs, BLKDBG_PWRITEV); 153704ed95f4SEric Blake while (bytes_remaining) { 153804ed95f4SEric Blake int num = MIN(bytes_remaining, max_transfer); 153904ed95f4SEric Blake QEMUIOVector local_qiov; 154004ed95f4SEric Blake int local_flags = flags; 154104ed95f4SEric Blake 154204ed95f4SEric Blake assert(num); 154304ed95f4SEric Blake if (num < bytes_remaining && (flags & BDRV_REQ_FUA) && 154404ed95f4SEric Blake !(bs->supported_write_flags & BDRV_REQ_FUA)) { 154504ed95f4SEric Blake /* If FUA is going to be emulated by flush, we only 154604ed95f4SEric Blake * need to flush on the last iteration */ 154704ed95f4SEric Blake local_flags &= ~BDRV_REQ_FUA; 154804ed95f4SEric Blake } 154904ed95f4SEric Blake qemu_iovec_init(&local_qiov, qiov->niov); 155004ed95f4SEric Blake qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num); 155104ed95f4SEric Blake 155204ed95f4SEric Blake ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining, 155304ed95f4SEric Blake num, &local_qiov, local_flags); 155404ed95f4SEric Blake qemu_iovec_destroy(&local_qiov); 155504ed95f4SEric Blake if (ret < 0) { 155604ed95f4SEric Blake break; 155704ed95f4SEric Blake } 155804ed95f4SEric Blake bytes_remaining -= num; 155904ed95f4SEric Blake } 156061007b31SStefan Hajnoczi } 15619a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE); 156261007b31SStefan Hajnoczi 156347fec599SPaolo Bonzini atomic_inc(&bs->write_gen); 15640fdf1a4fSEric Blake bdrv_set_dirty(bs, offset, bytes); 156561007b31SStefan Hajnoczi 1566f7946da2SPaolo Bonzini stat64_max(&bs->wr_highest_offset, offset + bytes); 156761007b31SStefan Hajnoczi 156861007b31SStefan Hajnoczi if (ret >= 0) { 15699896c876SKevin Wolf bs->total_sectors = MAX(bs->total_sectors, end_sector); 157004ed95f4SEric Blake ret = 0; 157161007b31SStefan Hajnoczi } 157261007b31SStefan Hajnoczi 157361007b31SStefan Hajnoczi return ret; 157461007b31SStefan Hajnoczi } 157561007b31SStefan Hajnoczi 157685c97ca7SKevin Wolf static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child, 15779eeb6dd1SFam Zheng int64_t offset, 15789eeb6dd1SFam Zheng unsigned int bytes, 15799eeb6dd1SFam Zheng BdrvRequestFlags flags, 15809eeb6dd1SFam Zheng BdrvTrackedRequest *req) 15819eeb6dd1SFam Zheng { 158285c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 15839eeb6dd1SFam Zheng uint8_t *buf = NULL; 15849eeb6dd1SFam Zheng QEMUIOVector local_qiov; 15859eeb6dd1SFam Zheng struct iovec iov; 1586a5b8dd2cSEric Blake uint64_t align = bs->bl.request_alignment; 15879eeb6dd1SFam Zheng unsigned int head_padding_bytes, tail_padding_bytes; 15889eeb6dd1SFam Zheng int ret = 0; 15899eeb6dd1SFam Zheng 15909eeb6dd1SFam Zheng head_padding_bytes = offset & (align - 1); 1591f13ce1beSDenis V. Lunev tail_padding_bytes = (align - (offset + bytes)) & (align - 1); 15929eeb6dd1SFam Zheng 15939eeb6dd1SFam Zheng 15949eeb6dd1SFam Zheng assert(flags & BDRV_REQ_ZERO_WRITE); 15959eeb6dd1SFam Zheng if (head_padding_bytes || tail_padding_bytes) { 15969eeb6dd1SFam Zheng buf = qemu_blockalign(bs, align); 15979eeb6dd1SFam Zheng iov = (struct iovec) { 15989eeb6dd1SFam Zheng .iov_base = buf, 15999eeb6dd1SFam Zheng .iov_len = align, 16009eeb6dd1SFam Zheng }; 16019eeb6dd1SFam Zheng qemu_iovec_init_external(&local_qiov, &iov, 1); 16029eeb6dd1SFam Zheng } 16039eeb6dd1SFam Zheng if (head_padding_bytes) { 16049eeb6dd1SFam Zheng uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes); 16059eeb6dd1SFam Zheng 16069eeb6dd1SFam Zheng /* RMW the unaligned part before head. */ 16079eeb6dd1SFam Zheng mark_request_serialising(req, align); 16089eeb6dd1SFam Zheng wait_serialising_requests(req); 16099a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 161085c97ca7SKevin Wolf ret = bdrv_aligned_preadv(child, req, offset & ~(align - 1), align, 16119eeb6dd1SFam Zheng align, &local_qiov, 0); 16129eeb6dd1SFam Zheng if (ret < 0) { 16139eeb6dd1SFam Zheng goto fail; 16149eeb6dd1SFam Zheng } 16159a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 16169eeb6dd1SFam Zheng 16179eeb6dd1SFam Zheng memset(buf + head_padding_bytes, 0, zero_bytes); 161885c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, req, offset & ~(align - 1), align, 1619cff86b38SEric Blake align, &local_qiov, 16209eeb6dd1SFam Zheng flags & ~BDRV_REQ_ZERO_WRITE); 16219eeb6dd1SFam Zheng if (ret < 0) { 16229eeb6dd1SFam Zheng goto fail; 16239eeb6dd1SFam Zheng } 16249eeb6dd1SFam Zheng offset += zero_bytes; 16259eeb6dd1SFam Zheng bytes -= zero_bytes; 16269eeb6dd1SFam Zheng } 16279eeb6dd1SFam Zheng 16289eeb6dd1SFam Zheng assert(!bytes || (offset & (align - 1)) == 0); 16299eeb6dd1SFam Zheng if (bytes >= align) { 16309eeb6dd1SFam Zheng /* Write the aligned part in the middle. */ 16319eeb6dd1SFam Zheng uint64_t aligned_bytes = bytes & ~(align - 1); 163285c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align, 16339eeb6dd1SFam Zheng NULL, flags); 16349eeb6dd1SFam Zheng if (ret < 0) { 16359eeb6dd1SFam Zheng goto fail; 16369eeb6dd1SFam Zheng } 16379eeb6dd1SFam Zheng bytes -= aligned_bytes; 16389eeb6dd1SFam Zheng offset += aligned_bytes; 16399eeb6dd1SFam Zheng } 16409eeb6dd1SFam Zheng 16419eeb6dd1SFam Zheng assert(!bytes || (offset & (align - 1)) == 0); 16429eeb6dd1SFam Zheng if (bytes) { 16439eeb6dd1SFam Zheng assert(align == tail_padding_bytes + bytes); 16449eeb6dd1SFam Zheng /* RMW the unaligned part after tail. */ 16459eeb6dd1SFam Zheng mark_request_serialising(req, align); 16469eeb6dd1SFam Zheng wait_serialising_requests(req); 16479a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 164885c97ca7SKevin Wolf ret = bdrv_aligned_preadv(child, req, offset, align, 16499eeb6dd1SFam Zheng align, &local_qiov, 0); 16509eeb6dd1SFam Zheng if (ret < 0) { 16519eeb6dd1SFam Zheng goto fail; 16529eeb6dd1SFam Zheng } 16539a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 16549eeb6dd1SFam Zheng 16559eeb6dd1SFam Zheng memset(buf, 0, bytes); 165685c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, req, offset, align, align, 16579eeb6dd1SFam Zheng &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE); 16589eeb6dd1SFam Zheng } 16599eeb6dd1SFam Zheng fail: 16609eeb6dd1SFam Zheng qemu_vfree(buf); 16619eeb6dd1SFam Zheng return ret; 16629eeb6dd1SFam Zheng 16639eeb6dd1SFam Zheng } 16649eeb6dd1SFam Zheng 166561007b31SStefan Hajnoczi /* 166661007b31SStefan Hajnoczi * Handle a write request in coroutine context 166761007b31SStefan Hajnoczi */ 1668a03ef88fSKevin Wolf int coroutine_fn bdrv_co_pwritev(BdrvChild *child, 166961007b31SStefan Hajnoczi int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 167061007b31SStefan Hajnoczi BdrvRequestFlags flags) 167161007b31SStefan Hajnoczi { 1672a03ef88fSKevin Wolf BlockDriverState *bs = child->bs; 167361007b31SStefan Hajnoczi BdrvTrackedRequest req; 1674a5b8dd2cSEric Blake uint64_t align = bs->bl.request_alignment; 167561007b31SStefan Hajnoczi uint8_t *head_buf = NULL; 167661007b31SStefan Hajnoczi uint8_t *tail_buf = NULL; 167761007b31SStefan Hajnoczi QEMUIOVector local_qiov; 167861007b31SStefan Hajnoczi bool use_local_qiov = false; 167961007b31SStefan Hajnoczi int ret; 168061007b31SStefan Hajnoczi 1681f42cf447SDaniel P. Berrange trace_bdrv_co_pwritev(child->bs, offset, bytes, flags); 1682f42cf447SDaniel P. Berrange 168361007b31SStefan Hajnoczi if (!bs->drv) { 168461007b31SStefan Hajnoczi return -ENOMEDIUM; 168561007b31SStefan Hajnoczi } 168661007b31SStefan Hajnoczi if (bs->read_only) { 1687eaf5fe2dSPaolo Bonzini return -EPERM; 168861007b31SStefan Hajnoczi } 168904c01a5cSKevin Wolf assert(!(bs->open_flags & BDRV_O_INACTIVE)); 169061007b31SStefan Hajnoczi 169161007b31SStefan Hajnoczi ret = bdrv_check_byte_request(bs, offset, bytes); 169261007b31SStefan Hajnoczi if (ret < 0) { 169361007b31SStefan Hajnoczi return ret; 169461007b31SStefan Hajnoczi } 169561007b31SStefan Hajnoczi 169699723548SPaolo Bonzini bdrv_inc_in_flight(bs); 169761007b31SStefan Hajnoczi /* 169861007b31SStefan Hajnoczi * Align write if necessary by performing a read-modify-write cycle. 169961007b31SStefan Hajnoczi * Pad qiov with the read parts and be sure to have a tracked request not 170061007b31SStefan Hajnoczi * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle. 170161007b31SStefan Hajnoczi */ 1702ebde595cSFam Zheng tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE); 170361007b31SStefan Hajnoczi 17049eeb6dd1SFam Zheng if (!qiov) { 170585c97ca7SKevin Wolf ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req); 17069eeb6dd1SFam Zheng goto out; 17079eeb6dd1SFam Zheng } 17089eeb6dd1SFam Zheng 170961007b31SStefan Hajnoczi if (offset & (align - 1)) { 171061007b31SStefan Hajnoczi QEMUIOVector head_qiov; 171161007b31SStefan Hajnoczi struct iovec head_iov; 171261007b31SStefan Hajnoczi 171361007b31SStefan Hajnoczi mark_request_serialising(&req, align); 171461007b31SStefan Hajnoczi wait_serialising_requests(&req); 171561007b31SStefan Hajnoczi 171661007b31SStefan Hajnoczi head_buf = qemu_blockalign(bs, align); 171761007b31SStefan Hajnoczi head_iov = (struct iovec) { 171861007b31SStefan Hajnoczi .iov_base = head_buf, 171961007b31SStefan Hajnoczi .iov_len = align, 172061007b31SStefan Hajnoczi }; 172161007b31SStefan Hajnoczi qemu_iovec_init_external(&head_qiov, &head_iov, 1); 172261007b31SStefan Hajnoczi 17239a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 172485c97ca7SKevin Wolf ret = bdrv_aligned_preadv(child, &req, offset & ~(align - 1), align, 172561007b31SStefan Hajnoczi align, &head_qiov, 0); 172661007b31SStefan Hajnoczi if (ret < 0) { 172761007b31SStefan Hajnoczi goto fail; 172861007b31SStefan Hajnoczi } 17299a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 173061007b31SStefan Hajnoczi 173161007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 2); 173261007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 173361007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 173461007b31SStefan Hajnoczi use_local_qiov = true; 173561007b31SStefan Hajnoczi 173661007b31SStefan Hajnoczi bytes += offset & (align - 1); 173761007b31SStefan Hajnoczi offset = offset & ~(align - 1); 1738117bc3faSPeter Lieven 1739117bc3faSPeter Lieven /* We have read the tail already if the request is smaller 1740117bc3faSPeter Lieven * than one aligned block. 1741117bc3faSPeter Lieven */ 1742117bc3faSPeter Lieven if (bytes < align) { 1743117bc3faSPeter Lieven qemu_iovec_add(&local_qiov, head_buf + bytes, align - bytes); 1744117bc3faSPeter Lieven bytes = align; 1745117bc3faSPeter Lieven } 174661007b31SStefan Hajnoczi } 174761007b31SStefan Hajnoczi 174861007b31SStefan Hajnoczi if ((offset + bytes) & (align - 1)) { 174961007b31SStefan Hajnoczi QEMUIOVector tail_qiov; 175061007b31SStefan Hajnoczi struct iovec tail_iov; 175161007b31SStefan Hajnoczi size_t tail_bytes; 175261007b31SStefan Hajnoczi bool waited; 175361007b31SStefan Hajnoczi 175461007b31SStefan Hajnoczi mark_request_serialising(&req, align); 175561007b31SStefan Hajnoczi waited = wait_serialising_requests(&req); 175661007b31SStefan Hajnoczi assert(!waited || !use_local_qiov); 175761007b31SStefan Hajnoczi 175861007b31SStefan Hajnoczi tail_buf = qemu_blockalign(bs, align); 175961007b31SStefan Hajnoczi tail_iov = (struct iovec) { 176061007b31SStefan Hajnoczi .iov_base = tail_buf, 176161007b31SStefan Hajnoczi .iov_len = align, 176261007b31SStefan Hajnoczi }; 176361007b31SStefan Hajnoczi qemu_iovec_init_external(&tail_qiov, &tail_iov, 1); 176461007b31SStefan Hajnoczi 17659a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 176685c97ca7SKevin Wolf ret = bdrv_aligned_preadv(child, &req, (offset + bytes) & ~(align - 1), 176785c97ca7SKevin Wolf align, align, &tail_qiov, 0); 176861007b31SStefan Hajnoczi if (ret < 0) { 176961007b31SStefan Hajnoczi goto fail; 177061007b31SStefan Hajnoczi } 17719a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 177261007b31SStefan Hajnoczi 177361007b31SStefan Hajnoczi if (!use_local_qiov) { 177461007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 1); 177561007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 177661007b31SStefan Hajnoczi use_local_qiov = true; 177761007b31SStefan Hajnoczi } 177861007b31SStefan Hajnoczi 177961007b31SStefan Hajnoczi tail_bytes = (offset + bytes) & (align - 1); 178061007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes); 178161007b31SStefan Hajnoczi 178261007b31SStefan Hajnoczi bytes = ROUND_UP(bytes, align); 178361007b31SStefan Hajnoczi } 178461007b31SStefan Hajnoczi 178585c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align, 178661007b31SStefan Hajnoczi use_local_qiov ? &local_qiov : qiov, 178761007b31SStefan Hajnoczi flags); 178861007b31SStefan Hajnoczi 178961007b31SStefan Hajnoczi fail: 179061007b31SStefan Hajnoczi 179161007b31SStefan Hajnoczi if (use_local_qiov) { 179261007b31SStefan Hajnoczi qemu_iovec_destroy(&local_qiov); 179361007b31SStefan Hajnoczi } 179461007b31SStefan Hajnoczi qemu_vfree(head_buf); 179561007b31SStefan Hajnoczi qemu_vfree(tail_buf); 17969eeb6dd1SFam Zheng out: 17979eeb6dd1SFam Zheng tracked_request_end(&req); 179899723548SPaolo Bonzini bdrv_dec_in_flight(bs); 179961007b31SStefan Hajnoczi return ret; 180061007b31SStefan Hajnoczi } 180161007b31SStefan Hajnoczi 1802adad6496SKevin Wolf static int coroutine_fn bdrv_co_do_writev(BdrvChild *child, 180361007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, 180461007b31SStefan Hajnoczi BdrvRequestFlags flags) 180561007b31SStefan Hajnoczi { 180661007b31SStefan Hajnoczi if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 180761007b31SStefan Hajnoczi return -EINVAL; 180861007b31SStefan Hajnoczi } 180961007b31SStefan Hajnoczi 1810a03ef88fSKevin Wolf return bdrv_co_pwritev(child, sector_num << BDRV_SECTOR_BITS, 181161007b31SStefan Hajnoczi nb_sectors << BDRV_SECTOR_BITS, qiov, flags); 181261007b31SStefan Hajnoczi } 181361007b31SStefan Hajnoczi 181425ec177dSKevin Wolf int coroutine_fn bdrv_co_writev(BdrvChild *child, int64_t sector_num, 181561007b31SStefan Hajnoczi int nb_sectors, QEMUIOVector *qiov) 181661007b31SStefan Hajnoczi { 1817adad6496SKevin Wolf return bdrv_co_do_writev(child, sector_num, nb_sectors, qiov, 0); 181861007b31SStefan Hajnoczi } 181961007b31SStefan Hajnoczi 1820a03ef88fSKevin Wolf int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset, 1821f5a5ca79SManos Pitsidianakis int bytes, BdrvRequestFlags flags) 182261007b31SStefan Hajnoczi { 1823f5a5ca79SManos Pitsidianakis trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags); 182461007b31SStefan Hajnoczi 1825a03ef88fSKevin Wolf if (!(child->bs->open_flags & BDRV_O_UNMAP)) { 182661007b31SStefan Hajnoczi flags &= ~BDRV_REQ_MAY_UNMAP; 182761007b31SStefan Hajnoczi } 182861007b31SStefan Hajnoczi 1829f5a5ca79SManos Pitsidianakis return bdrv_co_pwritev(child, offset, bytes, NULL, 183061007b31SStefan Hajnoczi BDRV_REQ_ZERO_WRITE | flags); 183161007b31SStefan Hajnoczi } 183261007b31SStefan Hajnoczi 18334085f5c7SJohn Snow /* 18344085f5c7SJohn Snow * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not. 18354085f5c7SJohn Snow */ 18364085f5c7SJohn Snow int bdrv_flush_all(void) 18374085f5c7SJohn Snow { 18384085f5c7SJohn Snow BdrvNextIterator it; 18394085f5c7SJohn Snow BlockDriverState *bs = NULL; 18404085f5c7SJohn Snow int result = 0; 18414085f5c7SJohn Snow 18424085f5c7SJohn Snow for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 18434085f5c7SJohn Snow AioContext *aio_context = bdrv_get_aio_context(bs); 18444085f5c7SJohn Snow int ret; 18454085f5c7SJohn Snow 18464085f5c7SJohn Snow aio_context_acquire(aio_context); 18474085f5c7SJohn Snow ret = bdrv_flush(bs); 18484085f5c7SJohn Snow if (ret < 0 && !result) { 18494085f5c7SJohn Snow result = ret; 18504085f5c7SJohn Snow } 18514085f5c7SJohn Snow aio_context_release(aio_context); 18524085f5c7SJohn Snow } 18534085f5c7SJohn Snow 18544085f5c7SJohn Snow return result; 18554085f5c7SJohn Snow } 18564085f5c7SJohn Snow 18574085f5c7SJohn Snow 18584bcd936eSEric Blake typedef struct BdrvCoBlockStatusData { 185961007b31SStefan Hajnoczi BlockDriverState *bs; 186061007b31SStefan Hajnoczi BlockDriverState *base; 1861c9ce8c4dSEric Blake bool want_zero; 18624bcd936eSEric Blake int64_t offset; 18634bcd936eSEric Blake int64_t bytes; 18644bcd936eSEric Blake int64_t *pnum; 18654bcd936eSEric Blake int64_t *map; 1866c9ce8c4dSEric Blake BlockDriverState **file; 18674bcd936eSEric Blake int ret; 186861007b31SStefan Hajnoczi bool done; 18694bcd936eSEric Blake } BdrvCoBlockStatusData; 187061007b31SStefan Hajnoczi 18713e4d0e72SEric Blake int coroutine_fn bdrv_co_block_status_from_file(BlockDriverState *bs, 18723e4d0e72SEric Blake bool want_zero, 18733e4d0e72SEric Blake int64_t offset, 18743e4d0e72SEric Blake int64_t bytes, 18753e4d0e72SEric Blake int64_t *pnum, 18763e4d0e72SEric Blake int64_t *map, 1877f7cc69b3SManos Pitsidianakis BlockDriverState **file) 1878f7cc69b3SManos Pitsidianakis { 1879f7cc69b3SManos Pitsidianakis assert(bs->file && bs->file->bs); 18803e4d0e72SEric Blake *pnum = bytes; 18813e4d0e72SEric Blake *map = offset; 1882f7cc69b3SManos Pitsidianakis *file = bs->file->bs; 18833e4d0e72SEric Blake return BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID; 1884f7cc69b3SManos Pitsidianakis } 1885f7cc69b3SManos Pitsidianakis 18863e4d0e72SEric Blake int coroutine_fn bdrv_co_block_status_from_backing(BlockDriverState *bs, 18873e4d0e72SEric Blake bool want_zero, 18883e4d0e72SEric Blake int64_t offset, 18893e4d0e72SEric Blake int64_t bytes, 18903e4d0e72SEric Blake int64_t *pnum, 18913e4d0e72SEric Blake int64_t *map, 1892f7cc69b3SManos Pitsidianakis BlockDriverState **file) 1893f7cc69b3SManos Pitsidianakis { 1894f7cc69b3SManos Pitsidianakis assert(bs->backing && bs->backing->bs); 18953e4d0e72SEric Blake *pnum = bytes; 18963e4d0e72SEric Blake *map = offset; 1897f7cc69b3SManos Pitsidianakis *file = bs->backing->bs; 18983e4d0e72SEric Blake return BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID; 1899f7cc69b3SManos Pitsidianakis } 1900f7cc69b3SManos Pitsidianakis 190161007b31SStefan Hajnoczi /* 190261007b31SStefan Hajnoczi * Returns the allocation status of the specified sectors. 190361007b31SStefan Hajnoczi * Drivers not implementing the functionality are assumed to not support 190461007b31SStefan Hajnoczi * backing files, hence all their sectors are reported as allocated. 190561007b31SStefan Hajnoczi * 190686a3d5c6SEric Blake * If 'want_zero' is true, the caller is querying for mapping 190786a3d5c6SEric Blake * purposes, with a focus on valid BDRV_BLOCK_OFFSET_VALID, _DATA, and 190886a3d5c6SEric Blake * _ZERO where possible; otherwise, the result favors larger 'pnum', 190986a3d5c6SEric Blake * with a focus on accurate BDRV_BLOCK_ALLOCATED. 1910c9ce8c4dSEric Blake * 19112e8bc787SEric Blake * If 'offset' is beyond the end of the disk image the return value is 1912fb0d8654SEric Blake * BDRV_BLOCK_EOF and 'pnum' is set to 0. 191361007b31SStefan Hajnoczi * 19142e8bc787SEric Blake * 'bytes' is the max value 'pnum' should be set to. If bytes goes 1915fb0d8654SEric Blake * beyond the end of the disk image it will be clamped; if 'pnum' is set to 1916fb0d8654SEric Blake * the end of the image, then the returned value will include BDRV_BLOCK_EOF. 191767a0fd2aSFam Zheng * 19182e8bc787SEric Blake * 'pnum' is set to the number of bytes (including and immediately 19192e8bc787SEric Blake * following the specified offset) that are easily known to be in the 19202e8bc787SEric Blake * same allocated/unallocated state. Note that a second call starting 19212e8bc787SEric Blake * at the original offset plus returned pnum may have the same status. 19222e8bc787SEric Blake * The returned value is non-zero on success except at end-of-file. 19232e8bc787SEric Blake * 19242e8bc787SEric Blake * Returns negative errno on failure. Otherwise, if the 19252e8bc787SEric Blake * BDRV_BLOCK_OFFSET_VALID bit is set, 'map' and 'file' (if non-NULL) are 19262e8bc787SEric Blake * set to the host mapping and BDS corresponding to the guest offset. 192761007b31SStefan Hajnoczi */ 19282e8bc787SEric Blake static int coroutine_fn bdrv_co_block_status(BlockDriverState *bs, 1929c9ce8c4dSEric Blake bool want_zero, 19302e8bc787SEric Blake int64_t offset, int64_t bytes, 19312e8bc787SEric Blake int64_t *pnum, int64_t *map, 193267a0fd2aSFam Zheng BlockDriverState **file) 193361007b31SStefan Hajnoczi { 19342e8bc787SEric Blake int64_t total_size; 19352e8bc787SEric Blake int64_t n; /* bytes */ 1936efa6e2edSEric Blake int ret; 19372e8bc787SEric Blake int64_t local_map = 0; 1938298a1665SEric Blake BlockDriverState *local_file = NULL; 1939efa6e2edSEric Blake int64_t aligned_offset, aligned_bytes; 1940efa6e2edSEric Blake uint32_t align; 194161007b31SStefan Hajnoczi 1942298a1665SEric Blake assert(pnum); 1943298a1665SEric Blake *pnum = 0; 19442e8bc787SEric Blake total_size = bdrv_getlength(bs); 19452e8bc787SEric Blake if (total_size < 0) { 19462e8bc787SEric Blake ret = total_size; 1947298a1665SEric Blake goto early_out; 194861007b31SStefan Hajnoczi } 194961007b31SStefan Hajnoczi 19502e8bc787SEric Blake if (offset >= total_size) { 1951298a1665SEric Blake ret = BDRV_BLOCK_EOF; 1952298a1665SEric Blake goto early_out; 195361007b31SStefan Hajnoczi } 19542e8bc787SEric Blake if (!bytes) { 1955298a1665SEric Blake ret = 0; 1956298a1665SEric Blake goto early_out; 19579cdcfd9fSEric Blake } 195861007b31SStefan Hajnoczi 19592e8bc787SEric Blake n = total_size - offset; 19602e8bc787SEric Blake if (n < bytes) { 19612e8bc787SEric Blake bytes = n; 196261007b31SStefan Hajnoczi } 196361007b31SStefan Hajnoczi 1964d470ad42SMax Reitz /* Must be non-NULL or bdrv_getlength() would have failed */ 1965d470ad42SMax Reitz assert(bs->drv); 1966*636cb512SEric Blake if (!bs->drv->bdrv_co_block_status) { 19672e8bc787SEric Blake *pnum = bytes; 196861007b31SStefan Hajnoczi ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED; 19692e8bc787SEric Blake if (offset + bytes == total_size) { 1970fb0d8654SEric Blake ret |= BDRV_BLOCK_EOF; 1971fb0d8654SEric Blake } 197261007b31SStefan Hajnoczi if (bs->drv->protocol_name) { 19732e8bc787SEric Blake ret |= BDRV_BLOCK_OFFSET_VALID; 19742e8bc787SEric Blake local_map = offset; 1975298a1665SEric Blake local_file = bs; 197661007b31SStefan Hajnoczi } 1977298a1665SEric Blake goto early_out; 197861007b31SStefan Hajnoczi } 197961007b31SStefan Hajnoczi 198099723548SPaolo Bonzini bdrv_inc_in_flight(bs); 1981efa6e2edSEric Blake 1982efa6e2edSEric Blake /* Round out to request_alignment boundaries */ 198386a3d5c6SEric Blake align = bs->bl.request_alignment; 1984efa6e2edSEric Blake aligned_offset = QEMU_ALIGN_DOWN(offset, align); 1985efa6e2edSEric Blake aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset; 1986efa6e2edSEric Blake 198786a3d5c6SEric Blake ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset, 198886a3d5c6SEric Blake aligned_bytes, pnum, &local_map, 198986a3d5c6SEric Blake &local_file); 199086a3d5c6SEric Blake if (ret < 0) { 199186a3d5c6SEric Blake *pnum = 0; 199286a3d5c6SEric Blake goto out; 199386a3d5c6SEric Blake } 1994efa6e2edSEric Blake 1995efa6e2edSEric Blake /* 1996*636cb512SEric Blake * The driver's result must be a non-zero multiple of request_alignment. 1997efa6e2edSEric Blake * Clamp pnum and adjust map to original request. 1998efa6e2edSEric Blake */ 1999*636cb512SEric Blake assert(*pnum && QEMU_IS_ALIGNED(*pnum, align) && 2000*636cb512SEric Blake align > offset - aligned_offset); 2001efa6e2edSEric Blake *pnum -= offset - aligned_offset; 2002efa6e2edSEric Blake if (*pnum > bytes) { 2003efa6e2edSEric Blake *pnum = bytes; 2004efa6e2edSEric Blake } 2005efa6e2edSEric Blake if (ret & BDRV_BLOCK_OFFSET_VALID) { 2006efa6e2edSEric Blake local_map += offset - aligned_offset; 2007efa6e2edSEric Blake } 200861007b31SStefan Hajnoczi 200961007b31SStefan Hajnoczi if (ret & BDRV_BLOCK_RAW) { 2010298a1665SEric Blake assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file); 20112e8bc787SEric Blake ret = bdrv_co_block_status(local_file, want_zero, local_map, 20122e8bc787SEric Blake *pnum, pnum, &local_map, &local_file); 201399723548SPaolo Bonzini goto out; 201461007b31SStefan Hajnoczi } 201561007b31SStefan Hajnoczi 201661007b31SStefan Hajnoczi if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) { 201761007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ALLOCATED; 2018c9ce8c4dSEric Blake } else if (want_zero) { 201961007b31SStefan Hajnoczi if (bdrv_unallocated_blocks_are_zero(bs)) { 202061007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 2021760e0063SKevin Wolf } else if (bs->backing) { 2022760e0063SKevin Wolf BlockDriverState *bs2 = bs->backing->bs; 20232e8bc787SEric Blake int64_t size2 = bdrv_getlength(bs2); 2024c9ce8c4dSEric Blake 20252e8bc787SEric Blake if (size2 >= 0 && offset >= size2) { 202661007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 202761007b31SStefan Hajnoczi } 202861007b31SStefan Hajnoczi } 202961007b31SStefan Hajnoczi } 203061007b31SStefan Hajnoczi 2031c9ce8c4dSEric Blake if (want_zero && local_file && local_file != bs && 203261007b31SStefan Hajnoczi (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) && 203361007b31SStefan Hajnoczi (ret & BDRV_BLOCK_OFFSET_VALID)) { 20342e8bc787SEric Blake int64_t file_pnum; 20352e8bc787SEric Blake int ret2; 203661007b31SStefan Hajnoczi 20372e8bc787SEric Blake ret2 = bdrv_co_block_status(local_file, want_zero, local_map, 20382e8bc787SEric Blake *pnum, &file_pnum, NULL, NULL); 203961007b31SStefan Hajnoczi if (ret2 >= 0) { 204061007b31SStefan Hajnoczi /* Ignore errors. This is just providing extra information, it 204161007b31SStefan Hajnoczi * is useful but not necessary. 204261007b31SStefan Hajnoczi */ 2043c61e684eSEric Blake if (ret2 & BDRV_BLOCK_EOF && 2044c61e684eSEric Blake (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) { 2045c61e684eSEric Blake /* 2046c61e684eSEric Blake * It is valid for the format block driver to read 2047c61e684eSEric Blake * beyond the end of the underlying file's current 2048c61e684eSEric Blake * size; such areas read as zero. 2049c61e684eSEric Blake */ 205061007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 205161007b31SStefan Hajnoczi } else { 205261007b31SStefan Hajnoczi /* Limit request to the range reported by the protocol driver */ 205361007b31SStefan Hajnoczi *pnum = file_pnum; 205461007b31SStefan Hajnoczi ret |= (ret2 & BDRV_BLOCK_ZERO); 205561007b31SStefan Hajnoczi } 205661007b31SStefan Hajnoczi } 205761007b31SStefan Hajnoczi } 205861007b31SStefan Hajnoczi 205999723548SPaolo Bonzini out: 206099723548SPaolo Bonzini bdrv_dec_in_flight(bs); 20612e8bc787SEric Blake if (ret >= 0 && offset + *pnum == total_size) { 2062fb0d8654SEric Blake ret |= BDRV_BLOCK_EOF; 2063fb0d8654SEric Blake } 2064298a1665SEric Blake early_out: 2065298a1665SEric Blake if (file) { 2066298a1665SEric Blake *file = local_file; 2067298a1665SEric Blake } 20682e8bc787SEric Blake if (map) { 20692e8bc787SEric Blake *map = local_map; 20702e8bc787SEric Blake } 207161007b31SStefan Hajnoczi return ret; 207261007b31SStefan Hajnoczi } 207361007b31SStefan Hajnoczi 20745b648c67SEric Blake static int coroutine_fn bdrv_co_block_status_above(BlockDriverState *bs, 2075ba3f0e25SFam Zheng BlockDriverState *base, 2076c9ce8c4dSEric Blake bool want_zero, 20775b648c67SEric Blake int64_t offset, 20785b648c67SEric Blake int64_t bytes, 20795b648c67SEric Blake int64_t *pnum, 20805b648c67SEric Blake int64_t *map, 208167a0fd2aSFam Zheng BlockDriverState **file) 2082ba3f0e25SFam Zheng { 2083ba3f0e25SFam Zheng BlockDriverState *p; 20845b648c67SEric Blake int ret = 0; 2085c61e684eSEric Blake bool first = true; 2086ba3f0e25SFam Zheng 2087ba3f0e25SFam Zheng assert(bs != base); 2088760e0063SKevin Wolf for (p = bs; p != base; p = backing_bs(p)) { 20895b648c67SEric Blake ret = bdrv_co_block_status(p, want_zero, offset, bytes, pnum, map, 20905b648c67SEric Blake file); 2091c61e684eSEric Blake if (ret < 0) { 2092c61e684eSEric Blake break; 2093c61e684eSEric Blake } 2094c61e684eSEric Blake if (ret & BDRV_BLOCK_ZERO && ret & BDRV_BLOCK_EOF && !first) { 2095c61e684eSEric Blake /* 2096c61e684eSEric Blake * Reading beyond the end of the file continues to read 2097c61e684eSEric Blake * zeroes, but we can only widen the result to the 2098c61e684eSEric Blake * unallocated length we learned from an earlier 2099c61e684eSEric Blake * iteration. 2100c61e684eSEric Blake */ 21015b648c67SEric Blake *pnum = bytes; 2102c61e684eSEric Blake } 2103c61e684eSEric Blake if (ret & (BDRV_BLOCK_ZERO | BDRV_BLOCK_DATA)) { 2104ba3f0e25SFam Zheng break; 2105ba3f0e25SFam Zheng } 21065b648c67SEric Blake /* [offset, pnum] unallocated on this layer, which could be only 21075b648c67SEric Blake * the first part of [offset, bytes]. */ 21085b648c67SEric Blake bytes = MIN(bytes, *pnum); 2109c61e684eSEric Blake first = false; 2110ba3f0e25SFam Zheng } 2111ba3f0e25SFam Zheng return ret; 2112ba3f0e25SFam Zheng } 2113ba3f0e25SFam Zheng 211431826642SEric Blake /* Coroutine wrapper for bdrv_block_status_above() */ 21155b648c67SEric Blake static void coroutine_fn bdrv_block_status_above_co_entry(void *opaque) 211661007b31SStefan Hajnoczi { 21174bcd936eSEric Blake BdrvCoBlockStatusData *data = opaque; 211861007b31SStefan Hajnoczi 21195b648c67SEric Blake data->ret = bdrv_co_block_status_above(data->bs, data->base, 2120c9ce8c4dSEric Blake data->want_zero, 21215b648c67SEric Blake data->offset, data->bytes, 21225b648c67SEric Blake data->pnum, data->map, data->file); 212361007b31SStefan Hajnoczi data->done = true; 212461007b31SStefan Hajnoczi } 212561007b31SStefan Hajnoczi 212661007b31SStefan Hajnoczi /* 21275b648c67SEric Blake * Synchronous wrapper around bdrv_co_block_status_above(). 212861007b31SStefan Hajnoczi * 21295b648c67SEric Blake * See bdrv_co_block_status_above() for details. 213061007b31SStefan Hajnoczi */ 21317ddb99b9SEric Blake static int bdrv_common_block_status_above(BlockDriverState *bs, 2132ba3f0e25SFam Zheng BlockDriverState *base, 21337ddb99b9SEric Blake bool want_zero, int64_t offset, 21347ddb99b9SEric Blake int64_t bytes, int64_t *pnum, 21357ddb99b9SEric Blake int64_t *map, 213667a0fd2aSFam Zheng BlockDriverState **file) 213761007b31SStefan Hajnoczi { 213861007b31SStefan Hajnoczi Coroutine *co; 21394bcd936eSEric Blake BdrvCoBlockStatusData data = { 214061007b31SStefan Hajnoczi .bs = bs, 2141ba3f0e25SFam Zheng .base = base, 2142c9ce8c4dSEric Blake .want_zero = want_zero, 21437ddb99b9SEric Blake .offset = offset, 21447ddb99b9SEric Blake .bytes = bytes, 21457ddb99b9SEric Blake .pnum = pnum, 21467ddb99b9SEric Blake .map = map, 2147c9ce8c4dSEric Blake .file = file, 214861007b31SStefan Hajnoczi .done = false, 214961007b31SStefan Hajnoczi }; 215061007b31SStefan Hajnoczi 215161007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 215261007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 21535b648c67SEric Blake bdrv_block_status_above_co_entry(&data); 215461007b31SStefan Hajnoczi } else { 21555b648c67SEric Blake co = qemu_coroutine_create(bdrv_block_status_above_co_entry, &data); 2156e92f0e19SFam Zheng bdrv_coroutine_enter(bs, co); 215788b062c2SPaolo Bonzini BDRV_POLL_WHILE(bs, !data.done); 215861007b31SStefan Hajnoczi } 215961007b31SStefan Hajnoczi return data.ret; 216061007b31SStefan Hajnoczi } 216161007b31SStefan Hajnoczi 216231826642SEric Blake int bdrv_block_status_above(BlockDriverState *bs, BlockDriverState *base, 216331826642SEric Blake int64_t offset, int64_t bytes, int64_t *pnum, 216431826642SEric Blake int64_t *map, BlockDriverState **file) 2165c9ce8c4dSEric Blake { 216631826642SEric Blake return bdrv_common_block_status_above(bs, base, true, offset, bytes, 216731826642SEric Blake pnum, map, file); 2168c9ce8c4dSEric Blake } 2169c9ce8c4dSEric Blake 2170237d78f8SEric Blake int bdrv_block_status(BlockDriverState *bs, int64_t offset, int64_t bytes, 2171237d78f8SEric Blake int64_t *pnum, int64_t *map, BlockDriverState **file) 2172ba3f0e25SFam Zheng { 217331826642SEric Blake return bdrv_block_status_above(bs, backing_bs(bs), 217431826642SEric Blake offset, bytes, pnum, map, file); 2175ba3f0e25SFam Zheng } 2176ba3f0e25SFam Zheng 2177d6a644bbSEric Blake int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t offset, 2178d6a644bbSEric Blake int64_t bytes, int64_t *pnum) 217961007b31SStefan Hajnoczi { 21807ddb99b9SEric Blake int ret; 21817ddb99b9SEric Blake int64_t dummy; 2182d6a644bbSEric Blake 21837ddb99b9SEric Blake ret = bdrv_common_block_status_above(bs, backing_bs(bs), false, offset, 21847ddb99b9SEric Blake bytes, pnum ? pnum : &dummy, NULL, 2185298a1665SEric Blake NULL); 218661007b31SStefan Hajnoczi if (ret < 0) { 218761007b31SStefan Hajnoczi return ret; 218861007b31SStefan Hajnoczi } 218961007b31SStefan Hajnoczi return !!(ret & BDRV_BLOCK_ALLOCATED); 219061007b31SStefan Hajnoczi } 219161007b31SStefan Hajnoczi 219261007b31SStefan Hajnoczi /* 219361007b31SStefan Hajnoczi * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP] 219461007b31SStefan Hajnoczi * 219551b0a488SEric Blake * Return true if (a prefix of) the given range is allocated in any image 219651b0a488SEric Blake * between BASE and TOP (inclusive). BASE can be NULL to check if the given 219751b0a488SEric Blake * offset is allocated in any image of the chain. Return false otherwise, 2198d6a644bbSEric Blake * or negative errno on failure. 219961007b31SStefan Hajnoczi * 220051b0a488SEric Blake * 'pnum' is set to the number of bytes (including and immediately 220151b0a488SEric Blake * following the specified offset) that are known to be in the same 220251b0a488SEric Blake * allocated/unallocated state. Note that a subsequent call starting 220351b0a488SEric Blake * at 'offset + *pnum' may return the same allocation status (in other 220451b0a488SEric Blake * words, the result is not necessarily the maximum possible range); 220551b0a488SEric Blake * but 'pnum' will only be 0 when end of file is reached. 220661007b31SStefan Hajnoczi * 220761007b31SStefan Hajnoczi */ 220861007b31SStefan Hajnoczi int bdrv_is_allocated_above(BlockDriverState *top, 220961007b31SStefan Hajnoczi BlockDriverState *base, 221051b0a488SEric Blake int64_t offset, int64_t bytes, int64_t *pnum) 221161007b31SStefan Hajnoczi { 221261007b31SStefan Hajnoczi BlockDriverState *intermediate; 221351b0a488SEric Blake int ret; 221451b0a488SEric Blake int64_t n = bytes; 221561007b31SStefan Hajnoczi 221661007b31SStefan Hajnoczi intermediate = top; 221761007b31SStefan Hajnoczi while (intermediate && intermediate != base) { 2218d6a644bbSEric Blake int64_t pnum_inter; 2219c00716beSEric Blake int64_t size_inter; 2220d6a644bbSEric Blake 222151b0a488SEric Blake ret = bdrv_is_allocated(intermediate, offset, bytes, &pnum_inter); 222261007b31SStefan Hajnoczi if (ret < 0) { 222361007b31SStefan Hajnoczi return ret; 2224d6a644bbSEric Blake } 2225d6a644bbSEric Blake if (ret) { 222651b0a488SEric Blake *pnum = pnum_inter; 222761007b31SStefan Hajnoczi return 1; 222861007b31SStefan Hajnoczi } 222961007b31SStefan Hajnoczi 223051b0a488SEric Blake size_inter = bdrv_getlength(intermediate); 2231c00716beSEric Blake if (size_inter < 0) { 2232c00716beSEric Blake return size_inter; 2233c00716beSEric Blake } 223451b0a488SEric Blake if (n > pnum_inter && 223551b0a488SEric Blake (intermediate == top || offset + pnum_inter < size_inter)) { 223651b0a488SEric Blake n = pnum_inter; 223761007b31SStefan Hajnoczi } 223861007b31SStefan Hajnoczi 2239760e0063SKevin Wolf intermediate = backing_bs(intermediate); 224061007b31SStefan Hajnoczi } 224161007b31SStefan Hajnoczi 224261007b31SStefan Hajnoczi *pnum = n; 224361007b31SStefan Hajnoczi return 0; 224461007b31SStefan Hajnoczi } 224561007b31SStefan Hajnoczi 22461a8ae822SKevin Wolf typedef struct BdrvVmstateCo { 22471a8ae822SKevin Wolf BlockDriverState *bs; 22481a8ae822SKevin Wolf QEMUIOVector *qiov; 22491a8ae822SKevin Wolf int64_t pos; 22501a8ae822SKevin Wolf bool is_read; 22511a8ae822SKevin Wolf int ret; 22521a8ae822SKevin Wolf } BdrvVmstateCo; 22531a8ae822SKevin Wolf 22541a8ae822SKevin Wolf static int coroutine_fn 22551a8ae822SKevin Wolf bdrv_co_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos, 22561a8ae822SKevin Wolf bool is_read) 22571a8ae822SKevin Wolf { 22581a8ae822SKevin Wolf BlockDriver *drv = bs->drv; 2259dc88a467SStefan Hajnoczi int ret = -ENOTSUP; 2260dc88a467SStefan Hajnoczi 2261dc88a467SStefan Hajnoczi bdrv_inc_in_flight(bs); 22621a8ae822SKevin Wolf 22631a8ae822SKevin Wolf if (!drv) { 2264dc88a467SStefan Hajnoczi ret = -ENOMEDIUM; 22651a8ae822SKevin Wolf } else if (drv->bdrv_load_vmstate) { 2266dc88a467SStefan Hajnoczi if (is_read) { 2267dc88a467SStefan Hajnoczi ret = drv->bdrv_load_vmstate(bs, qiov, pos); 2268dc88a467SStefan Hajnoczi } else { 2269dc88a467SStefan Hajnoczi ret = drv->bdrv_save_vmstate(bs, qiov, pos); 2270dc88a467SStefan Hajnoczi } 22711a8ae822SKevin Wolf } else if (bs->file) { 2272dc88a467SStefan Hajnoczi ret = bdrv_co_rw_vmstate(bs->file->bs, qiov, pos, is_read); 22731a8ae822SKevin Wolf } 22741a8ae822SKevin Wolf 2275dc88a467SStefan Hajnoczi bdrv_dec_in_flight(bs); 2276dc88a467SStefan Hajnoczi return ret; 22771a8ae822SKevin Wolf } 22781a8ae822SKevin Wolf 22791a8ae822SKevin Wolf static void coroutine_fn bdrv_co_rw_vmstate_entry(void *opaque) 22801a8ae822SKevin Wolf { 22811a8ae822SKevin Wolf BdrvVmstateCo *co = opaque; 22821a8ae822SKevin Wolf co->ret = bdrv_co_rw_vmstate(co->bs, co->qiov, co->pos, co->is_read); 22831a8ae822SKevin Wolf } 22841a8ae822SKevin Wolf 22851a8ae822SKevin Wolf static inline int 22861a8ae822SKevin Wolf bdrv_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos, 22871a8ae822SKevin Wolf bool is_read) 22881a8ae822SKevin Wolf { 22891a8ae822SKevin Wolf if (qemu_in_coroutine()) { 22901a8ae822SKevin Wolf return bdrv_co_rw_vmstate(bs, qiov, pos, is_read); 22911a8ae822SKevin Wolf } else { 22921a8ae822SKevin Wolf BdrvVmstateCo data = { 22931a8ae822SKevin Wolf .bs = bs, 22941a8ae822SKevin Wolf .qiov = qiov, 22951a8ae822SKevin Wolf .pos = pos, 22961a8ae822SKevin Wolf .is_read = is_read, 22971a8ae822SKevin Wolf .ret = -EINPROGRESS, 22981a8ae822SKevin Wolf }; 22990b8b8753SPaolo Bonzini Coroutine *co = qemu_coroutine_create(bdrv_co_rw_vmstate_entry, &data); 23001a8ae822SKevin Wolf 2301e92f0e19SFam Zheng bdrv_coroutine_enter(bs, co); 2302ea17c9d2SStefan Hajnoczi BDRV_POLL_WHILE(bs, data.ret == -EINPROGRESS); 23031a8ae822SKevin Wolf return data.ret; 23041a8ae822SKevin Wolf } 23051a8ae822SKevin Wolf } 23061a8ae822SKevin Wolf 230761007b31SStefan Hajnoczi int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf, 230861007b31SStefan Hajnoczi int64_t pos, int size) 230961007b31SStefan Hajnoczi { 231061007b31SStefan Hajnoczi QEMUIOVector qiov; 231161007b31SStefan Hajnoczi struct iovec iov = { 231261007b31SStefan Hajnoczi .iov_base = (void *) buf, 231361007b31SStefan Hajnoczi .iov_len = size, 231461007b31SStefan Hajnoczi }; 2315b433d942SKevin Wolf int ret; 231661007b31SStefan Hajnoczi 231761007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 2318b433d942SKevin Wolf 2319b433d942SKevin Wolf ret = bdrv_writev_vmstate(bs, &qiov, pos); 2320b433d942SKevin Wolf if (ret < 0) { 2321b433d942SKevin Wolf return ret; 2322b433d942SKevin Wolf } 2323b433d942SKevin Wolf 2324b433d942SKevin Wolf return size; 232561007b31SStefan Hajnoczi } 232661007b31SStefan Hajnoczi 232761007b31SStefan Hajnoczi int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 232861007b31SStefan Hajnoczi { 23291a8ae822SKevin Wolf return bdrv_rw_vmstate(bs, qiov, pos, false); 233061007b31SStefan Hajnoczi } 233161007b31SStefan Hajnoczi 233261007b31SStefan Hajnoczi int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf, 233361007b31SStefan Hajnoczi int64_t pos, int size) 233461007b31SStefan Hajnoczi { 23355ddda0b8SKevin Wolf QEMUIOVector qiov; 23365ddda0b8SKevin Wolf struct iovec iov = { 23375ddda0b8SKevin Wolf .iov_base = buf, 23385ddda0b8SKevin Wolf .iov_len = size, 23395ddda0b8SKevin Wolf }; 2340b433d942SKevin Wolf int ret; 23415ddda0b8SKevin Wolf 23425ddda0b8SKevin Wolf qemu_iovec_init_external(&qiov, &iov, 1); 2343b433d942SKevin Wolf ret = bdrv_readv_vmstate(bs, &qiov, pos); 2344b433d942SKevin Wolf if (ret < 0) { 2345b433d942SKevin Wolf return ret; 2346b433d942SKevin Wolf } 2347b433d942SKevin Wolf 2348b433d942SKevin Wolf return size; 23495ddda0b8SKevin Wolf } 23505ddda0b8SKevin Wolf 23515ddda0b8SKevin Wolf int bdrv_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 23525ddda0b8SKevin Wolf { 23531a8ae822SKevin Wolf return bdrv_rw_vmstate(bs, qiov, pos, true); 235461007b31SStefan Hajnoczi } 235561007b31SStefan Hajnoczi 235661007b31SStefan Hajnoczi /**************************************************************/ 235761007b31SStefan Hajnoczi /* async I/Os */ 235861007b31SStefan Hajnoczi 235961007b31SStefan Hajnoczi void bdrv_aio_cancel(BlockAIOCB *acb) 236061007b31SStefan Hajnoczi { 236161007b31SStefan Hajnoczi qemu_aio_ref(acb); 236261007b31SStefan Hajnoczi bdrv_aio_cancel_async(acb); 236361007b31SStefan Hajnoczi while (acb->refcnt > 1) { 236461007b31SStefan Hajnoczi if (acb->aiocb_info->get_aio_context) { 236561007b31SStefan Hajnoczi aio_poll(acb->aiocb_info->get_aio_context(acb), true); 236661007b31SStefan Hajnoczi } else if (acb->bs) { 23672f47da5fSPaolo Bonzini /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so 23682f47da5fSPaolo Bonzini * assert that we're not using an I/O thread. Thread-safe 23692f47da5fSPaolo Bonzini * code should use bdrv_aio_cancel_async exclusively. 23702f47da5fSPaolo Bonzini */ 23712f47da5fSPaolo Bonzini assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context()); 237261007b31SStefan Hajnoczi aio_poll(bdrv_get_aio_context(acb->bs), true); 237361007b31SStefan Hajnoczi } else { 237461007b31SStefan Hajnoczi abort(); 237561007b31SStefan Hajnoczi } 237661007b31SStefan Hajnoczi } 237761007b31SStefan Hajnoczi qemu_aio_unref(acb); 237861007b31SStefan Hajnoczi } 237961007b31SStefan Hajnoczi 238061007b31SStefan Hajnoczi /* Async version of aio cancel. The caller is not blocked if the acb implements 238161007b31SStefan Hajnoczi * cancel_async, otherwise we do nothing and let the request normally complete. 238261007b31SStefan Hajnoczi * In either case the completion callback must be called. */ 238361007b31SStefan Hajnoczi void bdrv_aio_cancel_async(BlockAIOCB *acb) 238461007b31SStefan Hajnoczi { 238561007b31SStefan Hajnoczi if (acb->aiocb_info->cancel_async) { 238661007b31SStefan Hajnoczi acb->aiocb_info->cancel_async(acb); 238761007b31SStefan Hajnoczi } 238861007b31SStefan Hajnoczi } 238961007b31SStefan Hajnoczi 239061007b31SStefan Hajnoczi /**************************************************************/ 239161007b31SStefan Hajnoczi /* Coroutine block device emulation */ 239261007b31SStefan Hajnoczi 2393e293b7a3SKevin Wolf typedef struct FlushCo { 2394e293b7a3SKevin Wolf BlockDriverState *bs; 2395e293b7a3SKevin Wolf int ret; 2396e293b7a3SKevin Wolf } FlushCo; 2397e293b7a3SKevin Wolf 2398e293b7a3SKevin Wolf 239961007b31SStefan Hajnoczi static void coroutine_fn bdrv_flush_co_entry(void *opaque) 240061007b31SStefan Hajnoczi { 2401e293b7a3SKevin Wolf FlushCo *rwco = opaque; 240261007b31SStefan Hajnoczi 240361007b31SStefan Hajnoczi rwco->ret = bdrv_co_flush(rwco->bs); 240461007b31SStefan Hajnoczi } 240561007b31SStefan Hajnoczi 240661007b31SStefan Hajnoczi int coroutine_fn bdrv_co_flush(BlockDriverState *bs) 240761007b31SStefan Hajnoczi { 240849ca6259SFam Zheng int current_gen; 240949ca6259SFam Zheng int ret = 0; 241061007b31SStefan Hajnoczi 241199723548SPaolo Bonzini bdrv_inc_in_flight(bs); 2412c32b82afSPavel Dovgalyuk 2413e914404eSFam Zheng if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs) || 241449ca6259SFam Zheng bdrv_is_sg(bs)) { 241549ca6259SFam Zheng goto early_exit; 241649ca6259SFam Zheng } 241749ca6259SFam Zheng 24183783fa3dSPaolo Bonzini qemu_co_mutex_lock(&bs->reqs_lock); 241947fec599SPaolo Bonzini current_gen = atomic_read(&bs->write_gen); 24203ff2f67aSEvgeny Yakovlev 24213ff2f67aSEvgeny Yakovlev /* Wait until any previous flushes are completed */ 242299723548SPaolo Bonzini while (bs->active_flush_req) { 24233783fa3dSPaolo Bonzini qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock); 24243ff2f67aSEvgeny Yakovlev } 24253ff2f67aSEvgeny Yakovlev 24263783fa3dSPaolo Bonzini /* Flushes reach this point in nondecreasing current_gen order. */ 242799723548SPaolo Bonzini bs->active_flush_req = true; 24283783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&bs->reqs_lock); 24293ff2f67aSEvgeny Yakovlev 2430c32b82afSPavel Dovgalyuk /* Write back all layers by calling one driver function */ 2431c32b82afSPavel Dovgalyuk if (bs->drv->bdrv_co_flush) { 2432c32b82afSPavel Dovgalyuk ret = bs->drv->bdrv_co_flush(bs); 2433c32b82afSPavel Dovgalyuk goto out; 2434c32b82afSPavel Dovgalyuk } 2435c32b82afSPavel Dovgalyuk 243661007b31SStefan Hajnoczi /* Write back cached data to the OS even with cache=unsafe */ 243761007b31SStefan Hajnoczi BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS); 243861007b31SStefan Hajnoczi if (bs->drv->bdrv_co_flush_to_os) { 243961007b31SStefan Hajnoczi ret = bs->drv->bdrv_co_flush_to_os(bs); 244061007b31SStefan Hajnoczi if (ret < 0) { 2441cdb5e315SFam Zheng goto out; 244261007b31SStefan Hajnoczi } 244361007b31SStefan Hajnoczi } 244461007b31SStefan Hajnoczi 244561007b31SStefan Hajnoczi /* But don't actually force it to the disk with cache=unsafe */ 244661007b31SStefan Hajnoczi if (bs->open_flags & BDRV_O_NO_FLUSH) { 244761007b31SStefan Hajnoczi goto flush_parent; 244861007b31SStefan Hajnoczi } 244961007b31SStefan Hajnoczi 24503ff2f67aSEvgeny Yakovlev /* Check if we really need to flush anything */ 24513ff2f67aSEvgeny Yakovlev if (bs->flushed_gen == current_gen) { 24523ff2f67aSEvgeny Yakovlev goto flush_parent; 24533ff2f67aSEvgeny Yakovlev } 24543ff2f67aSEvgeny Yakovlev 245561007b31SStefan Hajnoczi BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK); 2456d470ad42SMax Reitz if (!bs->drv) { 2457d470ad42SMax Reitz /* bs->drv->bdrv_co_flush() might have ejected the BDS 2458d470ad42SMax Reitz * (even in case of apparent success) */ 2459d470ad42SMax Reitz ret = -ENOMEDIUM; 2460d470ad42SMax Reitz goto out; 2461d470ad42SMax Reitz } 246261007b31SStefan Hajnoczi if (bs->drv->bdrv_co_flush_to_disk) { 246361007b31SStefan Hajnoczi ret = bs->drv->bdrv_co_flush_to_disk(bs); 246461007b31SStefan Hajnoczi } else if (bs->drv->bdrv_aio_flush) { 246561007b31SStefan Hajnoczi BlockAIOCB *acb; 246661007b31SStefan Hajnoczi CoroutineIOCompletion co = { 246761007b31SStefan Hajnoczi .coroutine = qemu_coroutine_self(), 246861007b31SStefan Hajnoczi }; 246961007b31SStefan Hajnoczi 247061007b31SStefan Hajnoczi acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co); 247161007b31SStefan Hajnoczi if (acb == NULL) { 247261007b31SStefan Hajnoczi ret = -EIO; 247361007b31SStefan Hajnoczi } else { 247461007b31SStefan Hajnoczi qemu_coroutine_yield(); 247561007b31SStefan Hajnoczi ret = co.ret; 247661007b31SStefan Hajnoczi } 247761007b31SStefan Hajnoczi } else { 247861007b31SStefan Hajnoczi /* 247961007b31SStefan Hajnoczi * Some block drivers always operate in either writethrough or unsafe 248061007b31SStefan Hajnoczi * mode and don't support bdrv_flush therefore. Usually qemu doesn't 248161007b31SStefan Hajnoczi * know how the server works (because the behaviour is hardcoded or 248261007b31SStefan Hajnoczi * depends on server-side configuration), so we can't ensure that 248361007b31SStefan Hajnoczi * everything is safe on disk. Returning an error doesn't work because 248461007b31SStefan Hajnoczi * that would break guests even if the server operates in writethrough 248561007b31SStefan Hajnoczi * mode. 248661007b31SStefan Hajnoczi * 248761007b31SStefan Hajnoczi * Let's hope the user knows what he's doing. 248861007b31SStefan Hajnoczi */ 248961007b31SStefan Hajnoczi ret = 0; 249061007b31SStefan Hajnoczi } 24913ff2f67aSEvgeny Yakovlev 249261007b31SStefan Hajnoczi if (ret < 0) { 2493cdb5e315SFam Zheng goto out; 249461007b31SStefan Hajnoczi } 249561007b31SStefan Hajnoczi 249661007b31SStefan Hajnoczi /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH 249761007b31SStefan Hajnoczi * in the case of cache=unsafe, so there are no useless flushes. 249861007b31SStefan Hajnoczi */ 249961007b31SStefan Hajnoczi flush_parent: 2500cdb5e315SFam Zheng ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0; 2501cdb5e315SFam Zheng out: 25023ff2f67aSEvgeny Yakovlev /* Notify any pending flushes that we have completed */ 2503e6af1e08SKevin Wolf if (ret == 0) { 25043ff2f67aSEvgeny Yakovlev bs->flushed_gen = current_gen; 2505e6af1e08SKevin Wolf } 25063783fa3dSPaolo Bonzini 25073783fa3dSPaolo Bonzini qemu_co_mutex_lock(&bs->reqs_lock); 250899723548SPaolo Bonzini bs->active_flush_req = false; 2509156af3acSDenis V. Lunev /* Return value is ignored - it's ok if wait queue is empty */ 2510156af3acSDenis V. Lunev qemu_co_queue_next(&bs->flush_queue); 25113783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&bs->reqs_lock); 25123ff2f67aSEvgeny Yakovlev 251349ca6259SFam Zheng early_exit: 251499723548SPaolo Bonzini bdrv_dec_in_flight(bs); 2515cdb5e315SFam Zheng return ret; 251661007b31SStefan Hajnoczi } 251761007b31SStefan Hajnoczi 251861007b31SStefan Hajnoczi int bdrv_flush(BlockDriverState *bs) 251961007b31SStefan Hajnoczi { 252061007b31SStefan Hajnoczi Coroutine *co; 2521e293b7a3SKevin Wolf FlushCo flush_co = { 252261007b31SStefan Hajnoczi .bs = bs, 252361007b31SStefan Hajnoczi .ret = NOT_DONE, 252461007b31SStefan Hajnoczi }; 252561007b31SStefan Hajnoczi 252661007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 252761007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 2528e293b7a3SKevin Wolf bdrv_flush_co_entry(&flush_co); 252961007b31SStefan Hajnoczi } else { 25300b8b8753SPaolo Bonzini co = qemu_coroutine_create(bdrv_flush_co_entry, &flush_co); 2531e92f0e19SFam Zheng bdrv_coroutine_enter(bs, co); 253288b062c2SPaolo Bonzini BDRV_POLL_WHILE(bs, flush_co.ret == NOT_DONE); 253361007b31SStefan Hajnoczi } 253461007b31SStefan Hajnoczi 2535e293b7a3SKevin Wolf return flush_co.ret; 253661007b31SStefan Hajnoczi } 253761007b31SStefan Hajnoczi 253861007b31SStefan Hajnoczi typedef struct DiscardCo { 253961007b31SStefan Hajnoczi BlockDriverState *bs; 25400c51a893SEric Blake int64_t offset; 2541f5a5ca79SManos Pitsidianakis int bytes; 254261007b31SStefan Hajnoczi int ret; 254361007b31SStefan Hajnoczi } DiscardCo; 25440c51a893SEric Blake static void coroutine_fn bdrv_pdiscard_co_entry(void *opaque) 254561007b31SStefan Hajnoczi { 254661007b31SStefan Hajnoczi DiscardCo *rwco = opaque; 254761007b31SStefan Hajnoczi 2548f5a5ca79SManos Pitsidianakis rwco->ret = bdrv_co_pdiscard(rwco->bs, rwco->offset, rwco->bytes); 254961007b31SStefan Hajnoczi } 255061007b31SStefan Hajnoczi 25519f1963b3SEric Blake int coroutine_fn bdrv_co_pdiscard(BlockDriverState *bs, int64_t offset, 2552f5a5ca79SManos Pitsidianakis int bytes) 255361007b31SStefan Hajnoczi { 2554b1066c87SFam Zheng BdrvTrackedRequest req; 25559f1963b3SEric Blake int max_pdiscard, ret; 25563482b9bcSEric Blake int head, tail, align; 255761007b31SStefan Hajnoczi 255861007b31SStefan Hajnoczi if (!bs->drv) { 255961007b31SStefan Hajnoczi return -ENOMEDIUM; 256061007b31SStefan Hajnoczi } 256161007b31SStefan Hajnoczi 2562d6883bc9SVladimir Sementsov-Ogievskiy if (bdrv_has_readonly_bitmaps(bs)) { 2563d6883bc9SVladimir Sementsov-Ogievskiy return -EPERM; 2564d6883bc9SVladimir Sementsov-Ogievskiy } 2565d6883bc9SVladimir Sementsov-Ogievskiy 2566f5a5ca79SManos Pitsidianakis ret = bdrv_check_byte_request(bs, offset, bytes); 256761007b31SStefan Hajnoczi if (ret < 0) { 256861007b31SStefan Hajnoczi return ret; 256961007b31SStefan Hajnoczi } else if (bs->read_only) { 2570eaf5fe2dSPaolo Bonzini return -EPERM; 257161007b31SStefan Hajnoczi } 257204c01a5cSKevin Wolf assert(!(bs->open_flags & BDRV_O_INACTIVE)); 257361007b31SStefan Hajnoczi 257461007b31SStefan Hajnoczi /* Do nothing if disabled. */ 257561007b31SStefan Hajnoczi if (!(bs->open_flags & BDRV_O_UNMAP)) { 257661007b31SStefan Hajnoczi return 0; 257761007b31SStefan Hajnoczi } 257861007b31SStefan Hajnoczi 257902aefe43SEric Blake if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) { 258061007b31SStefan Hajnoczi return 0; 258161007b31SStefan Hajnoczi } 258261007b31SStefan Hajnoczi 25833482b9bcSEric Blake /* Discard is advisory, but some devices track and coalesce 25843482b9bcSEric Blake * unaligned requests, so we must pass everything down rather than 25853482b9bcSEric Blake * round here. Still, most devices will just silently ignore 25863482b9bcSEric Blake * unaligned requests (by returning -ENOTSUP), so we must fragment 25873482b9bcSEric Blake * the request accordingly. */ 258802aefe43SEric Blake align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment); 2589b8d0a980SEric Blake assert(align % bs->bl.request_alignment == 0); 2590b8d0a980SEric Blake head = offset % align; 2591f5a5ca79SManos Pitsidianakis tail = (offset + bytes) % align; 25929f1963b3SEric Blake 259399723548SPaolo Bonzini bdrv_inc_in_flight(bs); 2594f5a5ca79SManos Pitsidianakis tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD); 259550824995SFam Zheng 2596ec050f77SDenis V. Lunev ret = notifier_with_return_list_notify(&bs->before_write_notifiers, &req); 2597ec050f77SDenis V. Lunev if (ret < 0) { 2598ec050f77SDenis V. Lunev goto out; 2599ec050f77SDenis V. Lunev } 2600ec050f77SDenis V. Lunev 26019f1963b3SEric Blake max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT_MAX), 26029f1963b3SEric Blake align); 26033482b9bcSEric Blake assert(max_pdiscard >= bs->bl.request_alignment); 26049f1963b3SEric Blake 2605f5a5ca79SManos Pitsidianakis while (bytes > 0) { 2606f5a5ca79SManos Pitsidianakis int num = bytes; 26073482b9bcSEric Blake 26083482b9bcSEric Blake if (head) { 26093482b9bcSEric Blake /* Make small requests to get to alignment boundaries. */ 2610f5a5ca79SManos Pitsidianakis num = MIN(bytes, align - head); 26113482b9bcSEric Blake if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) { 26123482b9bcSEric Blake num %= bs->bl.request_alignment; 26133482b9bcSEric Blake } 26143482b9bcSEric Blake head = (head + num) % align; 26153482b9bcSEric Blake assert(num < max_pdiscard); 26163482b9bcSEric Blake } else if (tail) { 26173482b9bcSEric Blake if (num > align) { 26183482b9bcSEric Blake /* Shorten the request to the last aligned cluster. */ 26193482b9bcSEric Blake num -= tail; 26203482b9bcSEric Blake } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) && 26213482b9bcSEric Blake tail > bs->bl.request_alignment) { 26223482b9bcSEric Blake tail %= bs->bl.request_alignment; 26233482b9bcSEric Blake num -= tail; 26243482b9bcSEric Blake } 26253482b9bcSEric Blake } 26263482b9bcSEric Blake /* limit request size */ 26273482b9bcSEric Blake if (num > max_pdiscard) { 26283482b9bcSEric Blake num = max_pdiscard; 26293482b9bcSEric Blake } 263061007b31SStefan Hajnoczi 2631d470ad42SMax Reitz if (!bs->drv) { 2632d470ad42SMax Reitz ret = -ENOMEDIUM; 2633d470ad42SMax Reitz goto out; 2634d470ad42SMax Reitz } 263547a5486dSEric Blake if (bs->drv->bdrv_co_pdiscard) { 263647a5486dSEric Blake ret = bs->drv->bdrv_co_pdiscard(bs, offset, num); 263761007b31SStefan Hajnoczi } else { 263861007b31SStefan Hajnoczi BlockAIOCB *acb; 263961007b31SStefan Hajnoczi CoroutineIOCompletion co = { 264061007b31SStefan Hajnoczi .coroutine = qemu_coroutine_self(), 264161007b31SStefan Hajnoczi }; 264261007b31SStefan Hajnoczi 26434da444a0SEric Blake acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num, 264461007b31SStefan Hajnoczi bdrv_co_io_em_complete, &co); 264561007b31SStefan Hajnoczi if (acb == NULL) { 2646b1066c87SFam Zheng ret = -EIO; 2647b1066c87SFam Zheng goto out; 264861007b31SStefan Hajnoczi } else { 264961007b31SStefan Hajnoczi qemu_coroutine_yield(); 265061007b31SStefan Hajnoczi ret = co.ret; 265161007b31SStefan Hajnoczi } 265261007b31SStefan Hajnoczi } 265361007b31SStefan Hajnoczi if (ret && ret != -ENOTSUP) { 2654b1066c87SFam Zheng goto out; 265561007b31SStefan Hajnoczi } 265661007b31SStefan Hajnoczi 26579f1963b3SEric Blake offset += num; 2658f5a5ca79SManos Pitsidianakis bytes -= num; 265961007b31SStefan Hajnoczi } 2660b1066c87SFam Zheng ret = 0; 2661b1066c87SFam Zheng out: 266247fec599SPaolo Bonzini atomic_inc(&bs->write_gen); 26630fdf1a4fSEric Blake bdrv_set_dirty(bs, req.offset, req.bytes); 2664b1066c87SFam Zheng tracked_request_end(&req); 266599723548SPaolo Bonzini bdrv_dec_in_flight(bs); 2666b1066c87SFam Zheng return ret; 266761007b31SStefan Hajnoczi } 266861007b31SStefan Hajnoczi 2669f5a5ca79SManos Pitsidianakis int bdrv_pdiscard(BlockDriverState *bs, int64_t offset, int bytes) 267061007b31SStefan Hajnoczi { 267161007b31SStefan Hajnoczi Coroutine *co; 267261007b31SStefan Hajnoczi DiscardCo rwco = { 267361007b31SStefan Hajnoczi .bs = bs, 26740c51a893SEric Blake .offset = offset, 2675f5a5ca79SManos Pitsidianakis .bytes = bytes, 267661007b31SStefan Hajnoczi .ret = NOT_DONE, 267761007b31SStefan Hajnoczi }; 267861007b31SStefan Hajnoczi 267961007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 268061007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 26810c51a893SEric Blake bdrv_pdiscard_co_entry(&rwco); 268261007b31SStefan Hajnoczi } else { 26830c51a893SEric Blake co = qemu_coroutine_create(bdrv_pdiscard_co_entry, &rwco); 2684e92f0e19SFam Zheng bdrv_coroutine_enter(bs, co); 268588b062c2SPaolo Bonzini BDRV_POLL_WHILE(bs, rwco.ret == NOT_DONE); 268661007b31SStefan Hajnoczi } 268761007b31SStefan Hajnoczi 268861007b31SStefan Hajnoczi return rwco.ret; 268961007b31SStefan Hajnoczi } 269061007b31SStefan Hajnoczi 269148af776aSKevin Wolf int bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf) 269261007b31SStefan Hajnoczi { 269361007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 26945c5ae76aSFam Zheng CoroutineIOCompletion co = { 26955c5ae76aSFam Zheng .coroutine = qemu_coroutine_self(), 26965c5ae76aSFam Zheng }; 26975c5ae76aSFam Zheng BlockAIOCB *acb; 269861007b31SStefan Hajnoczi 269999723548SPaolo Bonzini bdrv_inc_in_flight(bs); 270016a389dcSKevin Wolf if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) { 27015c5ae76aSFam Zheng co.ret = -ENOTSUP; 27025c5ae76aSFam Zheng goto out; 27035c5ae76aSFam Zheng } 27045c5ae76aSFam Zheng 270516a389dcSKevin Wolf if (drv->bdrv_co_ioctl) { 270616a389dcSKevin Wolf co.ret = drv->bdrv_co_ioctl(bs, req, buf); 270716a389dcSKevin Wolf } else { 27085c5ae76aSFam Zheng acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co); 27095c5ae76aSFam Zheng if (!acb) { 2710c8a9fd80SFam Zheng co.ret = -ENOTSUP; 2711c8a9fd80SFam Zheng goto out; 27125c5ae76aSFam Zheng } 27135c5ae76aSFam Zheng qemu_coroutine_yield(); 271416a389dcSKevin Wolf } 27155c5ae76aSFam Zheng out: 271699723548SPaolo Bonzini bdrv_dec_in_flight(bs); 27175c5ae76aSFam Zheng return co.ret; 27185c5ae76aSFam Zheng } 27195c5ae76aSFam Zheng 272061007b31SStefan Hajnoczi void *qemu_blockalign(BlockDriverState *bs, size_t size) 272161007b31SStefan Hajnoczi { 272261007b31SStefan Hajnoczi return qemu_memalign(bdrv_opt_mem_align(bs), size); 272361007b31SStefan Hajnoczi } 272461007b31SStefan Hajnoczi 272561007b31SStefan Hajnoczi void *qemu_blockalign0(BlockDriverState *bs, size_t size) 272661007b31SStefan Hajnoczi { 272761007b31SStefan Hajnoczi return memset(qemu_blockalign(bs, size), 0, size); 272861007b31SStefan Hajnoczi } 272961007b31SStefan Hajnoczi 273061007b31SStefan Hajnoczi void *qemu_try_blockalign(BlockDriverState *bs, size_t size) 273161007b31SStefan Hajnoczi { 273261007b31SStefan Hajnoczi size_t align = bdrv_opt_mem_align(bs); 273361007b31SStefan Hajnoczi 273461007b31SStefan Hajnoczi /* Ensure that NULL is never returned on success */ 273561007b31SStefan Hajnoczi assert(align > 0); 273661007b31SStefan Hajnoczi if (size == 0) { 273761007b31SStefan Hajnoczi size = align; 273861007b31SStefan Hajnoczi } 273961007b31SStefan Hajnoczi 274061007b31SStefan Hajnoczi return qemu_try_memalign(align, size); 274161007b31SStefan Hajnoczi } 274261007b31SStefan Hajnoczi 274361007b31SStefan Hajnoczi void *qemu_try_blockalign0(BlockDriverState *bs, size_t size) 274461007b31SStefan Hajnoczi { 274561007b31SStefan Hajnoczi void *mem = qemu_try_blockalign(bs, size); 274661007b31SStefan Hajnoczi 274761007b31SStefan Hajnoczi if (mem) { 274861007b31SStefan Hajnoczi memset(mem, 0, size); 274961007b31SStefan Hajnoczi } 275061007b31SStefan Hajnoczi 275161007b31SStefan Hajnoczi return mem; 275261007b31SStefan Hajnoczi } 275361007b31SStefan Hajnoczi 275461007b31SStefan Hajnoczi /* 275561007b31SStefan Hajnoczi * Check if all memory in this vector is sector aligned. 275661007b31SStefan Hajnoczi */ 275761007b31SStefan Hajnoczi bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov) 275861007b31SStefan Hajnoczi { 275961007b31SStefan Hajnoczi int i; 27604196d2f0SDenis V. Lunev size_t alignment = bdrv_min_mem_align(bs); 276161007b31SStefan Hajnoczi 276261007b31SStefan Hajnoczi for (i = 0; i < qiov->niov; i++) { 276361007b31SStefan Hajnoczi if ((uintptr_t) qiov->iov[i].iov_base % alignment) { 276461007b31SStefan Hajnoczi return false; 276561007b31SStefan Hajnoczi } 276661007b31SStefan Hajnoczi if (qiov->iov[i].iov_len % alignment) { 276761007b31SStefan Hajnoczi return false; 276861007b31SStefan Hajnoczi } 276961007b31SStefan Hajnoczi } 277061007b31SStefan Hajnoczi 277161007b31SStefan Hajnoczi return true; 277261007b31SStefan Hajnoczi } 277361007b31SStefan Hajnoczi 277461007b31SStefan Hajnoczi void bdrv_add_before_write_notifier(BlockDriverState *bs, 277561007b31SStefan Hajnoczi NotifierWithReturn *notifier) 277661007b31SStefan Hajnoczi { 277761007b31SStefan Hajnoczi notifier_with_return_list_add(&bs->before_write_notifiers, notifier); 277861007b31SStefan Hajnoczi } 277961007b31SStefan Hajnoczi 278061007b31SStefan Hajnoczi void bdrv_io_plug(BlockDriverState *bs) 278161007b31SStefan Hajnoczi { 27826b98bd64SPaolo Bonzini BdrvChild *child; 27836b98bd64SPaolo Bonzini 27846b98bd64SPaolo Bonzini QLIST_FOREACH(child, &bs->children, next) { 27856b98bd64SPaolo Bonzini bdrv_io_plug(child->bs); 27866b98bd64SPaolo Bonzini } 27876b98bd64SPaolo Bonzini 2788850d54a2SPaolo Bonzini if (atomic_fetch_inc(&bs->io_plugged) == 0) { 278961007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 279061007b31SStefan Hajnoczi if (drv && drv->bdrv_io_plug) { 279161007b31SStefan Hajnoczi drv->bdrv_io_plug(bs); 27926b98bd64SPaolo Bonzini } 279361007b31SStefan Hajnoczi } 279461007b31SStefan Hajnoczi } 279561007b31SStefan Hajnoczi 279661007b31SStefan Hajnoczi void bdrv_io_unplug(BlockDriverState *bs) 279761007b31SStefan Hajnoczi { 27986b98bd64SPaolo Bonzini BdrvChild *child; 27996b98bd64SPaolo Bonzini 28006b98bd64SPaolo Bonzini assert(bs->io_plugged); 2801850d54a2SPaolo Bonzini if (atomic_fetch_dec(&bs->io_plugged) == 1) { 280261007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 280361007b31SStefan Hajnoczi if (drv && drv->bdrv_io_unplug) { 280461007b31SStefan Hajnoczi drv->bdrv_io_unplug(bs); 280561007b31SStefan Hajnoczi } 280661007b31SStefan Hajnoczi } 280761007b31SStefan Hajnoczi 28086b98bd64SPaolo Bonzini QLIST_FOREACH(child, &bs->children, next) { 28096b98bd64SPaolo Bonzini bdrv_io_unplug(child->bs); 28106b98bd64SPaolo Bonzini } 28116b98bd64SPaolo Bonzini } 281223d0ba93SFam Zheng 281323d0ba93SFam Zheng void bdrv_register_buf(BlockDriverState *bs, void *host, size_t size) 281423d0ba93SFam Zheng { 281523d0ba93SFam Zheng BdrvChild *child; 281623d0ba93SFam Zheng 281723d0ba93SFam Zheng if (bs->drv && bs->drv->bdrv_register_buf) { 281823d0ba93SFam Zheng bs->drv->bdrv_register_buf(bs, host, size); 281923d0ba93SFam Zheng } 282023d0ba93SFam Zheng QLIST_FOREACH(child, &bs->children, next) { 282123d0ba93SFam Zheng bdrv_register_buf(child->bs, host, size); 282223d0ba93SFam Zheng } 282323d0ba93SFam Zheng } 282423d0ba93SFam Zheng 282523d0ba93SFam Zheng void bdrv_unregister_buf(BlockDriverState *bs, void *host) 282623d0ba93SFam Zheng { 282723d0ba93SFam Zheng BdrvChild *child; 282823d0ba93SFam Zheng 282923d0ba93SFam Zheng if (bs->drv && bs->drv->bdrv_unregister_buf) { 283023d0ba93SFam Zheng bs->drv->bdrv_unregister_buf(bs, host); 283123d0ba93SFam Zheng } 283223d0ba93SFam Zheng QLIST_FOREACH(child, &bs->children, next) { 283323d0ba93SFam Zheng bdrv_unregister_buf(child->bs, host); 283423d0ba93SFam Zheng } 283523d0ba93SFam Zheng } 2836