161007b31SStefan Hajnoczi /* 261007b31SStefan Hajnoczi * Block layer I/O functions 361007b31SStefan Hajnoczi * 461007b31SStefan Hajnoczi * Copyright (c) 2003 Fabrice Bellard 561007b31SStefan Hajnoczi * 661007b31SStefan Hajnoczi * Permission is hereby granted, free of charge, to any person obtaining a copy 761007b31SStefan Hajnoczi * of this software and associated documentation files (the "Software"), to deal 861007b31SStefan Hajnoczi * in the Software without restriction, including without limitation the rights 961007b31SStefan Hajnoczi * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 1061007b31SStefan Hajnoczi * copies of the Software, and to permit persons to whom the Software is 1161007b31SStefan Hajnoczi * furnished to do so, subject to the following conditions: 1261007b31SStefan Hajnoczi * 1361007b31SStefan Hajnoczi * The above copyright notice and this permission notice shall be included in 1461007b31SStefan Hajnoczi * all copies or substantial portions of the Software. 1561007b31SStefan Hajnoczi * 1661007b31SStefan Hajnoczi * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 1761007b31SStefan Hajnoczi * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 1861007b31SStefan Hajnoczi * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 1961007b31SStefan Hajnoczi * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 2061007b31SStefan Hajnoczi * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 2161007b31SStefan Hajnoczi * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 2261007b31SStefan Hajnoczi * THE SOFTWARE. 2361007b31SStefan Hajnoczi */ 2461007b31SStefan Hajnoczi 2580c71a24SPeter Maydell #include "qemu/osdep.h" 2661007b31SStefan Hajnoczi #include "trace.h" 277f0e9da6SMax Reitz #include "sysemu/block-backend.h" 2861007b31SStefan Hajnoczi #include "block/blockjob.h" 29f321dcb5SPaolo Bonzini #include "block/blockjob_int.h" 3061007b31SStefan Hajnoczi #include "block/block_int.h" 31f348b6d1SVeronia Bahaa #include "qemu/cutils.h" 32da34e65cSMarkus Armbruster #include "qapi/error.h" 33d49b6836SMarkus Armbruster #include "qemu/error-report.h" 3461007b31SStefan Hajnoczi 3561007b31SStefan Hajnoczi #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */ 3661007b31SStefan Hajnoczi 37cb2e2878SEric Blake /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */ 38cb2e2878SEric Blake #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS) 39cb2e2878SEric Blake 40d05aa8bbSEric Blake static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 41f5a5ca79SManos Pitsidianakis int64_t offset, int bytes, BdrvRequestFlags flags); 4261007b31SStefan Hajnoczi 4314e9559fSFam Zheng void bdrv_parent_drained_begin(BlockDriverState *bs) 4461007b31SStefan Hajnoczi { 45c2066af0SKevin Wolf BdrvChild *c; 4627ccdd52SKevin Wolf 47c2066af0SKevin Wolf QLIST_FOREACH(c, &bs->parents, next_parent) { 48c2066af0SKevin Wolf if (c->role->drained_begin) { 49c2066af0SKevin Wolf c->role->drained_begin(c); 50c2066af0SKevin Wolf } 51ce0f1412SPaolo Bonzini } 52ce0f1412SPaolo Bonzini } 53ce0f1412SPaolo Bonzini 5414e9559fSFam Zheng void bdrv_parent_drained_end(BlockDriverState *bs) 55ce0f1412SPaolo Bonzini { 56c2066af0SKevin Wolf BdrvChild *c; 5727ccdd52SKevin Wolf 58c2066af0SKevin Wolf QLIST_FOREACH(c, &bs->parents, next_parent) { 59c2066af0SKevin Wolf if (c->role->drained_end) { 60c2066af0SKevin Wolf c->role->drained_end(c); 6127ccdd52SKevin Wolf } 62c2066af0SKevin Wolf } 6361007b31SStefan Hajnoczi } 6461007b31SStefan Hajnoczi 65d9e0dfa2SEric Blake static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src) 66d9e0dfa2SEric Blake { 67d9e0dfa2SEric Blake dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer); 68d9e0dfa2SEric Blake dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer); 69d9e0dfa2SEric Blake dst->opt_mem_alignment = MAX(dst->opt_mem_alignment, 70d9e0dfa2SEric Blake src->opt_mem_alignment); 71d9e0dfa2SEric Blake dst->min_mem_alignment = MAX(dst->min_mem_alignment, 72d9e0dfa2SEric Blake src->min_mem_alignment); 73d9e0dfa2SEric Blake dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov); 74d9e0dfa2SEric Blake } 75d9e0dfa2SEric Blake 7661007b31SStefan Hajnoczi void bdrv_refresh_limits(BlockDriverState *bs, Error **errp) 7761007b31SStefan Hajnoczi { 7861007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 7961007b31SStefan Hajnoczi Error *local_err = NULL; 8061007b31SStefan Hajnoczi 8161007b31SStefan Hajnoczi memset(&bs->bl, 0, sizeof(bs->bl)); 8261007b31SStefan Hajnoczi 8361007b31SStefan Hajnoczi if (!drv) { 8461007b31SStefan Hajnoczi return; 8561007b31SStefan Hajnoczi } 8661007b31SStefan Hajnoczi 8779ba8c98SEric Blake /* Default alignment based on whether driver has byte interface */ 88a5b8dd2cSEric Blake bs->bl.request_alignment = drv->bdrv_co_preadv ? 1 : 512; 8979ba8c98SEric Blake 9061007b31SStefan Hajnoczi /* Take some limits from the children as a default */ 9161007b31SStefan Hajnoczi if (bs->file) { 929a4f4c31SKevin Wolf bdrv_refresh_limits(bs->file->bs, &local_err); 9361007b31SStefan Hajnoczi if (local_err) { 9461007b31SStefan Hajnoczi error_propagate(errp, local_err); 9561007b31SStefan Hajnoczi return; 9661007b31SStefan Hajnoczi } 97d9e0dfa2SEric Blake bdrv_merge_limits(&bs->bl, &bs->file->bs->bl); 9861007b31SStefan Hajnoczi } else { 994196d2f0SDenis V. Lunev bs->bl.min_mem_alignment = 512; 100459b4e66SDenis V. Lunev bs->bl.opt_mem_alignment = getpagesize(); 101bd44feb7SStefan Hajnoczi 102bd44feb7SStefan Hajnoczi /* Safe default since most protocols use readv()/writev()/etc */ 103bd44feb7SStefan Hajnoczi bs->bl.max_iov = IOV_MAX; 10461007b31SStefan Hajnoczi } 10561007b31SStefan Hajnoczi 106760e0063SKevin Wolf if (bs->backing) { 107760e0063SKevin Wolf bdrv_refresh_limits(bs->backing->bs, &local_err); 10861007b31SStefan Hajnoczi if (local_err) { 10961007b31SStefan Hajnoczi error_propagate(errp, local_err); 11061007b31SStefan Hajnoczi return; 11161007b31SStefan Hajnoczi } 112d9e0dfa2SEric Blake bdrv_merge_limits(&bs->bl, &bs->backing->bs->bl); 11361007b31SStefan Hajnoczi } 11461007b31SStefan Hajnoczi 11561007b31SStefan Hajnoczi /* Then let the driver override it */ 11661007b31SStefan Hajnoczi if (drv->bdrv_refresh_limits) { 11761007b31SStefan Hajnoczi drv->bdrv_refresh_limits(bs, errp); 11861007b31SStefan Hajnoczi } 11961007b31SStefan Hajnoczi } 12061007b31SStefan Hajnoczi 12161007b31SStefan Hajnoczi /** 12261007b31SStefan Hajnoczi * The copy-on-read flag is actually a reference count so multiple users may 12361007b31SStefan Hajnoczi * use the feature without worrying about clobbering its previous state. 12461007b31SStefan Hajnoczi * Copy-on-read stays enabled until all users have called to disable it. 12561007b31SStefan Hajnoczi */ 12661007b31SStefan Hajnoczi void bdrv_enable_copy_on_read(BlockDriverState *bs) 12761007b31SStefan Hajnoczi { 128d3faa13eSPaolo Bonzini atomic_inc(&bs->copy_on_read); 12961007b31SStefan Hajnoczi } 13061007b31SStefan Hajnoczi 13161007b31SStefan Hajnoczi void bdrv_disable_copy_on_read(BlockDriverState *bs) 13261007b31SStefan Hajnoczi { 133d3faa13eSPaolo Bonzini int old = atomic_fetch_dec(&bs->copy_on_read); 134d3faa13eSPaolo Bonzini assert(old >= 1); 13561007b31SStefan Hajnoczi } 13661007b31SStefan Hajnoczi 13761007b31SStefan Hajnoczi /* Check if any requests are in-flight (including throttled requests) */ 138439db28cSKevin Wolf bool bdrv_requests_pending(BlockDriverState *bs) 13961007b31SStefan Hajnoczi { 14037a639a7SKevin Wolf BdrvChild *child; 14137a639a7SKevin Wolf 14299723548SPaolo Bonzini if (atomic_read(&bs->in_flight)) { 14361007b31SStefan Hajnoczi return true; 14461007b31SStefan Hajnoczi } 14537a639a7SKevin Wolf 14637a639a7SKevin Wolf QLIST_FOREACH(child, &bs->children, next) { 14737a639a7SKevin Wolf if (bdrv_requests_pending(child->bs)) { 14861007b31SStefan Hajnoczi return true; 14961007b31SStefan Hajnoczi } 15061007b31SStefan Hajnoczi } 15137a639a7SKevin Wolf 15261007b31SStefan Hajnoczi return false; 15361007b31SStefan Hajnoczi } 15461007b31SStefan Hajnoczi 15561124f03SPaolo Bonzini typedef struct { 15661124f03SPaolo Bonzini Coroutine *co; 15761124f03SPaolo Bonzini BlockDriverState *bs; 15861124f03SPaolo Bonzini bool done; 159481cad48SManos Pitsidianakis bool begin; 16061124f03SPaolo Bonzini } BdrvCoDrainData; 16161124f03SPaolo Bonzini 16261124f03SPaolo Bonzini static void coroutine_fn bdrv_drain_invoke_entry(void *opaque) 16361124f03SPaolo Bonzini { 16461124f03SPaolo Bonzini BdrvCoDrainData *data = opaque; 16561124f03SPaolo Bonzini BlockDriverState *bs = data->bs; 16661124f03SPaolo Bonzini 167481cad48SManos Pitsidianakis if (data->begin) { 168f8ea8dacSManos Pitsidianakis bs->drv->bdrv_co_drain_begin(bs); 169481cad48SManos Pitsidianakis } else { 170481cad48SManos Pitsidianakis bs->drv->bdrv_co_drain_end(bs); 171481cad48SManos Pitsidianakis } 17261124f03SPaolo Bonzini 17361124f03SPaolo Bonzini /* Set data->done before reading bs->wakeup. */ 17461124f03SPaolo Bonzini atomic_mb_set(&data->done, true); 17561124f03SPaolo Bonzini bdrv_wakeup(bs); 17661124f03SPaolo Bonzini } 17761124f03SPaolo Bonzini 178481cad48SManos Pitsidianakis static void bdrv_drain_invoke(BlockDriverState *bs, bool begin) 17961124f03SPaolo Bonzini { 180481cad48SManos Pitsidianakis BdrvCoDrainData data = { .bs = bs, .done = false, .begin = begin}; 18161124f03SPaolo Bonzini 182f8ea8dacSManos Pitsidianakis if (!bs->drv || (begin && !bs->drv->bdrv_co_drain_begin) || 183481cad48SManos Pitsidianakis (!begin && !bs->drv->bdrv_co_drain_end)) { 18461124f03SPaolo Bonzini return; 18561124f03SPaolo Bonzini } 18661124f03SPaolo Bonzini 18761124f03SPaolo Bonzini data.co = qemu_coroutine_create(bdrv_drain_invoke_entry, &data); 18861124f03SPaolo Bonzini bdrv_coroutine_enter(bs, data.co); 18961124f03SPaolo Bonzini BDRV_POLL_WHILE(bs, !data.done); 19061124f03SPaolo Bonzini } 19161124f03SPaolo Bonzini 192481cad48SManos Pitsidianakis static bool bdrv_drain_recurse(BlockDriverState *bs, bool begin) 19367da1dc5SFam Zheng { 194178bd438SFam Zheng BdrvChild *child, *tmp; 195d42cf288SPaolo Bonzini bool waited; 196d42cf288SPaolo Bonzini 19761124f03SPaolo Bonzini /* Ensure any pending metadata writes are submitted to bs->file. */ 198481cad48SManos Pitsidianakis bdrv_drain_invoke(bs, begin); 199481cad48SManos Pitsidianakis 200481cad48SManos Pitsidianakis /* Wait for drained requests to finish */ 201481cad48SManos Pitsidianakis waited = BDRV_POLL_WHILE(bs, atomic_read(&bs->in_flight) > 0); 202d42cf288SPaolo Bonzini 203178bd438SFam Zheng QLIST_FOREACH_SAFE(child, &bs->children, next, tmp) { 204178bd438SFam Zheng BlockDriverState *bs = child->bs; 205178bd438SFam Zheng bool in_main_loop = 206178bd438SFam Zheng qemu_get_current_aio_context() == qemu_get_aio_context(); 207178bd438SFam Zheng assert(bs->refcnt > 0); 208178bd438SFam Zheng if (in_main_loop) { 209178bd438SFam Zheng /* In case the recursive bdrv_drain_recurse processes a 210178bd438SFam Zheng * block_job_defer_to_main_loop BH and modifies the graph, 211178bd438SFam Zheng * let's hold a reference to bs until we are done. 212178bd438SFam Zheng * 213178bd438SFam Zheng * IOThread doesn't have such a BH, and it is not safe to call 214178bd438SFam Zheng * bdrv_unref without BQL, so skip doing it there. 215178bd438SFam Zheng */ 216178bd438SFam Zheng bdrv_ref(bs); 217178bd438SFam Zheng } 218481cad48SManos Pitsidianakis waited |= bdrv_drain_recurse(bs, begin); 219178bd438SFam Zheng if (in_main_loop) { 220178bd438SFam Zheng bdrv_unref(bs); 221178bd438SFam Zheng } 22267da1dc5SFam Zheng } 223d42cf288SPaolo Bonzini 224d42cf288SPaolo Bonzini return waited; 22567da1dc5SFam Zheng } 22667da1dc5SFam Zheng 227a77fd4bbSFam Zheng static void bdrv_co_drain_bh_cb(void *opaque) 228a77fd4bbSFam Zheng { 229a77fd4bbSFam Zheng BdrvCoDrainData *data = opaque; 230a77fd4bbSFam Zheng Coroutine *co = data->co; 23199723548SPaolo Bonzini BlockDriverState *bs = data->bs; 232a77fd4bbSFam Zheng 23399723548SPaolo Bonzini bdrv_dec_in_flight(bs); 234481cad48SManos Pitsidianakis if (data->begin) { 235d42cf288SPaolo Bonzini bdrv_drained_begin(bs); 236481cad48SManos Pitsidianakis } else { 237481cad48SManos Pitsidianakis bdrv_drained_end(bs); 238481cad48SManos Pitsidianakis } 239481cad48SManos Pitsidianakis 240a77fd4bbSFam Zheng data->done = true; 2411919631eSPaolo Bonzini aio_co_wake(co); 242a77fd4bbSFam Zheng } 243a77fd4bbSFam Zheng 244481cad48SManos Pitsidianakis static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs, 245481cad48SManos Pitsidianakis bool begin) 246a77fd4bbSFam Zheng { 247a77fd4bbSFam Zheng BdrvCoDrainData data; 248a77fd4bbSFam Zheng 249a77fd4bbSFam Zheng /* Calling bdrv_drain() from a BH ensures the current coroutine yields and 250a77fd4bbSFam Zheng * other coroutines run if they were queued from 251a77fd4bbSFam Zheng * qemu_co_queue_run_restart(). */ 252a77fd4bbSFam Zheng 253a77fd4bbSFam Zheng assert(qemu_in_coroutine()); 254a77fd4bbSFam Zheng data = (BdrvCoDrainData) { 255a77fd4bbSFam Zheng .co = qemu_coroutine_self(), 256a77fd4bbSFam Zheng .bs = bs, 257a77fd4bbSFam Zheng .done = false, 258481cad48SManos Pitsidianakis .begin = begin, 259a77fd4bbSFam Zheng }; 26099723548SPaolo Bonzini bdrv_inc_in_flight(bs); 261fffb6e12SPaolo Bonzini aio_bh_schedule_oneshot(bdrv_get_aio_context(bs), 262fffb6e12SPaolo Bonzini bdrv_co_drain_bh_cb, &data); 263a77fd4bbSFam Zheng 264a77fd4bbSFam Zheng qemu_coroutine_yield(); 265a77fd4bbSFam Zheng /* If we are resumed from some other event (such as an aio completion or a 266a77fd4bbSFam Zheng * timer callback), it is a bug in the caller that should be fixed. */ 267a77fd4bbSFam Zheng assert(data.done); 268a77fd4bbSFam Zheng } 269a77fd4bbSFam Zheng 2706820643fSKevin Wolf void bdrv_drained_begin(BlockDriverState *bs) 2716820643fSKevin Wolf { 272d42cf288SPaolo Bonzini if (qemu_in_coroutine()) { 273481cad48SManos Pitsidianakis bdrv_co_yield_to_drain(bs, true); 274d42cf288SPaolo Bonzini return; 275d42cf288SPaolo Bonzini } 276d42cf288SPaolo Bonzini 277414c2ec3SPaolo Bonzini if (atomic_fetch_inc(&bs->quiesce_counter) == 0) { 2786820643fSKevin Wolf aio_disable_external(bdrv_get_aio_context(bs)); 2796820643fSKevin Wolf bdrv_parent_drained_begin(bs); 2806820643fSKevin Wolf } 2816820643fSKevin Wolf 282481cad48SManos Pitsidianakis bdrv_drain_recurse(bs, true); 2836820643fSKevin Wolf } 2846820643fSKevin Wolf 2856820643fSKevin Wolf void bdrv_drained_end(BlockDriverState *bs) 2866820643fSKevin Wolf { 287481cad48SManos Pitsidianakis if (qemu_in_coroutine()) { 288481cad48SManos Pitsidianakis bdrv_co_yield_to_drain(bs, false); 289481cad48SManos Pitsidianakis return; 290481cad48SManos Pitsidianakis } 2916820643fSKevin Wolf assert(bs->quiesce_counter > 0); 292414c2ec3SPaolo Bonzini if (atomic_fetch_dec(&bs->quiesce_counter) > 1) { 2936820643fSKevin Wolf return; 2946820643fSKevin Wolf } 2956820643fSKevin Wolf 2966820643fSKevin Wolf bdrv_parent_drained_end(bs); 297481cad48SManos Pitsidianakis bdrv_drain_recurse(bs, false); 2986820643fSKevin Wolf aio_enable_external(bdrv_get_aio_context(bs)); 2996820643fSKevin Wolf } 3006820643fSKevin Wolf 30161007b31SStefan Hajnoczi /* 30267da1dc5SFam Zheng * Wait for pending requests to complete on a single BlockDriverState subtree, 30367da1dc5SFam Zheng * and suspend block driver's internal I/O until next request arrives. 30461007b31SStefan Hajnoczi * 30561007b31SStefan Hajnoczi * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState 30661007b31SStefan Hajnoczi * AioContext. 3077a63f3cdSStefan Hajnoczi * 3087a63f3cdSStefan Hajnoczi * Only this BlockDriverState's AioContext is run, so in-flight requests must 3097a63f3cdSStefan Hajnoczi * not depend on events in other AioContexts. In that case, use 3107a63f3cdSStefan Hajnoczi * bdrv_drain_all() instead. 31161007b31SStefan Hajnoczi */ 312b6e84c97SPaolo Bonzini void coroutine_fn bdrv_co_drain(BlockDriverState *bs) 313b6e84c97SPaolo Bonzini { 3146820643fSKevin Wolf assert(qemu_in_coroutine()); 3156820643fSKevin Wolf bdrv_drained_begin(bs); 3166820643fSKevin Wolf bdrv_drained_end(bs); 317b6e84c97SPaolo Bonzini } 318b6e84c97SPaolo Bonzini 31961007b31SStefan Hajnoczi void bdrv_drain(BlockDriverState *bs) 32061007b31SStefan Hajnoczi { 3216820643fSKevin Wolf bdrv_drained_begin(bs); 3226820643fSKevin Wolf bdrv_drained_end(bs); 32361007b31SStefan Hajnoczi } 32461007b31SStefan Hajnoczi 32561007b31SStefan Hajnoczi /* 32661007b31SStefan Hajnoczi * Wait for pending requests to complete across all BlockDriverStates 32761007b31SStefan Hajnoczi * 32861007b31SStefan Hajnoczi * This function does not flush data to disk, use bdrv_flush_all() for that 32961007b31SStefan Hajnoczi * after calling this function. 330c0778f66SAlberto Garcia * 331c0778f66SAlberto Garcia * This pauses all block jobs and disables external clients. It must 332c0778f66SAlberto Garcia * be paired with bdrv_drain_all_end(). 333c0778f66SAlberto Garcia * 334c0778f66SAlberto Garcia * NOTE: no new block jobs or BlockDriverStates can be created between 335c0778f66SAlberto Garcia * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls. 33661007b31SStefan Hajnoczi */ 337c0778f66SAlberto Garcia void bdrv_drain_all_begin(void) 33861007b31SStefan Hajnoczi { 33961007b31SStefan Hajnoczi /* Always run first iteration so any pending completion BHs run */ 34099723548SPaolo Bonzini bool waited = true; 3417c8eece4SKevin Wolf BlockDriverState *bs; 34288be7b4bSKevin Wolf BdrvNextIterator it; 343f406c03cSAlexander Yarygin GSList *aio_ctxs = NULL, *ctx; 34461007b31SStefan Hajnoczi 345f321dcb5SPaolo Bonzini block_job_pause_all(); 346eb1364ceSAlberto Garcia 34788be7b4bSKevin Wolf for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 34861007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 34961007b31SStefan Hajnoczi 35061007b31SStefan Hajnoczi aio_context_acquire(aio_context); 351c2066af0SKevin Wolf bdrv_parent_drained_begin(bs); 352c0778f66SAlberto Garcia aio_disable_external(aio_context); 35361007b31SStefan Hajnoczi aio_context_release(aio_context); 354f406c03cSAlexander Yarygin 355764ba3aeSAlberto Garcia if (!g_slist_find(aio_ctxs, aio_context)) { 356f406c03cSAlexander Yarygin aio_ctxs = g_slist_prepend(aio_ctxs, aio_context); 357f406c03cSAlexander Yarygin } 35861007b31SStefan Hajnoczi } 35961007b31SStefan Hajnoczi 3607a63f3cdSStefan Hajnoczi /* Note that completion of an asynchronous I/O operation can trigger any 3617a63f3cdSStefan Hajnoczi * number of other I/O operations on other devices---for example a 3627a63f3cdSStefan Hajnoczi * coroutine can submit an I/O request to another device in response to 3637a63f3cdSStefan Hajnoczi * request completion. Therefore we must keep looping until there was no 3647a63f3cdSStefan Hajnoczi * more activity rather than simply draining each device independently. 3657a63f3cdSStefan Hajnoczi */ 36699723548SPaolo Bonzini while (waited) { 36799723548SPaolo Bonzini waited = false; 368f406c03cSAlexander Yarygin 369f406c03cSAlexander Yarygin for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) { 370f406c03cSAlexander Yarygin AioContext *aio_context = ctx->data; 37161007b31SStefan Hajnoczi 37261007b31SStefan Hajnoczi aio_context_acquire(aio_context); 37388be7b4bSKevin Wolf for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 374f406c03cSAlexander Yarygin if (aio_context == bdrv_get_aio_context(bs)) { 375481cad48SManos Pitsidianakis waited |= bdrv_drain_recurse(bs, true); 376f406c03cSAlexander Yarygin } 377f406c03cSAlexander Yarygin } 37861007b31SStefan Hajnoczi aio_context_release(aio_context); 37961007b31SStefan Hajnoczi } 38061007b31SStefan Hajnoczi } 38161007b31SStefan Hajnoczi 382c0778f66SAlberto Garcia g_slist_free(aio_ctxs); 383c0778f66SAlberto Garcia } 384c0778f66SAlberto Garcia 385c0778f66SAlberto Garcia void bdrv_drain_all_end(void) 386c0778f66SAlberto Garcia { 387c0778f66SAlberto Garcia BlockDriverState *bs; 388c0778f66SAlberto Garcia BdrvNextIterator it; 389c0778f66SAlberto Garcia 39088be7b4bSKevin Wolf for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 39161007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 39261007b31SStefan Hajnoczi 39361007b31SStefan Hajnoczi aio_context_acquire(aio_context); 394c0778f66SAlberto Garcia aio_enable_external(aio_context); 395c2066af0SKevin Wolf bdrv_parent_drained_end(bs); 396481cad48SManos Pitsidianakis bdrv_drain_recurse(bs, false); 39761007b31SStefan Hajnoczi aio_context_release(aio_context); 39861007b31SStefan Hajnoczi } 399eb1364ceSAlberto Garcia 400f321dcb5SPaolo Bonzini block_job_resume_all(); 40161007b31SStefan Hajnoczi } 40261007b31SStefan Hajnoczi 403c0778f66SAlberto Garcia void bdrv_drain_all(void) 404c0778f66SAlberto Garcia { 405c0778f66SAlberto Garcia bdrv_drain_all_begin(); 406c0778f66SAlberto Garcia bdrv_drain_all_end(); 407c0778f66SAlberto Garcia } 408c0778f66SAlberto Garcia 40961007b31SStefan Hajnoczi /** 41061007b31SStefan Hajnoczi * Remove an active request from the tracked requests list 41161007b31SStefan Hajnoczi * 41261007b31SStefan Hajnoczi * This function should be called when a tracked request is completing. 41361007b31SStefan Hajnoczi */ 41461007b31SStefan Hajnoczi static void tracked_request_end(BdrvTrackedRequest *req) 41561007b31SStefan Hajnoczi { 41661007b31SStefan Hajnoczi if (req->serialising) { 41720fc71b2SPaolo Bonzini atomic_dec(&req->bs->serialising_in_flight); 41861007b31SStefan Hajnoczi } 41961007b31SStefan Hajnoczi 4203783fa3dSPaolo Bonzini qemu_co_mutex_lock(&req->bs->reqs_lock); 42161007b31SStefan Hajnoczi QLIST_REMOVE(req, list); 42261007b31SStefan Hajnoczi qemu_co_queue_restart_all(&req->wait_queue); 4233783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&req->bs->reqs_lock); 42461007b31SStefan Hajnoczi } 42561007b31SStefan Hajnoczi 42661007b31SStefan Hajnoczi /** 42761007b31SStefan Hajnoczi * Add an active request to the tracked requests list 42861007b31SStefan Hajnoczi */ 42961007b31SStefan Hajnoczi static void tracked_request_begin(BdrvTrackedRequest *req, 43061007b31SStefan Hajnoczi BlockDriverState *bs, 43161007b31SStefan Hajnoczi int64_t offset, 432ebde595cSFam Zheng unsigned int bytes, 433ebde595cSFam Zheng enum BdrvTrackedRequestType type) 43461007b31SStefan Hajnoczi { 43561007b31SStefan Hajnoczi *req = (BdrvTrackedRequest){ 43661007b31SStefan Hajnoczi .bs = bs, 43761007b31SStefan Hajnoczi .offset = offset, 43861007b31SStefan Hajnoczi .bytes = bytes, 439ebde595cSFam Zheng .type = type, 44061007b31SStefan Hajnoczi .co = qemu_coroutine_self(), 44161007b31SStefan Hajnoczi .serialising = false, 44261007b31SStefan Hajnoczi .overlap_offset = offset, 44361007b31SStefan Hajnoczi .overlap_bytes = bytes, 44461007b31SStefan Hajnoczi }; 44561007b31SStefan Hajnoczi 44661007b31SStefan Hajnoczi qemu_co_queue_init(&req->wait_queue); 44761007b31SStefan Hajnoczi 4483783fa3dSPaolo Bonzini qemu_co_mutex_lock(&bs->reqs_lock); 44961007b31SStefan Hajnoczi QLIST_INSERT_HEAD(&bs->tracked_requests, req, list); 4503783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&bs->reqs_lock); 45161007b31SStefan Hajnoczi } 45261007b31SStefan Hajnoczi 45361007b31SStefan Hajnoczi static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align) 45461007b31SStefan Hajnoczi { 45561007b31SStefan Hajnoczi int64_t overlap_offset = req->offset & ~(align - 1); 45661007b31SStefan Hajnoczi unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align) 45761007b31SStefan Hajnoczi - overlap_offset; 45861007b31SStefan Hajnoczi 45961007b31SStefan Hajnoczi if (!req->serialising) { 46020fc71b2SPaolo Bonzini atomic_inc(&req->bs->serialising_in_flight); 46161007b31SStefan Hajnoczi req->serialising = true; 46261007b31SStefan Hajnoczi } 46361007b31SStefan Hajnoczi 46461007b31SStefan Hajnoczi req->overlap_offset = MIN(req->overlap_offset, overlap_offset); 46561007b31SStefan Hajnoczi req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes); 46661007b31SStefan Hajnoczi } 46761007b31SStefan Hajnoczi 46861007b31SStefan Hajnoczi /** 469244483e6SKevin Wolf * Round a region to cluster boundaries 470244483e6SKevin Wolf */ 471244483e6SKevin Wolf void bdrv_round_to_clusters(BlockDriverState *bs, 4727cfd5275SEric Blake int64_t offset, int64_t bytes, 473244483e6SKevin Wolf int64_t *cluster_offset, 4747cfd5275SEric Blake int64_t *cluster_bytes) 475244483e6SKevin Wolf { 476244483e6SKevin Wolf BlockDriverInfo bdi; 477244483e6SKevin Wolf 478244483e6SKevin Wolf if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 479244483e6SKevin Wolf *cluster_offset = offset; 480244483e6SKevin Wolf *cluster_bytes = bytes; 481244483e6SKevin Wolf } else { 482244483e6SKevin Wolf int64_t c = bdi.cluster_size; 483244483e6SKevin Wolf *cluster_offset = QEMU_ALIGN_DOWN(offset, c); 484244483e6SKevin Wolf *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c); 485244483e6SKevin Wolf } 486244483e6SKevin Wolf } 487244483e6SKevin Wolf 48861007b31SStefan Hajnoczi static int bdrv_get_cluster_size(BlockDriverState *bs) 48961007b31SStefan Hajnoczi { 49061007b31SStefan Hajnoczi BlockDriverInfo bdi; 49161007b31SStefan Hajnoczi int ret; 49261007b31SStefan Hajnoczi 49361007b31SStefan Hajnoczi ret = bdrv_get_info(bs, &bdi); 49461007b31SStefan Hajnoczi if (ret < 0 || bdi.cluster_size == 0) { 495a5b8dd2cSEric Blake return bs->bl.request_alignment; 49661007b31SStefan Hajnoczi } else { 49761007b31SStefan Hajnoczi return bdi.cluster_size; 49861007b31SStefan Hajnoczi } 49961007b31SStefan Hajnoczi } 50061007b31SStefan Hajnoczi 50161007b31SStefan Hajnoczi static bool tracked_request_overlaps(BdrvTrackedRequest *req, 50261007b31SStefan Hajnoczi int64_t offset, unsigned int bytes) 50361007b31SStefan Hajnoczi { 50461007b31SStefan Hajnoczi /* aaaa bbbb */ 50561007b31SStefan Hajnoczi if (offset >= req->overlap_offset + req->overlap_bytes) { 50661007b31SStefan Hajnoczi return false; 50761007b31SStefan Hajnoczi } 50861007b31SStefan Hajnoczi /* bbbb aaaa */ 50961007b31SStefan Hajnoczi if (req->overlap_offset >= offset + bytes) { 51061007b31SStefan Hajnoczi return false; 51161007b31SStefan Hajnoczi } 51261007b31SStefan Hajnoczi return true; 51361007b31SStefan Hajnoczi } 51461007b31SStefan Hajnoczi 51599723548SPaolo Bonzini void bdrv_inc_in_flight(BlockDriverState *bs) 51699723548SPaolo Bonzini { 51799723548SPaolo Bonzini atomic_inc(&bs->in_flight); 51899723548SPaolo Bonzini } 51999723548SPaolo Bonzini 520c9d1a561SPaolo Bonzini static void dummy_bh_cb(void *opaque) 521c9d1a561SPaolo Bonzini { 522c9d1a561SPaolo Bonzini } 523c9d1a561SPaolo Bonzini 524c9d1a561SPaolo Bonzini void bdrv_wakeup(BlockDriverState *bs) 525c9d1a561SPaolo Bonzini { 526e2a6ae7fSPaolo Bonzini /* The barrier (or an atomic op) is in the caller. */ 527e2a6ae7fSPaolo Bonzini if (atomic_read(&bs->wakeup)) { 528c9d1a561SPaolo Bonzini aio_bh_schedule_oneshot(qemu_get_aio_context(), dummy_bh_cb, NULL); 529c9d1a561SPaolo Bonzini } 530c9d1a561SPaolo Bonzini } 531c9d1a561SPaolo Bonzini 53299723548SPaolo Bonzini void bdrv_dec_in_flight(BlockDriverState *bs) 53399723548SPaolo Bonzini { 53499723548SPaolo Bonzini atomic_dec(&bs->in_flight); 535c9d1a561SPaolo Bonzini bdrv_wakeup(bs); 53699723548SPaolo Bonzini } 53799723548SPaolo Bonzini 53861007b31SStefan Hajnoczi static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self) 53961007b31SStefan Hajnoczi { 54061007b31SStefan Hajnoczi BlockDriverState *bs = self->bs; 54161007b31SStefan Hajnoczi BdrvTrackedRequest *req; 54261007b31SStefan Hajnoczi bool retry; 54361007b31SStefan Hajnoczi bool waited = false; 54461007b31SStefan Hajnoczi 54520fc71b2SPaolo Bonzini if (!atomic_read(&bs->serialising_in_flight)) { 54661007b31SStefan Hajnoczi return false; 54761007b31SStefan Hajnoczi } 54861007b31SStefan Hajnoczi 54961007b31SStefan Hajnoczi do { 55061007b31SStefan Hajnoczi retry = false; 5513783fa3dSPaolo Bonzini qemu_co_mutex_lock(&bs->reqs_lock); 55261007b31SStefan Hajnoczi QLIST_FOREACH(req, &bs->tracked_requests, list) { 55361007b31SStefan Hajnoczi if (req == self || (!req->serialising && !self->serialising)) { 55461007b31SStefan Hajnoczi continue; 55561007b31SStefan Hajnoczi } 55661007b31SStefan Hajnoczi if (tracked_request_overlaps(req, self->overlap_offset, 55761007b31SStefan Hajnoczi self->overlap_bytes)) 55861007b31SStefan Hajnoczi { 55961007b31SStefan Hajnoczi /* Hitting this means there was a reentrant request, for 56061007b31SStefan Hajnoczi * example, a block driver issuing nested requests. This must 56161007b31SStefan Hajnoczi * never happen since it means deadlock. 56261007b31SStefan Hajnoczi */ 56361007b31SStefan Hajnoczi assert(qemu_coroutine_self() != req->co); 56461007b31SStefan Hajnoczi 56561007b31SStefan Hajnoczi /* If the request is already (indirectly) waiting for us, or 56661007b31SStefan Hajnoczi * will wait for us as soon as it wakes up, then just go on 56761007b31SStefan Hajnoczi * (instead of producing a deadlock in the former case). */ 56861007b31SStefan Hajnoczi if (!req->waiting_for) { 56961007b31SStefan Hajnoczi self->waiting_for = req; 5703783fa3dSPaolo Bonzini qemu_co_queue_wait(&req->wait_queue, &bs->reqs_lock); 57161007b31SStefan Hajnoczi self->waiting_for = NULL; 57261007b31SStefan Hajnoczi retry = true; 57361007b31SStefan Hajnoczi waited = true; 57461007b31SStefan Hajnoczi break; 57561007b31SStefan Hajnoczi } 57661007b31SStefan Hajnoczi } 57761007b31SStefan Hajnoczi } 5783783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&bs->reqs_lock); 57961007b31SStefan Hajnoczi } while (retry); 58061007b31SStefan Hajnoczi 58161007b31SStefan Hajnoczi return waited; 58261007b31SStefan Hajnoczi } 58361007b31SStefan Hajnoczi 58461007b31SStefan Hajnoczi static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset, 58561007b31SStefan Hajnoczi size_t size) 58661007b31SStefan Hajnoczi { 58761007b31SStefan Hajnoczi if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) { 58861007b31SStefan Hajnoczi return -EIO; 58961007b31SStefan Hajnoczi } 59061007b31SStefan Hajnoczi 59161007b31SStefan Hajnoczi if (!bdrv_is_inserted(bs)) { 59261007b31SStefan Hajnoczi return -ENOMEDIUM; 59361007b31SStefan Hajnoczi } 59461007b31SStefan Hajnoczi 59561007b31SStefan Hajnoczi if (offset < 0) { 59661007b31SStefan Hajnoczi return -EIO; 59761007b31SStefan Hajnoczi } 59861007b31SStefan Hajnoczi 59961007b31SStefan Hajnoczi return 0; 60061007b31SStefan Hajnoczi } 60161007b31SStefan Hajnoczi 60261007b31SStefan Hajnoczi typedef struct RwCo { 603e293b7a3SKevin Wolf BdrvChild *child; 60461007b31SStefan Hajnoczi int64_t offset; 60561007b31SStefan Hajnoczi QEMUIOVector *qiov; 60661007b31SStefan Hajnoczi bool is_write; 60761007b31SStefan Hajnoczi int ret; 60861007b31SStefan Hajnoczi BdrvRequestFlags flags; 60961007b31SStefan Hajnoczi } RwCo; 61061007b31SStefan Hajnoczi 61161007b31SStefan Hajnoczi static void coroutine_fn bdrv_rw_co_entry(void *opaque) 61261007b31SStefan Hajnoczi { 61361007b31SStefan Hajnoczi RwCo *rwco = opaque; 61461007b31SStefan Hajnoczi 61561007b31SStefan Hajnoczi if (!rwco->is_write) { 616a03ef88fSKevin Wolf rwco->ret = bdrv_co_preadv(rwco->child, rwco->offset, 61761007b31SStefan Hajnoczi rwco->qiov->size, rwco->qiov, 61861007b31SStefan Hajnoczi rwco->flags); 61961007b31SStefan Hajnoczi } else { 620a03ef88fSKevin Wolf rwco->ret = bdrv_co_pwritev(rwco->child, rwco->offset, 62161007b31SStefan Hajnoczi rwco->qiov->size, rwco->qiov, 62261007b31SStefan Hajnoczi rwco->flags); 62361007b31SStefan Hajnoczi } 62461007b31SStefan Hajnoczi } 62561007b31SStefan Hajnoczi 62661007b31SStefan Hajnoczi /* 62761007b31SStefan Hajnoczi * Process a vectored synchronous request using coroutines 62861007b31SStefan Hajnoczi */ 629e293b7a3SKevin Wolf static int bdrv_prwv_co(BdrvChild *child, int64_t offset, 63061007b31SStefan Hajnoczi QEMUIOVector *qiov, bool is_write, 63161007b31SStefan Hajnoczi BdrvRequestFlags flags) 63261007b31SStefan Hajnoczi { 63361007b31SStefan Hajnoczi Coroutine *co; 63461007b31SStefan Hajnoczi RwCo rwco = { 635e293b7a3SKevin Wolf .child = child, 63661007b31SStefan Hajnoczi .offset = offset, 63761007b31SStefan Hajnoczi .qiov = qiov, 63861007b31SStefan Hajnoczi .is_write = is_write, 63961007b31SStefan Hajnoczi .ret = NOT_DONE, 64061007b31SStefan Hajnoczi .flags = flags, 64161007b31SStefan Hajnoczi }; 64261007b31SStefan Hajnoczi 64361007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 64461007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 64561007b31SStefan Hajnoczi bdrv_rw_co_entry(&rwco); 64661007b31SStefan Hajnoczi } else { 6470b8b8753SPaolo Bonzini co = qemu_coroutine_create(bdrv_rw_co_entry, &rwco); 648e92f0e19SFam Zheng bdrv_coroutine_enter(child->bs, co); 64988b062c2SPaolo Bonzini BDRV_POLL_WHILE(child->bs, rwco.ret == NOT_DONE); 65061007b31SStefan Hajnoczi } 65161007b31SStefan Hajnoczi return rwco.ret; 65261007b31SStefan Hajnoczi } 65361007b31SStefan Hajnoczi 65461007b31SStefan Hajnoczi /* 65561007b31SStefan Hajnoczi * Process a synchronous request using coroutines 65661007b31SStefan Hajnoczi */ 657e293b7a3SKevin Wolf static int bdrv_rw_co(BdrvChild *child, int64_t sector_num, uint8_t *buf, 65861007b31SStefan Hajnoczi int nb_sectors, bool is_write, BdrvRequestFlags flags) 65961007b31SStefan Hajnoczi { 66061007b31SStefan Hajnoczi QEMUIOVector qiov; 66161007b31SStefan Hajnoczi struct iovec iov = { 66261007b31SStefan Hajnoczi .iov_base = (void *)buf, 66361007b31SStefan Hajnoczi .iov_len = nb_sectors * BDRV_SECTOR_SIZE, 66461007b31SStefan Hajnoczi }; 66561007b31SStefan Hajnoczi 66661007b31SStefan Hajnoczi if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 66761007b31SStefan Hajnoczi return -EINVAL; 66861007b31SStefan Hajnoczi } 66961007b31SStefan Hajnoczi 67061007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 671e293b7a3SKevin Wolf return bdrv_prwv_co(child, sector_num << BDRV_SECTOR_BITS, 67261007b31SStefan Hajnoczi &qiov, is_write, flags); 67361007b31SStefan Hajnoczi } 67461007b31SStefan Hajnoczi 67561007b31SStefan Hajnoczi /* return < 0 if error. See bdrv_write() for the return codes */ 676fbcbbf4eSKevin Wolf int bdrv_read(BdrvChild *child, int64_t sector_num, 67761007b31SStefan Hajnoczi uint8_t *buf, int nb_sectors) 67861007b31SStefan Hajnoczi { 679e293b7a3SKevin Wolf return bdrv_rw_co(child, sector_num, buf, nb_sectors, false, 0); 68061007b31SStefan Hajnoczi } 68161007b31SStefan Hajnoczi 68261007b31SStefan Hajnoczi /* Return < 0 if error. Important errors are: 68361007b31SStefan Hajnoczi -EIO generic I/O error (may happen for all errors) 68461007b31SStefan Hajnoczi -ENOMEDIUM No media inserted. 68561007b31SStefan Hajnoczi -EINVAL Invalid sector number or nb_sectors 68661007b31SStefan Hajnoczi -EACCES Trying to write a read-only device 68761007b31SStefan Hajnoczi */ 68818d51c4bSKevin Wolf int bdrv_write(BdrvChild *child, int64_t sector_num, 68961007b31SStefan Hajnoczi const uint8_t *buf, int nb_sectors) 69061007b31SStefan Hajnoczi { 691e293b7a3SKevin Wolf return bdrv_rw_co(child, sector_num, (uint8_t *)buf, nb_sectors, true, 0); 69261007b31SStefan Hajnoczi } 69361007b31SStefan Hajnoczi 694720ff280SKevin Wolf int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset, 695f5a5ca79SManos Pitsidianakis int bytes, BdrvRequestFlags flags) 69661007b31SStefan Hajnoczi { 69774021bc4SEric Blake QEMUIOVector qiov; 69874021bc4SEric Blake struct iovec iov = { 69974021bc4SEric Blake .iov_base = NULL, 700f5a5ca79SManos Pitsidianakis .iov_len = bytes, 70174021bc4SEric Blake }; 70274021bc4SEric Blake 70374021bc4SEric Blake qemu_iovec_init_external(&qiov, &iov, 1); 704e293b7a3SKevin Wolf return bdrv_prwv_co(child, offset, &qiov, true, 70561007b31SStefan Hajnoczi BDRV_REQ_ZERO_WRITE | flags); 70661007b31SStefan Hajnoczi } 70761007b31SStefan Hajnoczi 70861007b31SStefan Hajnoczi /* 70974021bc4SEric Blake * Completely zero out a block device with the help of bdrv_pwrite_zeroes. 71061007b31SStefan Hajnoczi * The operation is sped up by checking the block status and only writing 71161007b31SStefan Hajnoczi * zeroes to the device if they currently do not return zeroes. Optional 71274021bc4SEric Blake * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP, 713465fe887SEric Blake * BDRV_REQ_FUA). 71461007b31SStefan Hajnoczi * 71561007b31SStefan Hajnoczi * Returns < 0 on error, 0 on success. For error codes see bdrv_write(). 71661007b31SStefan Hajnoczi */ 717720ff280SKevin Wolf int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags) 71861007b31SStefan Hajnoczi { 719*237d78f8SEric Blake int ret; 720*237d78f8SEric Blake int64_t target_size, bytes, offset = 0; 721720ff280SKevin Wolf BlockDriverState *bs = child->bs; 72261007b31SStefan Hajnoczi 7237286d610SEric Blake target_size = bdrv_getlength(bs); 7247286d610SEric Blake if (target_size < 0) { 7257286d610SEric Blake return target_size; 72661007b31SStefan Hajnoczi } 72761007b31SStefan Hajnoczi 72861007b31SStefan Hajnoczi for (;;) { 7297286d610SEric Blake bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES); 7307286d610SEric Blake if (bytes <= 0) { 73161007b31SStefan Hajnoczi return 0; 73261007b31SStefan Hajnoczi } 733*237d78f8SEric Blake ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL); 73461007b31SStefan Hajnoczi if (ret < 0) { 7357286d610SEric Blake error_report("error getting block status at offset %" PRId64 ": %s", 7367286d610SEric Blake offset, strerror(-ret)); 73761007b31SStefan Hajnoczi return ret; 73861007b31SStefan Hajnoczi } 73961007b31SStefan Hajnoczi if (ret & BDRV_BLOCK_ZERO) { 740*237d78f8SEric Blake offset += bytes; 74161007b31SStefan Hajnoczi continue; 74261007b31SStefan Hajnoczi } 743*237d78f8SEric Blake ret = bdrv_pwrite_zeroes(child, offset, bytes, flags); 74461007b31SStefan Hajnoczi if (ret < 0) { 7457286d610SEric Blake error_report("error writing zeroes at offset %" PRId64 ": %s", 7467286d610SEric Blake offset, strerror(-ret)); 74761007b31SStefan Hajnoczi return ret; 74861007b31SStefan Hajnoczi } 749*237d78f8SEric Blake offset += bytes; 75061007b31SStefan Hajnoczi } 75161007b31SStefan Hajnoczi } 75261007b31SStefan Hajnoczi 753cf2ab8fcSKevin Wolf int bdrv_preadv(BdrvChild *child, int64_t offset, QEMUIOVector *qiov) 754f1e84741SKevin Wolf { 755f1e84741SKevin Wolf int ret; 756f1e84741SKevin Wolf 757e293b7a3SKevin Wolf ret = bdrv_prwv_co(child, offset, qiov, false, 0); 758f1e84741SKevin Wolf if (ret < 0) { 759f1e84741SKevin Wolf return ret; 760f1e84741SKevin Wolf } 761f1e84741SKevin Wolf 762f1e84741SKevin Wolf return qiov->size; 763f1e84741SKevin Wolf } 764f1e84741SKevin Wolf 765cf2ab8fcSKevin Wolf int bdrv_pread(BdrvChild *child, int64_t offset, void *buf, int bytes) 76661007b31SStefan Hajnoczi { 76761007b31SStefan Hajnoczi QEMUIOVector qiov; 76861007b31SStefan Hajnoczi struct iovec iov = { 76961007b31SStefan Hajnoczi .iov_base = (void *)buf, 77061007b31SStefan Hajnoczi .iov_len = bytes, 77161007b31SStefan Hajnoczi }; 77261007b31SStefan Hajnoczi 77361007b31SStefan Hajnoczi if (bytes < 0) { 77461007b31SStefan Hajnoczi return -EINVAL; 77561007b31SStefan Hajnoczi } 77661007b31SStefan Hajnoczi 77761007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 778cf2ab8fcSKevin Wolf return bdrv_preadv(child, offset, &qiov); 77961007b31SStefan Hajnoczi } 78061007b31SStefan Hajnoczi 781d9ca2ea2SKevin Wolf int bdrv_pwritev(BdrvChild *child, int64_t offset, QEMUIOVector *qiov) 78261007b31SStefan Hajnoczi { 78361007b31SStefan Hajnoczi int ret; 78461007b31SStefan Hajnoczi 785e293b7a3SKevin Wolf ret = bdrv_prwv_co(child, offset, qiov, true, 0); 78661007b31SStefan Hajnoczi if (ret < 0) { 78761007b31SStefan Hajnoczi return ret; 78861007b31SStefan Hajnoczi } 78961007b31SStefan Hajnoczi 79061007b31SStefan Hajnoczi return qiov->size; 79161007b31SStefan Hajnoczi } 79261007b31SStefan Hajnoczi 793d9ca2ea2SKevin Wolf int bdrv_pwrite(BdrvChild *child, int64_t offset, const void *buf, int bytes) 79461007b31SStefan Hajnoczi { 79561007b31SStefan Hajnoczi QEMUIOVector qiov; 79661007b31SStefan Hajnoczi struct iovec iov = { 79761007b31SStefan Hajnoczi .iov_base = (void *) buf, 79861007b31SStefan Hajnoczi .iov_len = bytes, 79961007b31SStefan Hajnoczi }; 80061007b31SStefan Hajnoczi 80161007b31SStefan Hajnoczi if (bytes < 0) { 80261007b31SStefan Hajnoczi return -EINVAL; 80361007b31SStefan Hajnoczi } 80461007b31SStefan Hajnoczi 80561007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 806d9ca2ea2SKevin Wolf return bdrv_pwritev(child, offset, &qiov); 80761007b31SStefan Hajnoczi } 80861007b31SStefan Hajnoczi 80961007b31SStefan Hajnoczi /* 81061007b31SStefan Hajnoczi * Writes to the file and ensures that no writes are reordered across this 81161007b31SStefan Hajnoczi * request (acts as a barrier) 81261007b31SStefan Hajnoczi * 81361007b31SStefan Hajnoczi * Returns 0 on success, -errno in error cases. 81461007b31SStefan Hajnoczi */ 815d9ca2ea2SKevin Wolf int bdrv_pwrite_sync(BdrvChild *child, int64_t offset, 81661007b31SStefan Hajnoczi const void *buf, int count) 81761007b31SStefan Hajnoczi { 81861007b31SStefan Hajnoczi int ret; 81961007b31SStefan Hajnoczi 820d9ca2ea2SKevin Wolf ret = bdrv_pwrite(child, offset, buf, count); 82161007b31SStefan Hajnoczi if (ret < 0) { 82261007b31SStefan Hajnoczi return ret; 82361007b31SStefan Hajnoczi } 82461007b31SStefan Hajnoczi 825d9ca2ea2SKevin Wolf ret = bdrv_flush(child->bs); 826855a6a93SKevin Wolf if (ret < 0) { 827855a6a93SKevin Wolf return ret; 82861007b31SStefan Hajnoczi } 82961007b31SStefan Hajnoczi 83061007b31SStefan Hajnoczi return 0; 83161007b31SStefan Hajnoczi } 83261007b31SStefan Hajnoczi 83308844473SKevin Wolf typedef struct CoroutineIOCompletion { 83408844473SKevin Wolf Coroutine *coroutine; 83508844473SKevin Wolf int ret; 83608844473SKevin Wolf } CoroutineIOCompletion; 83708844473SKevin Wolf 83808844473SKevin Wolf static void bdrv_co_io_em_complete(void *opaque, int ret) 83908844473SKevin Wolf { 84008844473SKevin Wolf CoroutineIOCompletion *co = opaque; 84108844473SKevin Wolf 84208844473SKevin Wolf co->ret = ret; 843b9e413ddSPaolo Bonzini aio_co_wake(co->coroutine); 84408844473SKevin Wolf } 84508844473SKevin Wolf 846166fe960SKevin Wolf static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs, 847166fe960SKevin Wolf uint64_t offset, uint64_t bytes, 848166fe960SKevin Wolf QEMUIOVector *qiov, int flags) 849166fe960SKevin Wolf { 850166fe960SKevin Wolf BlockDriver *drv = bs->drv; 8513fb06697SKevin Wolf int64_t sector_num; 8523fb06697SKevin Wolf unsigned int nb_sectors; 8533fb06697SKevin Wolf 854fa166538SEric Blake assert(!(flags & ~BDRV_REQ_MASK)); 855fa166538SEric Blake 8563fb06697SKevin Wolf if (drv->bdrv_co_preadv) { 8573fb06697SKevin Wolf return drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags); 8583fb06697SKevin Wolf } 8593fb06697SKevin Wolf 8603fb06697SKevin Wolf sector_num = offset >> BDRV_SECTOR_BITS; 8613fb06697SKevin Wolf nb_sectors = bytes >> BDRV_SECTOR_BITS; 862166fe960SKevin Wolf 863166fe960SKevin Wolf assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 864166fe960SKevin Wolf assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 865166fe960SKevin Wolf assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS); 866166fe960SKevin Wolf 86708844473SKevin Wolf if (drv->bdrv_co_readv) { 868166fe960SKevin Wolf return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov); 86908844473SKevin Wolf } else { 87008844473SKevin Wolf BlockAIOCB *acb; 87108844473SKevin Wolf CoroutineIOCompletion co = { 87208844473SKevin Wolf .coroutine = qemu_coroutine_self(), 87308844473SKevin Wolf }; 87408844473SKevin Wolf 87508844473SKevin Wolf acb = bs->drv->bdrv_aio_readv(bs, sector_num, qiov, nb_sectors, 87608844473SKevin Wolf bdrv_co_io_em_complete, &co); 87708844473SKevin Wolf if (acb == NULL) { 87808844473SKevin Wolf return -EIO; 87908844473SKevin Wolf } else { 88008844473SKevin Wolf qemu_coroutine_yield(); 88108844473SKevin Wolf return co.ret; 88208844473SKevin Wolf } 88308844473SKevin Wolf } 884166fe960SKevin Wolf } 885166fe960SKevin Wolf 88678a07294SKevin Wolf static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs, 88778a07294SKevin Wolf uint64_t offset, uint64_t bytes, 88878a07294SKevin Wolf QEMUIOVector *qiov, int flags) 88978a07294SKevin Wolf { 89078a07294SKevin Wolf BlockDriver *drv = bs->drv; 8913fb06697SKevin Wolf int64_t sector_num; 8923fb06697SKevin Wolf unsigned int nb_sectors; 89378a07294SKevin Wolf int ret; 89478a07294SKevin Wolf 895fa166538SEric Blake assert(!(flags & ~BDRV_REQ_MASK)); 896fa166538SEric Blake 8973fb06697SKevin Wolf if (drv->bdrv_co_pwritev) { 898515c2f43SKevin Wolf ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, 899515c2f43SKevin Wolf flags & bs->supported_write_flags); 900515c2f43SKevin Wolf flags &= ~bs->supported_write_flags; 9013fb06697SKevin Wolf goto emulate_flags; 9023fb06697SKevin Wolf } 9033fb06697SKevin Wolf 9043fb06697SKevin Wolf sector_num = offset >> BDRV_SECTOR_BITS; 9053fb06697SKevin Wolf nb_sectors = bytes >> BDRV_SECTOR_BITS; 9063fb06697SKevin Wolf 90778a07294SKevin Wolf assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 90878a07294SKevin Wolf assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 90978a07294SKevin Wolf assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS); 91078a07294SKevin Wolf 91178a07294SKevin Wolf if (drv->bdrv_co_writev_flags) { 91278a07294SKevin Wolf ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov, 9134df863f3SEric Blake flags & bs->supported_write_flags); 9144df863f3SEric Blake flags &= ~bs->supported_write_flags; 91508844473SKevin Wolf } else if (drv->bdrv_co_writev) { 9164df863f3SEric Blake assert(!bs->supported_write_flags); 91778a07294SKevin Wolf ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov); 91808844473SKevin Wolf } else { 91908844473SKevin Wolf BlockAIOCB *acb; 92008844473SKevin Wolf CoroutineIOCompletion co = { 92108844473SKevin Wolf .coroutine = qemu_coroutine_self(), 92208844473SKevin Wolf }; 92308844473SKevin Wolf 92408844473SKevin Wolf acb = bs->drv->bdrv_aio_writev(bs, sector_num, qiov, nb_sectors, 92508844473SKevin Wolf bdrv_co_io_em_complete, &co); 92608844473SKevin Wolf if (acb == NULL) { 9273fb06697SKevin Wolf ret = -EIO; 92808844473SKevin Wolf } else { 92908844473SKevin Wolf qemu_coroutine_yield(); 9303fb06697SKevin Wolf ret = co.ret; 93108844473SKevin Wolf } 93278a07294SKevin Wolf } 93378a07294SKevin Wolf 9343fb06697SKevin Wolf emulate_flags: 9354df863f3SEric Blake if (ret == 0 && (flags & BDRV_REQ_FUA)) { 93678a07294SKevin Wolf ret = bdrv_co_flush(bs); 93778a07294SKevin Wolf } 93878a07294SKevin Wolf 93978a07294SKevin Wolf return ret; 94078a07294SKevin Wolf } 94178a07294SKevin Wolf 94229a298afSPavel Butsykin static int coroutine_fn 94329a298afSPavel Butsykin bdrv_driver_pwritev_compressed(BlockDriverState *bs, uint64_t offset, 94429a298afSPavel Butsykin uint64_t bytes, QEMUIOVector *qiov) 94529a298afSPavel Butsykin { 94629a298afSPavel Butsykin BlockDriver *drv = bs->drv; 94729a298afSPavel Butsykin 94829a298afSPavel Butsykin if (!drv->bdrv_co_pwritev_compressed) { 94929a298afSPavel Butsykin return -ENOTSUP; 95029a298afSPavel Butsykin } 95129a298afSPavel Butsykin 95229a298afSPavel Butsykin return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov); 95329a298afSPavel Butsykin } 95429a298afSPavel Butsykin 95585c97ca7SKevin Wolf static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child, 956244483e6SKevin Wolf int64_t offset, unsigned int bytes, QEMUIOVector *qiov) 95761007b31SStefan Hajnoczi { 95885c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 95985c97ca7SKevin Wolf 96061007b31SStefan Hajnoczi /* Perform I/O through a temporary buffer so that users who scribble over 96161007b31SStefan Hajnoczi * their read buffer while the operation is in progress do not end up 96261007b31SStefan Hajnoczi * modifying the image file. This is critical for zero-copy guest I/O 96361007b31SStefan Hajnoczi * where anything might happen inside guest memory. 96461007b31SStefan Hajnoczi */ 96561007b31SStefan Hajnoczi void *bounce_buffer; 96661007b31SStefan Hajnoczi 96761007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 96861007b31SStefan Hajnoczi struct iovec iov; 969cb2e2878SEric Blake QEMUIOVector local_qiov; 970244483e6SKevin Wolf int64_t cluster_offset; 9717cfd5275SEric Blake int64_t cluster_bytes; 97261007b31SStefan Hajnoczi size_t skip_bytes; 97361007b31SStefan Hajnoczi int ret; 974cb2e2878SEric Blake int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, 975cb2e2878SEric Blake BDRV_REQUEST_MAX_BYTES); 976cb2e2878SEric Blake unsigned int progress = 0; 97761007b31SStefan Hajnoczi 9781bf03e66SKevin Wolf /* FIXME We cannot require callers to have write permissions when all they 9791bf03e66SKevin Wolf * are doing is a read request. If we did things right, write permissions 9801bf03e66SKevin Wolf * would be obtained anyway, but internally by the copy-on-read code. As 981765d9df9SEric Blake * long as it is implemented here rather than in a separate filter driver, 9821bf03e66SKevin Wolf * the copy-on-read code doesn't have its own BdrvChild, however, for which 9831bf03e66SKevin Wolf * it could request permissions. Therefore we have to bypass the permission 9841bf03e66SKevin Wolf * system for the moment. */ 9851bf03e66SKevin Wolf // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE)); 986afa4b293SKevin Wolf 98761007b31SStefan Hajnoczi /* Cover entire cluster so no additional backing file I/O is required when 988cb2e2878SEric Blake * allocating cluster in the image file. Note that this value may exceed 989cb2e2878SEric Blake * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which 990cb2e2878SEric Blake * is one reason we loop rather than doing it all at once. 99161007b31SStefan Hajnoczi */ 992244483e6SKevin Wolf bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes); 993cb2e2878SEric Blake skip_bytes = offset - cluster_offset; 99461007b31SStefan Hajnoczi 995244483e6SKevin Wolf trace_bdrv_co_do_copy_on_readv(bs, offset, bytes, 996244483e6SKevin Wolf cluster_offset, cluster_bytes); 99761007b31SStefan Hajnoczi 998cb2e2878SEric Blake bounce_buffer = qemu_try_blockalign(bs, 999cb2e2878SEric Blake MIN(MIN(max_transfer, cluster_bytes), 1000cb2e2878SEric Blake MAX_BOUNCE_BUFFER)); 100161007b31SStefan Hajnoczi if (bounce_buffer == NULL) { 100261007b31SStefan Hajnoczi ret = -ENOMEM; 100361007b31SStefan Hajnoczi goto err; 100461007b31SStefan Hajnoczi } 100561007b31SStefan Hajnoczi 1006cb2e2878SEric Blake while (cluster_bytes) { 1007cb2e2878SEric Blake int64_t pnum; 100861007b31SStefan Hajnoczi 1009cb2e2878SEric Blake ret = bdrv_is_allocated(bs, cluster_offset, 1010cb2e2878SEric Blake MIN(cluster_bytes, max_transfer), &pnum); 1011cb2e2878SEric Blake if (ret < 0) { 1012cb2e2878SEric Blake /* Safe to treat errors in querying allocation as if 1013cb2e2878SEric Blake * unallocated; we'll probably fail again soon on the 1014cb2e2878SEric Blake * read, but at least that will set a decent errno. 1015cb2e2878SEric Blake */ 1016cb2e2878SEric Blake pnum = MIN(cluster_bytes, max_transfer); 1017cb2e2878SEric Blake } 1018cb2e2878SEric Blake 1019cb2e2878SEric Blake assert(skip_bytes < pnum); 1020cb2e2878SEric Blake 1021cb2e2878SEric Blake if (ret <= 0) { 1022cb2e2878SEric Blake /* Must copy-on-read; use the bounce buffer */ 1023cb2e2878SEric Blake iov.iov_base = bounce_buffer; 1024cb2e2878SEric Blake iov.iov_len = pnum = MIN(pnum, MAX_BOUNCE_BUFFER); 1025cb2e2878SEric Blake qemu_iovec_init_external(&local_qiov, &iov, 1); 1026cb2e2878SEric Blake 1027cb2e2878SEric Blake ret = bdrv_driver_preadv(bs, cluster_offset, pnum, 1028cb2e2878SEric Blake &local_qiov, 0); 102961007b31SStefan Hajnoczi if (ret < 0) { 103061007b31SStefan Hajnoczi goto err; 103161007b31SStefan Hajnoczi } 103261007b31SStefan Hajnoczi 1033d855ebcdSEric Blake bdrv_debug_event(bs, BLKDBG_COR_WRITE); 1034c1499a5eSEric Blake if (drv->bdrv_co_pwrite_zeroes && 1035cb2e2878SEric Blake buffer_is_zero(bounce_buffer, pnum)) { 1036a604fa2bSEric Blake /* FIXME: Should we (perhaps conditionally) be setting 1037a604fa2bSEric Blake * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy 1038a604fa2bSEric Blake * that still correctly reads as zero? */ 1039cb2e2878SEric Blake ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, pnum, 0); 104061007b31SStefan Hajnoczi } else { 1041cb2e2878SEric Blake /* This does not change the data on the disk, it is not 1042cb2e2878SEric Blake * necessary to flush even in cache=writethrough mode. 104361007b31SStefan Hajnoczi */ 1044cb2e2878SEric Blake ret = bdrv_driver_pwritev(bs, cluster_offset, pnum, 1045cb2e2878SEric Blake &local_qiov, 0); 104661007b31SStefan Hajnoczi } 104761007b31SStefan Hajnoczi 104861007b31SStefan Hajnoczi if (ret < 0) { 1049cb2e2878SEric Blake /* It might be okay to ignore write errors for guest 1050cb2e2878SEric Blake * requests. If this is a deliberate copy-on-read 1051cb2e2878SEric Blake * then we don't want to ignore the error. Simply 1052cb2e2878SEric Blake * report it in all cases. 105361007b31SStefan Hajnoczi */ 105461007b31SStefan Hajnoczi goto err; 105561007b31SStefan Hajnoczi } 105661007b31SStefan Hajnoczi 1057cb2e2878SEric Blake qemu_iovec_from_buf(qiov, progress, bounce_buffer + skip_bytes, 1058cb2e2878SEric Blake pnum - skip_bytes); 1059cb2e2878SEric Blake } else { 1060cb2e2878SEric Blake /* Read directly into the destination */ 1061cb2e2878SEric Blake qemu_iovec_init(&local_qiov, qiov->niov); 1062cb2e2878SEric Blake qemu_iovec_concat(&local_qiov, qiov, progress, pnum - skip_bytes); 1063cb2e2878SEric Blake ret = bdrv_driver_preadv(bs, offset + progress, local_qiov.size, 1064cb2e2878SEric Blake &local_qiov, 0); 1065cb2e2878SEric Blake qemu_iovec_destroy(&local_qiov); 1066cb2e2878SEric Blake if (ret < 0) { 1067cb2e2878SEric Blake goto err; 1068cb2e2878SEric Blake } 1069cb2e2878SEric Blake } 1070cb2e2878SEric Blake 1071cb2e2878SEric Blake cluster_offset += pnum; 1072cb2e2878SEric Blake cluster_bytes -= pnum; 1073cb2e2878SEric Blake progress += pnum - skip_bytes; 1074cb2e2878SEric Blake skip_bytes = 0; 1075cb2e2878SEric Blake } 1076cb2e2878SEric Blake ret = 0; 107761007b31SStefan Hajnoczi 107861007b31SStefan Hajnoczi err: 107961007b31SStefan Hajnoczi qemu_vfree(bounce_buffer); 108061007b31SStefan Hajnoczi return ret; 108161007b31SStefan Hajnoczi } 108261007b31SStefan Hajnoczi 108361007b31SStefan Hajnoczi /* 108461007b31SStefan Hajnoczi * Forwards an already correctly aligned request to the BlockDriver. This 10851a62d0acSEric Blake * handles copy on read, zeroing after EOF, and fragmentation of large 10861a62d0acSEric Blake * reads; any other features must be implemented by the caller. 108761007b31SStefan Hajnoczi */ 108885c97ca7SKevin Wolf static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child, 108961007b31SStefan Hajnoczi BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 109061007b31SStefan Hajnoczi int64_t align, QEMUIOVector *qiov, int flags) 109161007b31SStefan Hajnoczi { 109285c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 1093c9d20029SKevin Wolf int64_t total_bytes, max_bytes; 10941a62d0acSEric Blake int ret = 0; 10951a62d0acSEric Blake uint64_t bytes_remaining = bytes; 10961a62d0acSEric Blake int max_transfer; 109761007b31SStefan Hajnoczi 109849c07526SKevin Wolf assert(is_power_of_2(align)); 109949c07526SKevin Wolf assert((offset & (align - 1)) == 0); 110049c07526SKevin Wolf assert((bytes & (align - 1)) == 0); 110161007b31SStefan Hajnoczi assert(!qiov || bytes == qiov->size); 1102abb06c5aSDaniel P. Berrange assert((bs->open_flags & BDRV_O_NO_IO) == 0); 11031a62d0acSEric Blake max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 11041a62d0acSEric Blake align); 1105a604fa2bSEric Blake 1106a604fa2bSEric Blake /* TODO: We would need a per-BDS .supported_read_flags and 1107a604fa2bSEric Blake * potential fallback support, if we ever implement any read flags 1108a604fa2bSEric Blake * to pass through to drivers. For now, there aren't any 1109a604fa2bSEric Blake * passthrough flags. */ 1110a604fa2bSEric Blake assert(!(flags & ~(BDRV_REQ_NO_SERIALISING | BDRV_REQ_COPY_ON_READ))); 111161007b31SStefan Hajnoczi 111261007b31SStefan Hajnoczi /* Handle Copy on Read and associated serialisation */ 111361007b31SStefan Hajnoczi if (flags & BDRV_REQ_COPY_ON_READ) { 111461007b31SStefan Hajnoczi /* If we touch the same cluster it counts as an overlap. This 111561007b31SStefan Hajnoczi * guarantees that allocating writes will be serialized and not race 111661007b31SStefan Hajnoczi * with each other for the same cluster. For example, in copy-on-read 111761007b31SStefan Hajnoczi * it ensures that the CoR read and write operations are atomic and 111861007b31SStefan Hajnoczi * guest writes cannot interleave between them. */ 111961007b31SStefan Hajnoczi mark_request_serialising(req, bdrv_get_cluster_size(bs)); 112061007b31SStefan Hajnoczi } 112161007b31SStefan Hajnoczi 112261408b25SFam Zheng if (!(flags & BDRV_REQ_NO_SERIALISING)) { 112361007b31SStefan Hajnoczi wait_serialising_requests(req); 112461408b25SFam Zheng } 112561007b31SStefan Hajnoczi 112661007b31SStefan Hajnoczi if (flags & BDRV_REQ_COPY_ON_READ) { 1127d6a644bbSEric Blake /* TODO: Simplify further once bdrv_is_allocated no longer 1128d6a644bbSEric Blake * requires sector alignment */ 1129d6a644bbSEric Blake int64_t start = QEMU_ALIGN_DOWN(offset, BDRV_SECTOR_SIZE); 1130d6a644bbSEric Blake int64_t end = QEMU_ALIGN_UP(offset + bytes, BDRV_SECTOR_SIZE); 1131d6a644bbSEric Blake int64_t pnum; 113261007b31SStefan Hajnoczi 1133d6a644bbSEric Blake ret = bdrv_is_allocated(bs, start, end - start, &pnum); 113461007b31SStefan Hajnoczi if (ret < 0) { 113561007b31SStefan Hajnoczi goto out; 113661007b31SStefan Hajnoczi } 113761007b31SStefan Hajnoczi 1138d6a644bbSEric Blake if (!ret || pnum != end - start) { 113985c97ca7SKevin Wolf ret = bdrv_co_do_copy_on_readv(child, offset, bytes, qiov); 114061007b31SStefan Hajnoczi goto out; 114161007b31SStefan Hajnoczi } 114261007b31SStefan Hajnoczi } 114361007b31SStefan Hajnoczi 11441a62d0acSEric Blake /* Forward the request to the BlockDriver, possibly fragmenting it */ 114549c07526SKevin Wolf total_bytes = bdrv_getlength(bs); 114649c07526SKevin Wolf if (total_bytes < 0) { 114749c07526SKevin Wolf ret = total_bytes; 114861007b31SStefan Hajnoczi goto out; 114961007b31SStefan Hajnoczi } 115061007b31SStefan Hajnoczi 115149c07526SKevin Wolf max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align); 11521a62d0acSEric Blake if (bytes <= max_bytes && bytes <= max_transfer) { 1153166fe960SKevin Wolf ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0); 11541a62d0acSEric Blake goto out; 115561007b31SStefan Hajnoczi } 115661007b31SStefan Hajnoczi 11571a62d0acSEric Blake while (bytes_remaining) { 11581a62d0acSEric Blake int num; 11591a62d0acSEric Blake 11601a62d0acSEric Blake if (max_bytes) { 11611a62d0acSEric Blake QEMUIOVector local_qiov; 11621a62d0acSEric Blake 11631a62d0acSEric Blake num = MIN(bytes_remaining, MIN(max_bytes, max_transfer)); 11641a62d0acSEric Blake assert(num); 11651a62d0acSEric Blake qemu_iovec_init(&local_qiov, qiov->niov); 11661a62d0acSEric Blake qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num); 11671a62d0acSEric Blake 11681a62d0acSEric Blake ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining, 11691a62d0acSEric Blake num, &local_qiov, 0); 11701a62d0acSEric Blake max_bytes -= num; 11711a62d0acSEric Blake qemu_iovec_destroy(&local_qiov); 11721a62d0acSEric Blake } else { 11731a62d0acSEric Blake num = bytes_remaining; 11741a62d0acSEric Blake ret = qemu_iovec_memset(qiov, bytes - bytes_remaining, 0, 11751a62d0acSEric Blake bytes_remaining); 11761a62d0acSEric Blake } 11771a62d0acSEric Blake if (ret < 0) { 11781a62d0acSEric Blake goto out; 11791a62d0acSEric Blake } 11801a62d0acSEric Blake bytes_remaining -= num; 118161007b31SStefan Hajnoczi } 118261007b31SStefan Hajnoczi 118361007b31SStefan Hajnoczi out: 11841a62d0acSEric Blake return ret < 0 ? ret : 0; 118561007b31SStefan Hajnoczi } 118661007b31SStefan Hajnoczi 118761007b31SStefan Hajnoczi /* 118861007b31SStefan Hajnoczi * Handle a read request in coroutine context 118961007b31SStefan Hajnoczi */ 1190a03ef88fSKevin Wolf int coroutine_fn bdrv_co_preadv(BdrvChild *child, 119161007b31SStefan Hajnoczi int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 119261007b31SStefan Hajnoczi BdrvRequestFlags flags) 119361007b31SStefan Hajnoczi { 1194a03ef88fSKevin Wolf BlockDriverState *bs = child->bs; 119561007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 119661007b31SStefan Hajnoczi BdrvTrackedRequest req; 119761007b31SStefan Hajnoczi 1198a5b8dd2cSEric Blake uint64_t align = bs->bl.request_alignment; 119961007b31SStefan Hajnoczi uint8_t *head_buf = NULL; 120061007b31SStefan Hajnoczi uint8_t *tail_buf = NULL; 120161007b31SStefan Hajnoczi QEMUIOVector local_qiov; 120261007b31SStefan Hajnoczi bool use_local_qiov = false; 120361007b31SStefan Hajnoczi int ret; 120461007b31SStefan Hajnoczi 1205f42cf447SDaniel P. Berrange trace_bdrv_co_preadv(child->bs, offset, bytes, flags); 1206f42cf447SDaniel P. Berrange 120761007b31SStefan Hajnoczi if (!drv) { 120861007b31SStefan Hajnoczi return -ENOMEDIUM; 120961007b31SStefan Hajnoczi } 121061007b31SStefan Hajnoczi 121161007b31SStefan Hajnoczi ret = bdrv_check_byte_request(bs, offset, bytes); 121261007b31SStefan Hajnoczi if (ret < 0) { 121361007b31SStefan Hajnoczi return ret; 121461007b31SStefan Hajnoczi } 121561007b31SStefan Hajnoczi 121699723548SPaolo Bonzini bdrv_inc_in_flight(bs); 121799723548SPaolo Bonzini 12189568b511SWen Congyang /* Don't do copy-on-read if we read data before write operation */ 1219d3faa13eSPaolo Bonzini if (atomic_read(&bs->copy_on_read) && !(flags & BDRV_REQ_NO_SERIALISING)) { 122061007b31SStefan Hajnoczi flags |= BDRV_REQ_COPY_ON_READ; 122161007b31SStefan Hajnoczi } 122261007b31SStefan Hajnoczi 122361007b31SStefan Hajnoczi /* Align read if necessary by padding qiov */ 122461007b31SStefan Hajnoczi if (offset & (align - 1)) { 122561007b31SStefan Hajnoczi head_buf = qemu_blockalign(bs, align); 122661007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 2); 122761007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 122861007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 122961007b31SStefan Hajnoczi use_local_qiov = true; 123061007b31SStefan Hajnoczi 123161007b31SStefan Hajnoczi bytes += offset & (align - 1); 123261007b31SStefan Hajnoczi offset = offset & ~(align - 1); 123361007b31SStefan Hajnoczi } 123461007b31SStefan Hajnoczi 123561007b31SStefan Hajnoczi if ((offset + bytes) & (align - 1)) { 123661007b31SStefan Hajnoczi if (!use_local_qiov) { 123761007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 1); 123861007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 123961007b31SStefan Hajnoczi use_local_qiov = true; 124061007b31SStefan Hajnoczi } 124161007b31SStefan Hajnoczi tail_buf = qemu_blockalign(bs, align); 124261007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, tail_buf, 124361007b31SStefan Hajnoczi align - ((offset + bytes) & (align - 1))); 124461007b31SStefan Hajnoczi 124561007b31SStefan Hajnoczi bytes = ROUND_UP(bytes, align); 124661007b31SStefan Hajnoczi } 124761007b31SStefan Hajnoczi 1248ebde595cSFam Zheng tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ); 124985c97ca7SKevin Wolf ret = bdrv_aligned_preadv(child, &req, offset, bytes, align, 125061007b31SStefan Hajnoczi use_local_qiov ? &local_qiov : qiov, 125161007b31SStefan Hajnoczi flags); 125261007b31SStefan Hajnoczi tracked_request_end(&req); 125399723548SPaolo Bonzini bdrv_dec_in_flight(bs); 125461007b31SStefan Hajnoczi 125561007b31SStefan Hajnoczi if (use_local_qiov) { 125661007b31SStefan Hajnoczi qemu_iovec_destroy(&local_qiov); 125761007b31SStefan Hajnoczi qemu_vfree(head_buf); 125861007b31SStefan Hajnoczi qemu_vfree(tail_buf); 125961007b31SStefan Hajnoczi } 126061007b31SStefan Hajnoczi 126161007b31SStefan Hajnoczi return ret; 126261007b31SStefan Hajnoczi } 126361007b31SStefan Hajnoczi 1264adad6496SKevin Wolf static int coroutine_fn bdrv_co_do_readv(BdrvChild *child, 126561007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, 126661007b31SStefan Hajnoczi BdrvRequestFlags flags) 126761007b31SStefan Hajnoczi { 126861007b31SStefan Hajnoczi if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 126961007b31SStefan Hajnoczi return -EINVAL; 127061007b31SStefan Hajnoczi } 127161007b31SStefan Hajnoczi 1272a03ef88fSKevin Wolf return bdrv_co_preadv(child, sector_num << BDRV_SECTOR_BITS, 127361007b31SStefan Hajnoczi nb_sectors << BDRV_SECTOR_BITS, qiov, flags); 127461007b31SStefan Hajnoczi } 127561007b31SStefan Hajnoczi 127628b04a8fSKevin Wolf int coroutine_fn bdrv_co_readv(BdrvChild *child, int64_t sector_num, 127761007b31SStefan Hajnoczi int nb_sectors, QEMUIOVector *qiov) 127861007b31SStefan Hajnoczi { 1279adad6496SKevin Wolf return bdrv_co_do_readv(child, sector_num, nb_sectors, qiov, 0); 128061007b31SStefan Hajnoczi } 128161007b31SStefan Hajnoczi 1282d05aa8bbSEric Blake static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 1283f5a5ca79SManos Pitsidianakis int64_t offset, int bytes, BdrvRequestFlags flags) 128461007b31SStefan Hajnoczi { 128561007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 128661007b31SStefan Hajnoczi QEMUIOVector qiov; 128761007b31SStefan Hajnoczi struct iovec iov = {0}; 128861007b31SStefan Hajnoczi int ret = 0; 1289465fe887SEric Blake bool need_flush = false; 1290443668caSDenis V. Lunev int head = 0; 1291443668caSDenis V. Lunev int tail = 0; 129261007b31SStefan Hajnoczi 1293cf081fcaSEric Blake int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, INT_MAX); 1294a5b8dd2cSEric Blake int alignment = MAX(bs->bl.pwrite_zeroes_alignment, 1295a5b8dd2cSEric Blake bs->bl.request_alignment); 1296cb2e2878SEric Blake int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER); 1297cf081fcaSEric Blake 1298b8d0a980SEric Blake assert(alignment % bs->bl.request_alignment == 0); 1299b8d0a980SEric Blake head = offset % alignment; 1300f5a5ca79SManos Pitsidianakis tail = (offset + bytes) % alignment; 1301b8d0a980SEric Blake max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment); 1302b8d0a980SEric Blake assert(max_write_zeroes >= bs->bl.request_alignment); 130361007b31SStefan Hajnoczi 1304f5a5ca79SManos Pitsidianakis while (bytes > 0 && !ret) { 1305f5a5ca79SManos Pitsidianakis int num = bytes; 130661007b31SStefan Hajnoczi 130761007b31SStefan Hajnoczi /* Align request. Block drivers can expect the "bulk" of the request 1308443668caSDenis V. Lunev * to be aligned, and that unaligned requests do not cross cluster 1309443668caSDenis V. Lunev * boundaries. 131061007b31SStefan Hajnoczi */ 1311443668caSDenis V. Lunev if (head) { 1312b2f95feeSEric Blake /* Make a small request up to the first aligned sector. For 1313b2f95feeSEric Blake * convenience, limit this request to max_transfer even if 1314b2f95feeSEric Blake * we don't need to fall back to writes. */ 1315f5a5ca79SManos Pitsidianakis num = MIN(MIN(bytes, max_transfer), alignment - head); 1316b2f95feeSEric Blake head = (head + num) % alignment; 1317b2f95feeSEric Blake assert(num < max_write_zeroes); 1318d05aa8bbSEric Blake } else if (tail && num > alignment) { 1319443668caSDenis V. Lunev /* Shorten the request to the last aligned sector. */ 1320443668caSDenis V. Lunev num -= tail; 132161007b31SStefan Hajnoczi } 132261007b31SStefan Hajnoczi 132361007b31SStefan Hajnoczi /* limit request size */ 132461007b31SStefan Hajnoczi if (num > max_write_zeroes) { 132561007b31SStefan Hajnoczi num = max_write_zeroes; 132661007b31SStefan Hajnoczi } 132761007b31SStefan Hajnoczi 132861007b31SStefan Hajnoczi ret = -ENOTSUP; 132961007b31SStefan Hajnoczi /* First try the efficient write zeroes operation */ 1330d05aa8bbSEric Blake if (drv->bdrv_co_pwrite_zeroes) { 1331d05aa8bbSEric Blake ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num, 1332d05aa8bbSEric Blake flags & bs->supported_zero_flags); 1333d05aa8bbSEric Blake if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) && 1334d05aa8bbSEric Blake !(bs->supported_zero_flags & BDRV_REQ_FUA)) { 1335d05aa8bbSEric Blake need_flush = true; 1336d05aa8bbSEric Blake } 1337465fe887SEric Blake } else { 1338465fe887SEric Blake assert(!bs->supported_zero_flags); 133961007b31SStefan Hajnoczi } 134061007b31SStefan Hajnoczi 134161007b31SStefan Hajnoczi if (ret == -ENOTSUP) { 134261007b31SStefan Hajnoczi /* Fall back to bounce buffer if write zeroes is unsupported */ 1343465fe887SEric Blake BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE; 1344465fe887SEric Blake 1345465fe887SEric Blake if ((flags & BDRV_REQ_FUA) && 1346465fe887SEric Blake !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1347465fe887SEric Blake /* No need for bdrv_driver_pwrite() to do a fallback 1348465fe887SEric Blake * flush on each chunk; use just one at the end */ 1349465fe887SEric Blake write_flags &= ~BDRV_REQ_FUA; 1350465fe887SEric Blake need_flush = true; 1351465fe887SEric Blake } 13525def6b80SEric Blake num = MIN(num, max_transfer); 1353d05aa8bbSEric Blake iov.iov_len = num; 135461007b31SStefan Hajnoczi if (iov.iov_base == NULL) { 1355d05aa8bbSEric Blake iov.iov_base = qemu_try_blockalign(bs, num); 135661007b31SStefan Hajnoczi if (iov.iov_base == NULL) { 135761007b31SStefan Hajnoczi ret = -ENOMEM; 135861007b31SStefan Hajnoczi goto fail; 135961007b31SStefan Hajnoczi } 1360d05aa8bbSEric Blake memset(iov.iov_base, 0, num); 136161007b31SStefan Hajnoczi } 136261007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 136361007b31SStefan Hajnoczi 1364d05aa8bbSEric Blake ret = bdrv_driver_pwritev(bs, offset, num, &qiov, write_flags); 136561007b31SStefan Hajnoczi 136661007b31SStefan Hajnoczi /* Keep bounce buffer around if it is big enough for all 136761007b31SStefan Hajnoczi * all future requests. 136861007b31SStefan Hajnoczi */ 13695def6b80SEric Blake if (num < max_transfer) { 137061007b31SStefan Hajnoczi qemu_vfree(iov.iov_base); 137161007b31SStefan Hajnoczi iov.iov_base = NULL; 137261007b31SStefan Hajnoczi } 137361007b31SStefan Hajnoczi } 137461007b31SStefan Hajnoczi 1375d05aa8bbSEric Blake offset += num; 1376f5a5ca79SManos Pitsidianakis bytes -= num; 137761007b31SStefan Hajnoczi } 137861007b31SStefan Hajnoczi 137961007b31SStefan Hajnoczi fail: 1380465fe887SEric Blake if (ret == 0 && need_flush) { 1381465fe887SEric Blake ret = bdrv_co_flush(bs); 1382465fe887SEric Blake } 138361007b31SStefan Hajnoczi qemu_vfree(iov.iov_base); 138461007b31SStefan Hajnoczi return ret; 138561007b31SStefan Hajnoczi } 138661007b31SStefan Hajnoczi 138761007b31SStefan Hajnoczi /* 138804ed95f4SEric Blake * Forwards an already correctly aligned write request to the BlockDriver, 138904ed95f4SEric Blake * after possibly fragmenting it. 139061007b31SStefan Hajnoczi */ 139185c97ca7SKevin Wolf static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child, 139261007b31SStefan Hajnoczi BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 1393cff86b38SEric Blake int64_t align, QEMUIOVector *qiov, int flags) 139461007b31SStefan Hajnoczi { 139585c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 139661007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 139761007b31SStefan Hajnoczi bool waited; 139861007b31SStefan Hajnoczi int ret; 139961007b31SStefan Hajnoczi 14009896c876SKevin Wolf int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 140104ed95f4SEric Blake uint64_t bytes_remaining = bytes; 140204ed95f4SEric Blake int max_transfer; 140361007b31SStefan Hajnoczi 1404d6883bc9SVladimir Sementsov-Ogievskiy if (bdrv_has_readonly_bitmaps(bs)) { 1405d6883bc9SVladimir Sementsov-Ogievskiy return -EPERM; 1406d6883bc9SVladimir Sementsov-Ogievskiy } 1407d6883bc9SVladimir Sementsov-Ogievskiy 1408cff86b38SEric Blake assert(is_power_of_2(align)); 1409cff86b38SEric Blake assert((offset & (align - 1)) == 0); 1410cff86b38SEric Blake assert((bytes & (align - 1)) == 0); 141161007b31SStefan Hajnoczi assert(!qiov || bytes == qiov->size); 1412abb06c5aSDaniel P. Berrange assert((bs->open_flags & BDRV_O_NO_IO) == 0); 1413fa166538SEric Blake assert(!(flags & ~BDRV_REQ_MASK)); 141404ed95f4SEric Blake max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 141504ed95f4SEric Blake align); 141661007b31SStefan Hajnoczi 141761007b31SStefan Hajnoczi waited = wait_serialising_requests(req); 141861007b31SStefan Hajnoczi assert(!waited || !req->serialising); 141961007b31SStefan Hajnoczi assert(req->overlap_offset <= offset); 142061007b31SStefan Hajnoczi assert(offset + bytes <= req->overlap_offset + req->overlap_bytes); 1421362b3786SMax Reitz assert(child->perm & BLK_PERM_WRITE); 1422362b3786SMax Reitz assert(end_sector <= bs->total_sectors || child->perm & BLK_PERM_RESIZE); 142361007b31SStefan Hajnoczi 142461007b31SStefan Hajnoczi ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req); 142561007b31SStefan Hajnoczi 142661007b31SStefan Hajnoczi if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF && 1427c1499a5eSEric Blake !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes && 142861007b31SStefan Hajnoczi qemu_iovec_is_zero(qiov)) { 142961007b31SStefan Hajnoczi flags |= BDRV_REQ_ZERO_WRITE; 143061007b31SStefan Hajnoczi if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) { 143161007b31SStefan Hajnoczi flags |= BDRV_REQ_MAY_UNMAP; 143261007b31SStefan Hajnoczi } 143361007b31SStefan Hajnoczi } 143461007b31SStefan Hajnoczi 143561007b31SStefan Hajnoczi if (ret < 0) { 143661007b31SStefan Hajnoczi /* Do nothing, write notifier decided to fail this request */ 143761007b31SStefan Hajnoczi } else if (flags & BDRV_REQ_ZERO_WRITE) { 14389a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO); 14399896c876SKevin Wolf ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags); 14403ea1a091SPavel Butsykin } else if (flags & BDRV_REQ_WRITE_COMPRESSED) { 14413ea1a091SPavel Butsykin ret = bdrv_driver_pwritev_compressed(bs, offset, bytes, qiov); 144204ed95f4SEric Blake } else if (bytes <= max_transfer) { 14439a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV); 144478a07294SKevin Wolf ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, flags); 144504ed95f4SEric Blake } else { 144604ed95f4SEric Blake bdrv_debug_event(bs, BLKDBG_PWRITEV); 144704ed95f4SEric Blake while (bytes_remaining) { 144804ed95f4SEric Blake int num = MIN(bytes_remaining, max_transfer); 144904ed95f4SEric Blake QEMUIOVector local_qiov; 145004ed95f4SEric Blake int local_flags = flags; 145104ed95f4SEric Blake 145204ed95f4SEric Blake assert(num); 145304ed95f4SEric Blake if (num < bytes_remaining && (flags & BDRV_REQ_FUA) && 145404ed95f4SEric Blake !(bs->supported_write_flags & BDRV_REQ_FUA)) { 145504ed95f4SEric Blake /* If FUA is going to be emulated by flush, we only 145604ed95f4SEric Blake * need to flush on the last iteration */ 145704ed95f4SEric Blake local_flags &= ~BDRV_REQ_FUA; 145804ed95f4SEric Blake } 145904ed95f4SEric Blake qemu_iovec_init(&local_qiov, qiov->niov); 146004ed95f4SEric Blake qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num); 146104ed95f4SEric Blake 146204ed95f4SEric Blake ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining, 146304ed95f4SEric Blake num, &local_qiov, local_flags); 146404ed95f4SEric Blake qemu_iovec_destroy(&local_qiov); 146504ed95f4SEric Blake if (ret < 0) { 146604ed95f4SEric Blake break; 146704ed95f4SEric Blake } 146804ed95f4SEric Blake bytes_remaining -= num; 146904ed95f4SEric Blake } 147061007b31SStefan Hajnoczi } 14719a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE); 147261007b31SStefan Hajnoczi 147347fec599SPaolo Bonzini atomic_inc(&bs->write_gen); 14740fdf1a4fSEric Blake bdrv_set_dirty(bs, offset, bytes); 147561007b31SStefan Hajnoczi 1476f7946da2SPaolo Bonzini stat64_max(&bs->wr_highest_offset, offset + bytes); 147761007b31SStefan Hajnoczi 147861007b31SStefan Hajnoczi if (ret >= 0) { 14799896c876SKevin Wolf bs->total_sectors = MAX(bs->total_sectors, end_sector); 148004ed95f4SEric Blake ret = 0; 148161007b31SStefan Hajnoczi } 148261007b31SStefan Hajnoczi 148361007b31SStefan Hajnoczi return ret; 148461007b31SStefan Hajnoczi } 148561007b31SStefan Hajnoczi 148685c97ca7SKevin Wolf static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child, 14879eeb6dd1SFam Zheng int64_t offset, 14889eeb6dd1SFam Zheng unsigned int bytes, 14899eeb6dd1SFam Zheng BdrvRequestFlags flags, 14909eeb6dd1SFam Zheng BdrvTrackedRequest *req) 14919eeb6dd1SFam Zheng { 149285c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 14939eeb6dd1SFam Zheng uint8_t *buf = NULL; 14949eeb6dd1SFam Zheng QEMUIOVector local_qiov; 14959eeb6dd1SFam Zheng struct iovec iov; 1496a5b8dd2cSEric Blake uint64_t align = bs->bl.request_alignment; 14979eeb6dd1SFam Zheng unsigned int head_padding_bytes, tail_padding_bytes; 14989eeb6dd1SFam Zheng int ret = 0; 14999eeb6dd1SFam Zheng 15009eeb6dd1SFam Zheng head_padding_bytes = offset & (align - 1); 1501f13ce1beSDenis V. Lunev tail_padding_bytes = (align - (offset + bytes)) & (align - 1); 15029eeb6dd1SFam Zheng 15039eeb6dd1SFam Zheng 15049eeb6dd1SFam Zheng assert(flags & BDRV_REQ_ZERO_WRITE); 15059eeb6dd1SFam Zheng if (head_padding_bytes || tail_padding_bytes) { 15069eeb6dd1SFam Zheng buf = qemu_blockalign(bs, align); 15079eeb6dd1SFam Zheng iov = (struct iovec) { 15089eeb6dd1SFam Zheng .iov_base = buf, 15099eeb6dd1SFam Zheng .iov_len = align, 15109eeb6dd1SFam Zheng }; 15119eeb6dd1SFam Zheng qemu_iovec_init_external(&local_qiov, &iov, 1); 15129eeb6dd1SFam Zheng } 15139eeb6dd1SFam Zheng if (head_padding_bytes) { 15149eeb6dd1SFam Zheng uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes); 15159eeb6dd1SFam Zheng 15169eeb6dd1SFam Zheng /* RMW the unaligned part before head. */ 15179eeb6dd1SFam Zheng mark_request_serialising(req, align); 15189eeb6dd1SFam Zheng wait_serialising_requests(req); 15199a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 152085c97ca7SKevin Wolf ret = bdrv_aligned_preadv(child, req, offset & ~(align - 1), align, 15219eeb6dd1SFam Zheng align, &local_qiov, 0); 15229eeb6dd1SFam Zheng if (ret < 0) { 15239eeb6dd1SFam Zheng goto fail; 15249eeb6dd1SFam Zheng } 15259a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 15269eeb6dd1SFam Zheng 15279eeb6dd1SFam Zheng memset(buf + head_padding_bytes, 0, zero_bytes); 152885c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, req, offset & ~(align - 1), align, 1529cff86b38SEric Blake align, &local_qiov, 15309eeb6dd1SFam Zheng flags & ~BDRV_REQ_ZERO_WRITE); 15319eeb6dd1SFam Zheng if (ret < 0) { 15329eeb6dd1SFam Zheng goto fail; 15339eeb6dd1SFam Zheng } 15349eeb6dd1SFam Zheng offset += zero_bytes; 15359eeb6dd1SFam Zheng bytes -= zero_bytes; 15369eeb6dd1SFam Zheng } 15379eeb6dd1SFam Zheng 15389eeb6dd1SFam Zheng assert(!bytes || (offset & (align - 1)) == 0); 15399eeb6dd1SFam Zheng if (bytes >= align) { 15409eeb6dd1SFam Zheng /* Write the aligned part in the middle. */ 15419eeb6dd1SFam Zheng uint64_t aligned_bytes = bytes & ~(align - 1); 154285c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align, 15439eeb6dd1SFam Zheng NULL, flags); 15449eeb6dd1SFam Zheng if (ret < 0) { 15459eeb6dd1SFam Zheng goto fail; 15469eeb6dd1SFam Zheng } 15479eeb6dd1SFam Zheng bytes -= aligned_bytes; 15489eeb6dd1SFam Zheng offset += aligned_bytes; 15499eeb6dd1SFam Zheng } 15509eeb6dd1SFam Zheng 15519eeb6dd1SFam Zheng assert(!bytes || (offset & (align - 1)) == 0); 15529eeb6dd1SFam Zheng if (bytes) { 15539eeb6dd1SFam Zheng assert(align == tail_padding_bytes + bytes); 15549eeb6dd1SFam Zheng /* RMW the unaligned part after tail. */ 15559eeb6dd1SFam Zheng mark_request_serialising(req, align); 15569eeb6dd1SFam Zheng wait_serialising_requests(req); 15579a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 155885c97ca7SKevin Wolf ret = bdrv_aligned_preadv(child, req, offset, align, 15599eeb6dd1SFam Zheng align, &local_qiov, 0); 15609eeb6dd1SFam Zheng if (ret < 0) { 15619eeb6dd1SFam Zheng goto fail; 15629eeb6dd1SFam Zheng } 15639a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 15649eeb6dd1SFam Zheng 15659eeb6dd1SFam Zheng memset(buf, 0, bytes); 156685c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, req, offset, align, align, 15679eeb6dd1SFam Zheng &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE); 15689eeb6dd1SFam Zheng } 15699eeb6dd1SFam Zheng fail: 15709eeb6dd1SFam Zheng qemu_vfree(buf); 15719eeb6dd1SFam Zheng return ret; 15729eeb6dd1SFam Zheng 15739eeb6dd1SFam Zheng } 15749eeb6dd1SFam Zheng 157561007b31SStefan Hajnoczi /* 157661007b31SStefan Hajnoczi * Handle a write request in coroutine context 157761007b31SStefan Hajnoczi */ 1578a03ef88fSKevin Wolf int coroutine_fn bdrv_co_pwritev(BdrvChild *child, 157961007b31SStefan Hajnoczi int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 158061007b31SStefan Hajnoczi BdrvRequestFlags flags) 158161007b31SStefan Hajnoczi { 1582a03ef88fSKevin Wolf BlockDriverState *bs = child->bs; 158361007b31SStefan Hajnoczi BdrvTrackedRequest req; 1584a5b8dd2cSEric Blake uint64_t align = bs->bl.request_alignment; 158561007b31SStefan Hajnoczi uint8_t *head_buf = NULL; 158661007b31SStefan Hajnoczi uint8_t *tail_buf = NULL; 158761007b31SStefan Hajnoczi QEMUIOVector local_qiov; 158861007b31SStefan Hajnoczi bool use_local_qiov = false; 158961007b31SStefan Hajnoczi int ret; 159061007b31SStefan Hajnoczi 1591f42cf447SDaniel P. Berrange trace_bdrv_co_pwritev(child->bs, offset, bytes, flags); 1592f42cf447SDaniel P. Berrange 159361007b31SStefan Hajnoczi if (!bs->drv) { 159461007b31SStefan Hajnoczi return -ENOMEDIUM; 159561007b31SStefan Hajnoczi } 159661007b31SStefan Hajnoczi if (bs->read_only) { 1597eaf5fe2dSPaolo Bonzini return -EPERM; 159861007b31SStefan Hajnoczi } 159904c01a5cSKevin Wolf assert(!(bs->open_flags & BDRV_O_INACTIVE)); 160061007b31SStefan Hajnoczi 160161007b31SStefan Hajnoczi ret = bdrv_check_byte_request(bs, offset, bytes); 160261007b31SStefan Hajnoczi if (ret < 0) { 160361007b31SStefan Hajnoczi return ret; 160461007b31SStefan Hajnoczi } 160561007b31SStefan Hajnoczi 160699723548SPaolo Bonzini bdrv_inc_in_flight(bs); 160761007b31SStefan Hajnoczi /* 160861007b31SStefan Hajnoczi * Align write if necessary by performing a read-modify-write cycle. 160961007b31SStefan Hajnoczi * Pad qiov with the read parts and be sure to have a tracked request not 161061007b31SStefan Hajnoczi * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle. 161161007b31SStefan Hajnoczi */ 1612ebde595cSFam Zheng tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE); 161361007b31SStefan Hajnoczi 16149eeb6dd1SFam Zheng if (!qiov) { 161585c97ca7SKevin Wolf ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req); 16169eeb6dd1SFam Zheng goto out; 16179eeb6dd1SFam Zheng } 16189eeb6dd1SFam Zheng 161961007b31SStefan Hajnoczi if (offset & (align - 1)) { 162061007b31SStefan Hajnoczi QEMUIOVector head_qiov; 162161007b31SStefan Hajnoczi struct iovec head_iov; 162261007b31SStefan Hajnoczi 162361007b31SStefan Hajnoczi mark_request_serialising(&req, align); 162461007b31SStefan Hajnoczi wait_serialising_requests(&req); 162561007b31SStefan Hajnoczi 162661007b31SStefan Hajnoczi head_buf = qemu_blockalign(bs, align); 162761007b31SStefan Hajnoczi head_iov = (struct iovec) { 162861007b31SStefan Hajnoczi .iov_base = head_buf, 162961007b31SStefan Hajnoczi .iov_len = align, 163061007b31SStefan Hajnoczi }; 163161007b31SStefan Hajnoczi qemu_iovec_init_external(&head_qiov, &head_iov, 1); 163261007b31SStefan Hajnoczi 16339a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 163485c97ca7SKevin Wolf ret = bdrv_aligned_preadv(child, &req, offset & ~(align - 1), align, 163561007b31SStefan Hajnoczi align, &head_qiov, 0); 163661007b31SStefan Hajnoczi if (ret < 0) { 163761007b31SStefan Hajnoczi goto fail; 163861007b31SStefan Hajnoczi } 16399a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 164061007b31SStefan Hajnoczi 164161007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 2); 164261007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 164361007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 164461007b31SStefan Hajnoczi use_local_qiov = true; 164561007b31SStefan Hajnoczi 164661007b31SStefan Hajnoczi bytes += offset & (align - 1); 164761007b31SStefan Hajnoczi offset = offset & ~(align - 1); 1648117bc3faSPeter Lieven 1649117bc3faSPeter Lieven /* We have read the tail already if the request is smaller 1650117bc3faSPeter Lieven * than one aligned block. 1651117bc3faSPeter Lieven */ 1652117bc3faSPeter Lieven if (bytes < align) { 1653117bc3faSPeter Lieven qemu_iovec_add(&local_qiov, head_buf + bytes, align - bytes); 1654117bc3faSPeter Lieven bytes = align; 1655117bc3faSPeter Lieven } 165661007b31SStefan Hajnoczi } 165761007b31SStefan Hajnoczi 165861007b31SStefan Hajnoczi if ((offset + bytes) & (align - 1)) { 165961007b31SStefan Hajnoczi QEMUIOVector tail_qiov; 166061007b31SStefan Hajnoczi struct iovec tail_iov; 166161007b31SStefan Hajnoczi size_t tail_bytes; 166261007b31SStefan Hajnoczi bool waited; 166361007b31SStefan Hajnoczi 166461007b31SStefan Hajnoczi mark_request_serialising(&req, align); 166561007b31SStefan Hajnoczi waited = wait_serialising_requests(&req); 166661007b31SStefan Hajnoczi assert(!waited || !use_local_qiov); 166761007b31SStefan Hajnoczi 166861007b31SStefan Hajnoczi tail_buf = qemu_blockalign(bs, align); 166961007b31SStefan Hajnoczi tail_iov = (struct iovec) { 167061007b31SStefan Hajnoczi .iov_base = tail_buf, 167161007b31SStefan Hajnoczi .iov_len = align, 167261007b31SStefan Hajnoczi }; 167361007b31SStefan Hajnoczi qemu_iovec_init_external(&tail_qiov, &tail_iov, 1); 167461007b31SStefan Hajnoczi 16759a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 167685c97ca7SKevin Wolf ret = bdrv_aligned_preadv(child, &req, (offset + bytes) & ~(align - 1), 167785c97ca7SKevin Wolf align, align, &tail_qiov, 0); 167861007b31SStefan Hajnoczi if (ret < 0) { 167961007b31SStefan Hajnoczi goto fail; 168061007b31SStefan Hajnoczi } 16819a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 168261007b31SStefan Hajnoczi 168361007b31SStefan Hajnoczi if (!use_local_qiov) { 168461007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 1); 168561007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 168661007b31SStefan Hajnoczi use_local_qiov = true; 168761007b31SStefan Hajnoczi } 168861007b31SStefan Hajnoczi 168961007b31SStefan Hajnoczi tail_bytes = (offset + bytes) & (align - 1); 169061007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes); 169161007b31SStefan Hajnoczi 169261007b31SStefan Hajnoczi bytes = ROUND_UP(bytes, align); 169361007b31SStefan Hajnoczi } 169461007b31SStefan Hajnoczi 169585c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align, 169661007b31SStefan Hajnoczi use_local_qiov ? &local_qiov : qiov, 169761007b31SStefan Hajnoczi flags); 169861007b31SStefan Hajnoczi 169961007b31SStefan Hajnoczi fail: 170061007b31SStefan Hajnoczi 170161007b31SStefan Hajnoczi if (use_local_qiov) { 170261007b31SStefan Hajnoczi qemu_iovec_destroy(&local_qiov); 170361007b31SStefan Hajnoczi } 170461007b31SStefan Hajnoczi qemu_vfree(head_buf); 170561007b31SStefan Hajnoczi qemu_vfree(tail_buf); 17069eeb6dd1SFam Zheng out: 17079eeb6dd1SFam Zheng tracked_request_end(&req); 170899723548SPaolo Bonzini bdrv_dec_in_flight(bs); 170961007b31SStefan Hajnoczi return ret; 171061007b31SStefan Hajnoczi } 171161007b31SStefan Hajnoczi 1712adad6496SKevin Wolf static int coroutine_fn bdrv_co_do_writev(BdrvChild *child, 171361007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, 171461007b31SStefan Hajnoczi BdrvRequestFlags flags) 171561007b31SStefan Hajnoczi { 171661007b31SStefan Hajnoczi if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 171761007b31SStefan Hajnoczi return -EINVAL; 171861007b31SStefan Hajnoczi } 171961007b31SStefan Hajnoczi 1720a03ef88fSKevin Wolf return bdrv_co_pwritev(child, sector_num << BDRV_SECTOR_BITS, 172161007b31SStefan Hajnoczi nb_sectors << BDRV_SECTOR_BITS, qiov, flags); 172261007b31SStefan Hajnoczi } 172361007b31SStefan Hajnoczi 172425ec177dSKevin Wolf int coroutine_fn bdrv_co_writev(BdrvChild *child, int64_t sector_num, 172561007b31SStefan Hajnoczi int nb_sectors, QEMUIOVector *qiov) 172661007b31SStefan Hajnoczi { 1727adad6496SKevin Wolf return bdrv_co_do_writev(child, sector_num, nb_sectors, qiov, 0); 172861007b31SStefan Hajnoczi } 172961007b31SStefan Hajnoczi 1730a03ef88fSKevin Wolf int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset, 1731f5a5ca79SManos Pitsidianakis int bytes, BdrvRequestFlags flags) 173261007b31SStefan Hajnoczi { 1733f5a5ca79SManos Pitsidianakis trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags); 173461007b31SStefan Hajnoczi 1735a03ef88fSKevin Wolf if (!(child->bs->open_flags & BDRV_O_UNMAP)) { 173661007b31SStefan Hajnoczi flags &= ~BDRV_REQ_MAY_UNMAP; 173761007b31SStefan Hajnoczi } 173861007b31SStefan Hajnoczi 1739f5a5ca79SManos Pitsidianakis return bdrv_co_pwritev(child, offset, bytes, NULL, 174061007b31SStefan Hajnoczi BDRV_REQ_ZERO_WRITE | flags); 174161007b31SStefan Hajnoczi } 174261007b31SStefan Hajnoczi 17434085f5c7SJohn Snow /* 17444085f5c7SJohn Snow * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not. 17454085f5c7SJohn Snow */ 17464085f5c7SJohn Snow int bdrv_flush_all(void) 17474085f5c7SJohn Snow { 17484085f5c7SJohn Snow BdrvNextIterator it; 17494085f5c7SJohn Snow BlockDriverState *bs = NULL; 17504085f5c7SJohn Snow int result = 0; 17514085f5c7SJohn Snow 17524085f5c7SJohn Snow for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 17534085f5c7SJohn Snow AioContext *aio_context = bdrv_get_aio_context(bs); 17544085f5c7SJohn Snow int ret; 17554085f5c7SJohn Snow 17564085f5c7SJohn Snow aio_context_acquire(aio_context); 17574085f5c7SJohn Snow ret = bdrv_flush(bs); 17584085f5c7SJohn Snow if (ret < 0 && !result) { 17594085f5c7SJohn Snow result = ret; 17604085f5c7SJohn Snow } 17614085f5c7SJohn Snow aio_context_release(aio_context); 17624085f5c7SJohn Snow } 17634085f5c7SJohn Snow 17644085f5c7SJohn Snow return result; 17654085f5c7SJohn Snow } 17664085f5c7SJohn Snow 17674085f5c7SJohn Snow 176861007b31SStefan Hajnoczi typedef struct BdrvCoGetBlockStatusData { 176961007b31SStefan Hajnoczi BlockDriverState *bs; 177061007b31SStefan Hajnoczi BlockDriverState *base; 1771c9ce8c4dSEric Blake bool want_zero; 177261007b31SStefan Hajnoczi int64_t sector_num; 177361007b31SStefan Hajnoczi int nb_sectors; 177461007b31SStefan Hajnoczi int *pnum; 1775c9ce8c4dSEric Blake BlockDriverState **file; 177661007b31SStefan Hajnoczi int64_t ret; 177761007b31SStefan Hajnoczi bool done; 177861007b31SStefan Hajnoczi } BdrvCoGetBlockStatusData; 177961007b31SStefan Hajnoczi 1780f7cc69b3SManos Pitsidianakis int64_t coroutine_fn bdrv_co_get_block_status_from_file(BlockDriverState *bs, 1781f7cc69b3SManos Pitsidianakis int64_t sector_num, 1782f7cc69b3SManos Pitsidianakis int nb_sectors, 1783f7cc69b3SManos Pitsidianakis int *pnum, 1784f7cc69b3SManos Pitsidianakis BlockDriverState **file) 1785f7cc69b3SManos Pitsidianakis { 1786f7cc69b3SManos Pitsidianakis assert(bs->file && bs->file->bs); 1787f7cc69b3SManos Pitsidianakis *pnum = nb_sectors; 1788f7cc69b3SManos Pitsidianakis *file = bs->file->bs; 1789f7cc69b3SManos Pitsidianakis return BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID | 1790f7cc69b3SManos Pitsidianakis (sector_num << BDRV_SECTOR_BITS); 1791f7cc69b3SManos Pitsidianakis } 1792f7cc69b3SManos Pitsidianakis 1793f7cc69b3SManos Pitsidianakis int64_t coroutine_fn bdrv_co_get_block_status_from_backing(BlockDriverState *bs, 1794f7cc69b3SManos Pitsidianakis int64_t sector_num, 1795f7cc69b3SManos Pitsidianakis int nb_sectors, 1796f7cc69b3SManos Pitsidianakis int *pnum, 1797f7cc69b3SManos Pitsidianakis BlockDriverState **file) 1798f7cc69b3SManos Pitsidianakis { 1799f7cc69b3SManos Pitsidianakis assert(bs->backing && bs->backing->bs); 1800f7cc69b3SManos Pitsidianakis *pnum = nb_sectors; 1801f7cc69b3SManos Pitsidianakis *file = bs->backing->bs; 1802f7cc69b3SManos Pitsidianakis return BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID | 1803f7cc69b3SManos Pitsidianakis (sector_num << BDRV_SECTOR_BITS); 1804f7cc69b3SManos Pitsidianakis } 1805f7cc69b3SManos Pitsidianakis 180661007b31SStefan Hajnoczi /* 180761007b31SStefan Hajnoczi * Returns the allocation status of the specified sectors. 180861007b31SStefan Hajnoczi * Drivers not implementing the functionality are assumed to not support 180961007b31SStefan Hajnoczi * backing files, hence all their sectors are reported as allocated. 181061007b31SStefan Hajnoczi * 1811c9ce8c4dSEric Blake * If 'want_zero' is true, the caller is querying for mapping purposes, 1812c9ce8c4dSEric Blake * and the result should include BDRV_BLOCK_OFFSET_VALID and 1813c9ce8c4dSEric Blake * BDRV_BLOCK_ZERO where possible; otherwise, the result may omit those 1814c9ce8c4dSEric Blake * bits particularly if it allows for a larger value in 'pnum'. 1815c9ce8c4dSEric Blake * 1816fb0d8654SEric Blake * If 'sector_num' is beyond the end of the disk image the return value is 1817fb0d8654SEric Blake * BDRV_BLOCK_EOF and 'pnum' is set to 0. 181861007b31SStefan Hajnoczi * 181961007b31SStefan Hajnoczi * 'pnum' is set to the number of sectors (including and immediately following 182061007b31SStefan Hajnoczi * the specified sector) that are known to be in the same 182161007b31SStefan Hajnoczi * allocated/unallocated state. 182261007b31SStefan Hajnoczi * 182361007b31SStefan Hajnoczi * 'nb_sectors' is the max value 'pnum' should be set to. If nb_sectors goes 1824fb0d8654SEric Blake * beyond the end of the disk image it will be clamped; if 'pnum' is set to 1825fb0d8654SEric Blake * the end of the image, then the returned value will include BDRV_BLOCK_EOF. 182667a0fd2aSFam Zheng * 1827298a1665SEric Blake * If returned value is positive, BDRV_BLOCK_OFFSET_VALID bit is set, and 1828298a1665SEric Blake * 'file' is non-NULL, then '*file' points to the BDS which the sector range 1829298a1665SEric Blake * is allocated in. 183061007b31SStefan Hajnoczi */ 183161007b31SStefan Hajnoczi static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs, 1832c9ce8c4dSEric Blake bool want_zero, 183361007b31SStefan Hajnoczi int64_t sector_num, 183467a0fd2aSFam Zheng int nb_sectors, int *pnum, 183567a0fd2aSFam Zheng BlockDriverState **file) 183661007b31SStefan Hajnoczi { 183761007b31SStefan Hajnoczi int64_t total_sectors; 183861007b31SStefan Hajnoczi int64_t n; 183961007b31SStefan Hajnoczi int64_t ret, ret2; 1840298a1665SEric Blake BlockDriverState *local_file = NULL; 184161007b31SStefan Hajnoczi 1842298a1665SEric Blake assert(pnum); 1843298a1665SEric Blake *pnum = 0; 184461007b31SStefan Hajnoczi total_sectors = bdrv_nb_sectors(bs); 184561007b31SStefan Hajnoczi if (total_sectors < 0) { 1846298a1665SEric Blake ret = total_sectors; 1847298a1665SEric Blake goto early_out; 184861007b31SStefan Hajnoczi } 184961007b31SStefan Hajnoczi 185061007b31SStefan Hajnoczi if (sector_num >= total_sectors) { 1851298a1665SEric Blake ret = BDRV_BLOCK_EOF; 1852298a1665SEric Blake goto early_out; 185361007b31SStefan Hajnoczi } 18549cdcfd9fSEric Blake if (!nb_sectors) { 1855298a1665SEric Blake ret = 0; 1856298a1665SEric Blake goto early_out; 18579cdcfd9fSEric Blake } 185861007b31SStefan Hajnoczi 185961007b31SStefan Hajnoczi n = total_sectors - sector_num; 186061007b31SStefan Hajnoczi if (n < nb_sectors) { 186161007b31SStefan Hajnoczi nb_sectors = n; 186261007b31SStefan Hajnoczi } 186361007b31SStefan Hajnoczi 186461007b31SStefan Hajnoczi if (!bs->drv->bdrv_co_get_block_status) { 186561007b31SStefan Hajnoczi *pnum = nb_sectors; 186661007b31SStefan Hajnoczi ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED; 1867fb0d8654SEric Blake if (sector_num + nb_sectors == total_sectors) { 1868fb0d8654SEric Blake ret |= BDRV_BLOCK_EOF; 1869fb0d8654SEric Blake } 187061007b31SStefan Hajnoczi if (bs->drv->protocol_name) { 187161007b31SStefan Hajnoczi ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE); 1872298a1665SEric Blake local_file = bs; 187361007b31SStefan Hajnoczi } 1874298a1665SEric Blake goto early_out; 187561007b31SStefan Hajnoczi } 187661007b31SStefan Hajnoczi 187799723548SPaolo Bonzini bdrv_inc_in_flight(bs); 187867a0fd2aSFam Zheng ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum, 1879298a1665SEric Blake &local_file); 188061007b31SStefan Hajnoczi if (ret < 0) { 188161007b31SStefan Hajnoczi *pnum = 0; 188299723548SPaolo Bonzini goto out; 188361007b31SStefan Hajnoczi } 188461007b31SStefan Hajnoczi 188561007b31SStefan Hajnoczi if (ret & BDRV_BLOCK_RAW) { 1886298a1665SEric Blake assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file); 1887c9ce8c4dSEric Blake ret = bdrv_co_get_block_status(local_file, want_zero, 1888c9ce8c4dSEric Blake ret >> BDRV_SECTOR_BITS, 1889298a1665SEric Blake *pnum, pnum, &local_file); 189099723548SPaolo Bonzini goto out; 189161007b31SStefan Hajnoczi } 189261007b31SStefan Hajnoczi 189361007b31SStefan Hajnoczi if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) { 189461007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ALLOCATED; 1895c9ce8c4dSEric Blake } else if (want_zero) { 189661007b31SStefan Hajnoczi if (bdrv_unallocated_blocks_are_zero(bs)) { 189761007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 1898760e0063SKevin Wolf } else if (bs->backing) { 1899760e0063SKevin Wolf BlockDriverState *bs2 = bs->backing->bs; 190061007b31SStefan Hajnoczi int64_t nb_sectors2 = bdrv_nb_sectors(bs2); 1901c9ce8c4dSEric Blake 190261007b31SStefan Hajnoczi if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) { 190361007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 190461007b31SStefan Hajnoczi } 190561007b31SStefan Hajnoczi } 190661007b31SStefan Hajnoczi } 190761007b31SStefan Hajnoczi 1908c9ce8c4dSEric Blake if (want_zero && local_file && local_file != bs && 190961007b31SStefan Hajnoczi (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) && 191061007b31SStefan Hajnoczi (ret & BDRV_BLOCK_OFFSET_VALID)) { 191161007b31SStefan Hajnoczi int file_pnum; 191261007b31SStefan Hajnoczi 1913c9ce8c4dSEric Blake ret2 = bdrv_co_get_block_status(local_file, want_zero, 1914c9ce8c4dSEric Blake ret >> BDRV_SECTOR_BITS, 1915298a1665SEric Blake *pnum, &file_pnum, NULL); 191661007b31SStefan Hajnoczi if (ret2 >= 0) { 191761007b31SStefan Hajnoczi /* Ignore errors. This is just providing extra information, it 191861007b31SStefan Hajnoczi * is useful but not necessary. 191961007b31SStefan Hajnoczi */ 1920c61e684eSEric Blake if (ret2 & BDRV_BLOCK_EOF && 1921c61e684eSEric Blake (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) { 1922c61e684eSEric Blake /* 1923c61e684eSEric Blake * It is valid for the format block driver to read 1924c61e684eSEric Blake * beyond the end of the underlying file's current 1925c61e684eSEric Blake * size; such areas read as zero. 1926c61e684eSEric Blake */ 192761007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 192861007b31SStefan Hajnoczi } else { 192961007b31SStefan Hajnoczi /* Limit request to the range reported by the protocol driver */ 193061007b31SStefan Hajnoczi *pnum = file_pnum; 193161007b31SStefan Hajnoczi ret |= (ret2 & BDRV_BLOCK_ZERO); 193261007b31SStefan Hajnoczi } 193361007b31SStefan Hajnoczi } 193461007b31SStefan Hajnoczi } 193561007b31SStefan Hajnoczi 193699723548SPaolo Bonzini out: 193799723548SPaolo Bonzini bdrv_dec_in_flight(bs); 1938fb0d8654SEric Blake if (ret >= 0 && sector_num + *pnum == total_sectors) { 1939fb0d8654SEric Blake ret |= BDRV_BLOCK_EOF; 1940fb0d8654SEric Blake } 1941298a1665SEric Blake early_out: 1942298a1665SEric Blake if (file) { 1943298a1665SEric Blake *file = local_file; 1944298a1665SEric Blake } 194561007b31SStefan Hajnoczi return ret; 194661007b31SStefan Hajnoczi } 194761007b31SStefan Hajnoczi 1948ba3f0e25SFam Zheng static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs, 1949ba3f0e25SFam Zheng BlockDriverState *base, 1950c9ce8c4dSEric Blake bool want_zero, 1951ba3f0e25SFam Zheng int64_t sector_num, 1952ba3f0e25SFam Zheng int nb_sectors, 195367a0fd2aSFam Zheng int *pnum, 195467a0fd2aSFam Zheng BlockDriverState **file) 1955ba3f0e25SFam Zheng { 1956ba3f0e25SFam Zheng BlockDriverState *p; 1957ba3f0e25SFam Zheng int64_t ret = 0; 1958c61e684eSEric Blake bool first = true; 1959ba3f0e25SFam Zheng 1960ba3f0e25SFam Zheng assert(bs != base); 1961760e0063SKevin Wolf for (p = bs; p != base; p = backing_bs(p)) { 1962c9ce8c4dSEric Blake ret = bdrv_co_get_block_status(p, want_zero, sector_num, nb_sectors, 1963c9ce8c4dSEric Blake pnum, file); 1964c61e684eSEric Blake if (ret < 0) { 1965c61e684eSEric Blake break; 1966c61e684eSEric Blake } 1967c61e684eSEric Blake if (ret & BDRV_BLOCK_ZERO && ret & BDRV_BLOCK_EOF && !first) { 1968c61e684eSEric Blake /* 1969c61e684eSEric Blake * Reading beyond the end of the file continues to read 1970c61e684eSEric Blake * zeroes, but we can only widen the result to the 1971c61e684eSEric Blake * unallocated length we learned from an earlier 1972c61e684eSEric Blake * iteration. 1973c61e684eSEric Blake */ 1974c61e684eSEric Blake *pnum = nb_sectors; 1975c61e684eSEric Blake } 1976c61e684eSEric Blake if (ret & (BDRV_BLOCK_ZERO | BDRV_BLOCK_DATA)) { 1977ba3f0e25SFam Zheng break; 1978ba3f0e25SFam Zheng } 1979ba3f0e25SFam Zheng /* [sector_num, pnum] unallocated on this layer, which could be only 1980ba3f0e25SFam Zheng * the first part of [sector_num, nb_sectors]. */ 1981ba3f0e25SFam Zheng nb_sectors = MIN(nb_sectors, *pnum); 1982c61e684eSEric Blake first = false; 1983ba3f0e25SFam Zheng } 1984ba3f0e25SFam Zheng return ret; 1985ba3f0e25SFam Zheng } 1986ba3f0e25SFam Zheng 1987ba3f0e25SFam Zheng /* Coroutine wrapper for bdrv_get_block_status_above() */ 1988ba3f0e25SFam Zheng static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque) 198961007b31SStefan Hajnoczi { 199061007b31SStefan Hajnoczi BdrvCoGetBlockStatusData *data = opaque; 199161007b31SStefan Hajnoczi 1992ba3f0e25SFam Zheng data->ret = bdrv_co_get_block_status_above(data->bs, data->base, 1993c9ce8c4dSEric Blake data->want_zero, 1994ba3f0e25SFam Zheng data->sector_num, 1995ba3f0e25SFam Zheng data->nb_sectors, 199667a0fd2aSFam Zheng data->pnum, 199767a0fd2aSFam Zheng data->file); 199861007b31SStefan Hajnoczi data->done = true; 199961007b31SStefan Hajnoczi } 200061007b31SStefan Hajnoczi 200161007b31SStefan Hajnoczi /* 2002ba3f0e25SFam Zheng * Synchronous wrapper around bdrv_co_get_block_status_above(). 200361007b31SStefan Hajnoczi * 2004ba3f0e25SFam Zheng * See bdrv_co_get_block_status_above() for details. 200561007b31SStefan Hajnoczi */ 2006c9ce8c4dSEric Blake static int64_t bdrv_common_block_status_above(BlockDriverState *bs, 2007ba3f0e25SFam Zheng BlockDriverState *base, 2008c9ce8c4dSEric Blake bool want_zero, 2009ba3f0e25SFam Zheng int64_t sector_num, 201067a0fd2aSFam Zheng int nb_sectors, int *pnum, 201167a0fd2aSFam Zheng BlockDriverState **file) 201261007b31SStefan Hajnoczi { 201361007b31SStefan Hajnoczi Coroutine *co; 201461007b31SStefan Hajnoczi BdrvCoGetBlockStatusData data = { 201561007b31SStefan Hajnoczi .bs = bs, 2016ba3f0e25SFam Zheng .base = base, 2017c9ce8c4dSEric Blake .want_zero = want_zero, 201861007b31SStefan Hajnoczi .sector_num = sector_num, 201961007b31SStefan Hajnoczi .nb_sectors = nb_sectors, 202061007b31SStefan Hajnoczi .pnum = pnum, 2021c9ce8c4dSEric Blake .file = file, 202261007b31SStefan Hajnoczi .done = false, 202361007b31SStefan Hajnoczi }; 202461007b31SStefan Hajnoczi 202561007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 202661007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 2027ba3f0e25SFam Zheng bdrv_get_block_status_above_co_entry(&data); 202861007b31SStefan Hajnoczi } else { 20290b8b8753SPaolo Bonzini co = qemu_coroutine_create(bdrv_get_block_status_above_co_entry, 20300b8b8753SPaolo Bonzini &data); 2031e92f0e19SFam Zheng bdrv_coroutine_enter(bs, co); 203288b062c2SPaolo Bonzini BDRV_POLL_WHILE(bs, !data.done); 203361007b31SStefan Hajnoczi } 203461007b31SStefan Hajnoczi return data.ret; 203561007b31SStefan Hajnoczi } 203661007b31SStefan Hajnoczi 2037c9ce8c4dSEric Blake int64_t bdrv_get_block_status_above(BlockDriverState *bs, 2038c9ce8c4dSEric Blake BlockDriverState *base, 2039c9ce8c4dSEric Blake int64_t sector_num, 2040c9ce8c4dSEric Blake int nb_sectors, int *pnum, 2041c9ce8c4dSEric Blake BlockDriverState **file) 2042c9ce8c4dSEric Blake { 2043c9ce8c4dSEric Blake return bdrv_common_block_status_above(bs, base, true, sector_num, 2044c9ce8c4dSEric Blake nb_sectors, pnum, file); 2045c9ce8c4dSEric Blake } 2046c9ce8c4dSEric Blake 2047*237d78f8SEric Blake int bdrv_block_status(BlockDriverState *bs, int64_t offset, int64_t bytes, 2048*237d78f8SEric Blake int64_t *pnum, int64_t *map, BlockDriverState **file) 2049ba3f0e25SFam Zheng { 2050*237d78f8SEric Blake int64_t ret; 2051*237d78f8SEric Blake int n; 2052*237d78f8SEric Blake 2053*237d78f8SEric Blake assert(QEMU_IS_ALIGNED(offset | bytes, BDRV_SECTOR_SIZE)); 2054*237d78f8SEric Blake assert(pnum); 2055*237d78f8SEric Blake /* 2056*237d78f8SEric Blake * The contract allows us to return pnum smaller than bytes, even 2057*237d78f8SEric Blake * if the next query would see the same status; we truncate the 2058*237d78f8SEric Blake * request to avoid overflowing the driver's 32-bit interface. 2059*237d78f8SEric Blake */ 2060*237d78f8SEric Blake bytes = MIN(bytes, BDRV_REQUEST_MAX_BYTES); 2061*237d78f8SEric Blake ret = bdrv_get_block_status_above(bs, backing_bs(bs), 2062*237d78f8SEric Blake offset >> BDRV_SECTOR_BITS, 2063*237d78f8SEric Blake bytes >> BDRV_SECTOR_BITS, &n, file); 2064*237d78f8SEric Blake if (ret < 0) { 2065*237d78f8SEric Blake assert(INT_MIN <= ret); 2066*237d78f8SEric Blake *pnum = 0; 2067*237d78f8SEric Blake return ret; 2068*237d78f8SEric Blake } 2069*237d78f8SEric Blake *pnum = n * BDRV_SECTOR_SIZE; 2070*237d78f8SEric Blake if (map) { 2071*237d78f8SEric Blake *map = ret & BDRV_BLOCK_OFFSET_MASK; 2072*237d78f8SEric Blake } else { 2073*237d78f8SEric Blake ret &= ~BDRV_BLOCK_OFFSET_VALID; 2074*237d78f8SEric Blake } 2075*237d78f8SEric Blake return ret & ~BDRV_BLOCK_OFFSET_MASK; 2076ba3f0e25SFam Zheng } 2077ba3f0e25SFam Zheng 2078d6a644bbSEric Blake int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t offset, 2079d6a644bbSEric Blake int64_t bytes, int64_t *pnum) 208061007b31SStefan Hajnoczi { 2081d6a644bbSEric Blake int64_t ret; 2082d6a644bbSEric Blake int psectors; 2083d6a644bbSEric Blake 2084d6a644bbSEric Blake assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE)); 2085d6a644bbSEric Blake assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE) && bytes < INT_MAX); 2086c9ce8c4dSEric Blake ret = bdrv_common_block_status_above(bs, backing_bs(bs), false, 2087c9ce8c4dSEric Blake offset >> BDRV_SECTOR_BITS, 2088c9ce8c4dSEric Blake bytes >> BDRV_SECTOR_BITS, &psectors, 2089298a1665SEric Blake NULL); 209061007b31SStefan Hajnoczi if (ret < 0) { 209161007b31SStefan Hajnoczi return ret; 209261007b31SStefan Hajnoczi } 2093d6a644bbSEric Blake if (pnum) { 2094d6a644bbSEric Blake *pnum = psectors * BDRV_SECTOR_SIZE; 2095d6a644bbSEric Blake } 209661007b31SStefan Hajnoczi return !!(ret & BDRV_BLOCK_ALLOCATED); 209761007b31SStefan Hajnoczi } 209861007b31SStefan Hajnoczi 209961007b31SStefan Hajnoczi /* 210061007b31SStefan Hajnoczi * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP] 210161007b31SStefan Hajnoczi * 210251b0a488SEric Blake * Return true if (a prefix of) the given range is allocated in any image 210351b0a488SEric Blake * between BASE and TOP (inclusive). BASE can be NULL to check if the given 210451b0a488SEric Blake * offset is allocated in any image of the chain. Return false otherwise, 2105d6a644bbSEric Blake * or negative errno on failure. 210661007b31SStefan Hajnoczi * 210751b0a488SEric Blake * 'pnum' is set to the number of bytes (including and immediately 210851b0a488SEric Blake * following the specified offset) that are known to be in the same 210951b0a488SEric Blake * allocated/unallocated state. Note that a subsequent call starting 211051b0a488SEric Blake * at 'offset + *pnum' may return the same allocation status (in other 211151b0a488SEric Blake * words, the result is not necessarily the maximum possible range); 211251b0a488SEric Blake * but 'pnum' will only be 0 when end of file is reached. 211361007b31SStefan Hajnoczi * 211461007b31SStefan Hajnoczi */ 211561007b31SStefan Hajnoczi int bdrv_is_allocated_above(BlockDriverState *top, 211661007b31SStefan Hajnoczi BlockDriverState *base, 211751b0a488SEric Blake int64_t offset, int64_t bytes, int64_t *pnum) 211861007b31SStefan Hajnoczi { 211961007b31SStefan Hajnoczi BlockDriverState *intermediate; 212051b0a488SEric Blake int ret; 212151b0a488SEric Blake int64_t n = bytes; 212261007b31SStefan Hajnoczi 212361007b31SStefan Hajnoczi intermediate = top; 212461007b31SStefan Hajnoczi while (intermediate && intermediate != base) { 2125d6a644bbSEric Blake int64_t pnum_inter; 2126c00716beSEric Blake int64_t size_inter; 2127d6a644bbSEric Blake 212851b0a488SEric Blake ret = bdrv_is_allocated(intermediate, offset, bytes, &pnum_inter); 212961007b31SStefan Hajnoczi if (ret < 0) { 213061007b31SStefan Hajnoczi return ret; 2131d6a644bbSEric Blake } 2132d6a644bbSEric Blake if (ret) { 213351b0a488SEric Blake *pnum = pnum_inter; 213461007b31SStefan Hajnoczi return 1; 213561007b31SStefan Hajnoczi } 213661007b31SStefan Hajnoczi 213751b0a488SEric Blake size_inter = bdrv_getlength(intermediate); 2138c00716beSEric Blake if (size_inter < 0) { 2139c00716beSEric Blake return size_inter; 2140c00716beSEric Blake } 214151b0a488SEric Blake if (n > pnum_inter && 214251b0a488SEric Blake (intermediate == top || offset + pnum_inter < size_inter)) { 214351b0a488SEric Blake n = pnum_inter; 214461007b31SStefan Hajnoczi } 214561007b31SStefan Hajnoczi 2146760e0063SKevin Wolf intermediate = backing_bs(intermediate); 214761007b31SStefan Hajnoczi } 214861007b31SStefan Hajnoczi 214961007b31SStefan Hajnoczi *pnum = n; 215061007b31SStefan Hajnoczi return 0; 215161007b31SStefan Hajnoczi } 215261007b31SStefan Hajnoczi 21531a8ae822SKevin Wolf typedef struct BdrvVmstateCo { 21541a8ae822SKevin Wolf BlockDriverState *bs; 21551a8ae822SKevin Wolf QEMUIOVector *qiov; 21561a8ae822SKevin Wolf int64_t pos; 21571a8ae822SKevin Wolf bool is_read; 21581a8ae822SKevin Wolf int ret; 21591a8ae822SKevin Wolf } BdrvVmstateCo; 21601a8ae822SKevin Wolf 21611a8ae822SKevin Wolf static int coroutine_fn 21621a8ae822SKevin Wolf bdrv_co_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos, 21631a8ae822SKevin Wolf bool is_read) 21641a8ae822SKevin Wolf { 21651a8ae822SKevin Wolf BlockDriver *drv = bs->drv; 2166dc88a467SStefan Hajnoczi int ret = -ENOTSUP; 2167dc88a467SStefan Hajnoczi 2168dc88a467SStefan Hajnoczi bdrv_inc_in_flight(bs); 21691a8ae822SKevin Wolf 21701a8ae822SKevin Wolf if (!drv) { 2171dc88a467SStefan Hajnoczi ret = -ENOMEDIUM; 21721a8ae822SKevin Wolf } else if (drv->bdrv_load_vmstate) { 2173dc88a467SStefan Hajnoczi if (is_read) { 2174dc88a467SStefan Hajnoczi ret = drv->bdrv_load_vmstate(bs, qiov, pos); 2175dc88a467SStefan Hajnoczi } else { 2176dc88a467SStefan Hajnoczi ret = drv->bdrv_save_vmstate(bs, qiov, pos); 2177dc88a467SStefan Hajnoczi } 21781a8ae822SKevin Wolf } else if (bs->file) { 2179dc88a467SStefan Hajnoczi ret = bdrv_co_rw_vmstate(bs->file->bs, qiov, pos, is_read); 21801a8ae822SKevin Wolf } 21811a8ae822SKevin Wolf 2182dc88a467SStefan Hajnoczi bdrv_dec_in_flight(bs); 2183dc88a467SStefan Hajnoczi return ret; 21841a8ae822SKevin Wolf } 21851a8ae822SKevin Wolf 21861a8ae822SKevin Wolf static void coroutine_fn bdrv_co_rw_vmstate_entry(void *opaque) 21871a8ae822SKevin Wolf { 21881a8ae822SKevin Wolf BdrvVmstateCo *co = opaque; 21891a8ae822SKevin Wolf co->ret = bdrv_co_rw_vmstate(co->bs, co->qiov, co->pos, co->is_read); 21901a8ae822SKevin Wolf } 21911a8ae822SKevin Wolf 21921a8ae822SKevin Wolf static inline int 21931a8ae822SKevin Wolf bdrv_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos, 21941a8ae822SKevin Wolf bool is_read) 21951a8ae822SKevin Wolf { 21961a8ae822SKevin Wolf if (qemu_in_coroutine()) { 21971a8ae822SKevin Wolf return bdrv_co_rw_vmstate(bs, qiov, pos, is_read); 21981a8ae822SKevin Wolf } else { 21991a8ae822SKevin Wolf BdrvVmstateCo data = { 22001a8ae822SKevin Wolf .bs = bs, 22011a8ae822SKevin Wolf .qiov = qiov, 22021a8ae822SKevin Wolf .pos = pos, 22031a8ae822SKevin Wolf .is_read = is_read, 22041a8ae822SKevin Wolf .ret = -EINPROGRESS, 22051a8ae822SKevin Wolf }; 22060b8b8753SPaolo Bonzini Coroutine *co = qemu_coroutine_create(bdrv_co_rw_vmstate_entry, &data); 22071a8ae822SKevin Wolf 2208e92f0e19SFam Zheng bdrv_coroutine_enter(bs, co); 2209ea17c9d2SStefan Hajnoczi BDRV_POLL_WHILE(bs, data.ret == -EINPROGRESS); 22101a8ae822SKevin Wolf return data.ret; 22111a8ae822SKevin Wolf } 22121a8ae822SKevin Wolf } 22131a8ae822SKevin Wolf 221461007b31SStefan Hajnoczi int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf, 221561007b31SStefan Hajnoczi int64_t pos, int size) 221661007b31SStefan Hajnoczi { 221761007b31SStefan Hajnoczi QEMUIOVector qiov; 221861007b31SStefan Hajnoczi struct iovec iov = { 221961007b31SStefan Hajnoczi .iov_base = (void *) buf, 222061007b31SStefan Hajnoczi .iov_len = size, 222161007b31SStefan Hajnoczi }; 2222b433d942SKevin Wolf int ret; 222361007b31SStefan Hajnoczi 222461007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 2225b433d942SKevin Wolf 2226b433d942SKevin Wolf ret = bdrv_writev_vmstate(bs, &qiov, pos); 2227b433d942SKevin Wolf if (ret < 0) { 2228b433d942SKevin Wolf return ret; 2229b433d942SKevin Wolf } 2230b433d942SKevin Wolf 2231b433d942SKevin Wolf return size; 223261007b31SStefan Hajnoczi } 223361007b31SStefan Hajnoczi 223461007b31SStefan Hajnoczi int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 223561007b31SStefan Hajnoczi { 22361a8ae822SKevin Wolf return bdrv_rw_vmstate(bs, qiov, pos, false); 223761007b31SStefan Hajnoczi } 223861007b31SStefan Hajnoczi 223961007b31SStefan Hajnoczi int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf, 224061007b31SStefan Hajnoczi int64_t pos, int size) 224161007b31SStefan Hajnoczi { 22425ddda0b8SKevin Wolf QEMUIOVector qiov; 22435ddda0b8SKevin Wolf struct iovec iov = { 22445ddda0b8SKevin Wolf .iov_base = buf, 22455ddda0b8SKevin Wolf .iov_len = size, 22465ddda0b8SKevin Wolf }; 2247b433d942SKevin Wolf int ret; 22485ddda0b8SKevin Wolf 22495ddda0b8SKevin Wolf qemu_iovec_init_external(&qiov, &iov, 1); 2250b433d942SKevin Wolf ret = bdrv_readv_vmstate(bs, &qiov, pos); 2251b433d942SKevin Wolf if (ret < 0) { 2252b433d942SKevin Wolf return ret; 2253b433d942SKevin Wolf } 2254b433d942SKevin Wolf 2255b433d942SKevin Wolf return size; 22565ddda0b8SKevin Wolf } 22575ddda0b8SKevin Wolf 22585ddda0b8SKevin Wolf int bdrv_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 22595ddda0b8SKevin Wolf { 22601a8ae822SKevin Wolf return bdrv_rw_vmstate(bs, qiov, pos, true); 226161007b31SStefan Hajnoczi } 226261007b31SStefan Hajnoczi 226361007b31SStefan Hajnoczi /**************************************************************/ 226461007b31SStefan Hajnoczi /* async I/Os */ 226561007b31SStefan Hajnoczi 226661007b31SStefan Hajnoczi void bdrv_aio_cancel(BlockAIOCB *acb) 226761007b31SStefan Hajnoczi { 226861007b31SStefan Hajnoczi qemu_aio_ref(acb); 226961007b31SStefan Hajnoczi bdrv_aio_cancel_async(acb); 227061007b31SStefan Hajnoczi while (acb->refcnt > 1) { 227161007b31SStefan Hajnoczi if (acb->aiocb_info->get_aio_context) { 227261007b31SStefan Hajnoczi aio_poll(acb->aiocb_info->get_aio_context(acb), true); 227361007b31SStefan Hajnoczi } else if (acb->bs) { 22742f47da5fSPaolo Bonzini /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so 22752f47da5fSPaolo Bonzini * assert that we're not using an I/O thread. Thread-safe 22762f47da5fSPaolo Bonzini * code should use bdrv_aio_cancel_async exclusively. 22772f47da5fSPaolo Bonzini */ 22782f47da5fSPaolo Bonzini assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context()); 227961007b31SStefan Hajnoczi aio_poll(bdrv_get_aio_context(acb->bs), true); 228061007b31SStefan Hajnoczi } else { 228161007b31SStefan Hajnoczi abort(); 228261007b31SStefan Hajnoczi } 228361007b31SStefan Hajnoczi } 228461007b31SStefan Hajnoczi qemu_aio_unref(acb); 228561007b31SStefan Hajnoczi } 228661007b31SStefan Hajnoczi 228761007b31SStefan Hajnoczi /* Async version of aio cancel. The caller is not blocked if the acb implements 228861007b31SStefan Hajnoczi * cancel_async, otherwise we do nothing and let the request normally complete. 228961007b31SStefan Hajnoczi * In either case the completion callback must be called. */ 229061007b31SStefan Hajnoczi void bdrv_aio_cancel_async(BlockAIOCB *acb) 229161007b31SStefan Hajnoczi { 229261007b31SStefan Hajnoczi if (acb->aiocb_info->cancel_async) { 229361007b31SStefan Hajnoczi acb->aiocb_info->cancel_async(acb); 229461007b31SStefan Hajnoczi } 229561007b31SStefan Hajnoczi } 229661007b31SStefan Hajnoczi 229761007b31SStefan Hajnoczi /**************************************************************/ 229861007b31SStefan Hajnoczi /* Coroutine block device emulation */ 229961007b31SStefan Hajnoczi 2300e293b7a3SKevin Wolf typedef struct FlushCo { 2301e293b7a3SKevin Wolf BlockDriverState *bs; 2302e293b7a3SKevin Wolf int ret; 2303e293b7a3SKevin Wolf } FlushCo; 2304e293b7a3SKevin Wolf 2305e293b7a3SKevin Wolf 230661007b31SStefan Hajnoczi static void coroutine_fn bdrv_flush_co_entry(void *opaque) 230761007b31SStefan Hajnoczi { 2308e293b7a3SKevin Wolf FlushCo *rwco = opaque; 230961007b31SStefan Hajnoczi 231061007b31SStefan Hajnoczi rwco->ret = bdrv_co_flush(rwco->bs); 231161007b31SStefan Hajnoczi } 231261007b31SStefan Hajnoczi 231361007b31SStefan Hajnoczi int coroutine_fn bdrv_co_flush(BlockDriverState *bs) 231461007b31SStefan Hajnoczi { 231549ca6259SFam Zheng int current_gen; 231649ca6259SFam Zheng int ret = 0; 231761007b31SStefan Hajnoczi 231899723548SPaolo Bonzini bdrv_inc_in_flight(bs); 2319c32b82afSPavel Dovgalyuk 2320e914404eSFam Zheng if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs) || 232149ca6259SFam Zheng bdrv_is_sg(bs)) { 232249ca6259SFam Zheng goto early_exit; 232349ca6259SFam Zheng } 232449ca6259SFam Zheng 23253783fa3dSPaolo Bonzini qemu_co_mutex_lock(&bs->reqs_lock); 232647fec599SPaolo Bonzini current_gen = atomic_read(&bs->write_gen); 23273ff2f67aSEvgeny Yakovlev 23283ff2f67aSEvgeny Yakovlev /* Wait until any previous flushes are completed */ 232999723548SPaolo Bonzini while (bs->active_flush_req) { 23303783fa3dSPaolo Bonzini qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock); 23313ff2f67aSEvgeny Yakovlev } 23323ff2f67aSEvgeny Yakovlev 23333783fa3dSPaolo Bonzini /* Flushes reach this point in nondecreasing current_gen order. */ 233499723548SPaolo Bonzini bs->active_flush_req = true; 23353783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&bs->reqs_lock); 23363ff2f67aSEvgeny Yakovlev 2337c32b82afSPavel Dovgalyuk /* Write back all layers by calling one driver function */ 2338c32b82afSPavel Dovgalyuk if (bs->drv->bdrv_co_flush) { 2339c32b82afSPavel Dovgalyuk ret = bs->drv->bdrv_co_flush(bs); 2340c32b82afSPavel Dovgalyuk goto out; 2341c32b82afSPavel Dovgalyuk } 2342c32b82afSPavel Dovgalyuk 234361007b31SStefan Hajnoczi /* Write back cached data to the OS even with cache=unsafe */ 234461007b31SStefan Hajnoczi BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS); 234561007b31SStefan Hajnoczi if (bs->drv->bdrv_co_flush_to_os) { 234661007b31SStefan Hajnoczi ret = bs->drv->bdrv_co_flush_to_os(bs); 234761007b31SStefan Hajnoczi if (ret < 0) { 2348cdb5e315SFam Zheng goto out; 234961007b31SStefan Hajnoczi } 235061007b31SStefan Hajnoczi } 235161007b31SStefan Hajnoczi 235261007b31SStefan Hajnoczi /* But don't actually force it to the disk with cache=unsafe */ 235361007b31SStefan Hajnoczi if (bs->open_flags & BDRV_O_NO_FLUSH) { 235461007b31SStefan Hajnoczi goto flush_parent; 235561007b31SStefan Hajnoczi } 235661007b31SStefan Hajnoczi 23573ff2f67aSEvgeny Yakovlev /* Check if we really need to flush anything */ 23583ff2f67aSEvgeny Yakovlev if (bs->flushed_gen == current_gen) { 23593ff2f67aSEvgeny Yakovlev goto flush_parent; 23603ff2f67aSEvgeny Yakovlev } 23613ff2f67aSEvgeny Yakovlev 236261007b31SStefan Hajnoczi BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK); 236361007b31SStefan Hajnoczi if (bs->drv->bdrv_co_flush_to_disk) { 236461007b31SStefan Hajnoczi ret = bs->drv->bdrv_co_flush_to_disk(bs); 236561007b31SStefan Hajnoczi } else if (bs->drv->bdrv_aio_flush) { 236661007b31SStefan Hajnoczi BlockAIOCB *acb; 236761007b31SStefan Hajnoczi CoroutineIOCompletion co = { 236861007b31SStefan Hajnoczi .coroutine = qemu_coroutine_self(), 236961007b31SStefan Hajnoczi }; 237061007b31SStefan Hajnoczi 237161007b31SStefan Hajnoczi acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co); 237261007b31SStefan Hajnoczi if (acb == NULL) { 237361007b31SStefan Hajnoczi ret = -EIO; 237461007b31SStefan Hajnoczi } else { 237561007b31SStefan Hajnoczi qemu_coroutine_yield(); 237661007b31SStefan Hajnoczi ret = co.ret; 237761007b31SStefan Hajnoczi } 237861007b31SStefan Hajnoczi } else { 237961007b31SStefan Hajnoczi /* 238061007b31SStefan Hajnoczi * Some block drivers always operate in either writethrough or unsafe 238161007b31SStefan Hajnoczi * mode and don't support bdrv_flush therefore. Usually qemu doesn't 238261007b31SStefan Hajnoczi * know how the server works (because the behaviour is hardcoded or 238361007b31SStefan Hajnoczi * depends on server-side configuration), so we can't ensure that 238461007b31SStefan Hajnoczi * everything is safe on disk. Returning an error doesn't work because 238561007b31SStefan Hajnoczi * that would break guests even if the server operates in writethrough 238661007b31SStefan Hajnoczi * mode. 238761007b31SStefan Hajnoczi * 238861007b31SStefan Hajnoczi * Let's hope the user knows what he's doing. 238961007b31SStefan Hajnoczi */ 239061007b31SStefan Hajnoczi ret = 0; 239161007b31SStefan Hajnoczi } 23923ff2f67aSEvgeny Yakovlev 239361007b31SStefan Hajnoczi if (ret < 0) { 2394cdb5e315SFam Zheng goto out; 239561007b31SStefan Hajnoczi } 239661007b31SStefan Hajnoczi 239761007b31SStefan Hajnoczi /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH 239861007b31SStefan Hajnoczi * in the case of cache=unsafe, so there are no useless flushes. 239961007b31SStefan Hajnoczi */ 240061007b31SStefan Hajnoczi flush_parent: 2401cdb5e315SFam Zheng ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0; 2402cdb5e315SFam Zheng out: 24033ff2f67aSEvgeny Yakovlev /* Notify any pending flushes that we have completed */ 2404e6af1e08SKevin Wolf if (ret == 0) { 24053ff2f67aSEvgeny Yakovlev bs->flushed_gen = current_gen; 2406e6af1e08SKevin Wolf } 24073783fa3dSPaolo Bonzini 24083783fa3dSPaolo Bonzini qemu_co_mutex_lock(&bs->reqs_lock); 240999723548SPaolo Bonzini bs->active_flush_req = false; 2410156af3acSDenis V. Lunev /* Return value is ignored - it's ok if wait queue is empty */ 2411156af3acSDenis V. Lunev qemu_co_queue_next(&bs->flush_queue); 24123783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&bs->reqs_lock); 24133ff2f67aSEvgeny Yakovlev 241449ca6259SFam Zheng early_exit: 241599723548SPaolo Bonzini bdrv_dec_in_flight(bs); 2416cdb5e315SFam Zheng return ret; 241761007b31SStefan Hajnoczi } 241861007b31SStefan Hajnoczi 241961007b31SStefan Hajnoczi int bdrv_flush(BlockDriverState *bs) 242061007b31SStefan Hajnoczi { 242161007b31SStefan Hajnoczi Coroutine *co; 2422e293b7a3SKevin Wolf FlushCo flush_co = { 242361007b31SStefan Hajnoczi .bs = bs, 242461007b31SStefan Hajnoczi .ret = NOT_DONE, 242561007b31SStefan Hajnoczi }; 242661007b31SStefan Hajnoczi 242761007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 242861007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 2429e293b7a3SKevin Wolf bdrv_flush_co_entry(&flush_co); 243061007b31SStefan Hajnoczi } else { 24310b8b8753SPaolo Bonzini co = qemu_coroutine_create(bdrv_flush_co_entry, &flush_co); 2432e92f0e19SFam Zheng bdrv_coroutine_enter(bs, co); 243388b062c2SPaolo Bonzini BDRV_POLL_WHILE(bs, flush_co.ret == NOT_DONE); 243461007b31SStefan Hajnoczi } 243561007b31SStefan Hajnoczi 2436e293b7a3SKevin Wolf return flush_co.ret; 243761007b31SStefan Hajnoczi } 243861007b31SStefan Hajnoczi 243961007b31SStefan Hajnoczi typedef struct DiscardCo { 244061007b31SStefan Hajnoczi BlockDriverState *bs; 24410c51a893SEric Blake int64_t offset; 2442f5a5ca79SManos Pitsidianakis int bytes; 244361007b31SStefan Hajnoczi int ret; 244461007b31SStefan Hajnoczi } DiscardCo; 24450c51a893SEric Blake static void coroutine_fn bdrv_pdiscard_co_entry(void *opaque) 244661007b31SStefan Hajnoczi { 244761007b31SStefan Hajnoczi DiscardCo *rwco = opaque; 244861007b31SStefan Hajnoczi 2449f5a5ca79SManos Pitsidianakis rwco->ret = bdrv_co_pdiscard(rwco->bs, rwco->offset, rwco->bytes); 245061007b31SStefan Hajnoczi } 245161007b31SStefan Hajnoczi 24529f1963b3SEric Blake int coroutine_fn bdrv_co_pdiscard(BlockDriverState *bs, int64_t offset, 2453f5a5ca79SManos Pitsidianakis int bytes) 245461007b31SStefan Hajnoczi { 2455b1066c87SFam Zheng BdrvTrackedRequest req; 24569f1963b3SEric Blake int max_pdiscard, ret; 24573482b9bcSEric Blake int head, tail, align; 245861007b31SStefan Hajnoczi 245961007b31SStefan Hajnoczi if (!bs->drv) { 246061007b31SStefan Hajnoczi return -ENOMEDIUM; 246161007b31SStefan Hajnoczi } 246261007b31SStefan Hajnoczi 2463d6883bc9SVladimir Sementsov-Ogievskiy if (bdrv_has_readonly_bitmaps(bs)) { 2464d6883bc9SVladimir Sementsov-Ogievskiy return -EPERM; 2465d6883bc9SVladimir Sementsov-Ogievskiy } 2466d6883bc9SVladimir Sementsov-Ogievskiy 2467f5a5ca79SManos Pitsidianakis ret = bdrv_check_byte_request(bs, offset, bytes); 246861007b31SStefan Hajnoczi if (ret < 0) { 246961007b31SStefan Hajnoczi return ret; 247061007b31SStefan Hajnoczi } else if (bs->read_only) { 2471eaf5fe2dSPaolo Bonzini return -EPERM; 247261007b31SStefan Hajnoczi } 247304c01a5cSKevin Wolf assert(!(bs->open_flags & BDRV_O_INACTIVE)); 247461007b31SStefan Hajnoczi 247561007b31SStefan Hajnoczi /* Do nothing if disabled. */ 247661007b31SStefan Hajnoczi if (!(bs->open_flags & BDRV_O_UNMAP)) { 247761007b31SStefan Hajnoczi return 0; 247861007b31SStefan Hajnoczi } 247961007b31SStefan Hajnoczi 248002aefe43SEric Blake if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) { 248161007b31SStefan Hajnoczi return 0; 248261007b31SStefan Hajnoczi } 248361007b31SStefan Hajnoczi 24843482b9bcSEric Blake /* Discard is advisory, but some devices track and coalesce 24853482b9bcSEric Blake * unaligned requests, so we must pass everything down rather than 24863482b9bcSEric Blake * round here. Still, most devices will just silently ignore 24873482b9bcSEric Blake * unaligned requests (by returning -ENOTSUP), so we must fragment 24883482b9bcSEric Blake * the request accordingly. */ 248902aefe43SEric Blake align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment); 2490b8d0a980SEric Blake assert(align % bs->bl.request_alignment == 0); 2491b8d0a980SEric Blake head = offset % align; 2492f5a5ca79SManos Pitsidianakis tail = (offset + bytes) % align; 24939f1963b3SEric Blake 249499723548SPaolo Bonzini bdrv_inc_in_flight(bs); 2495f5a5ca79SManos Pitsidianakis tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD); 249650824995SFam Zheng 2497ec050f77SDenis V. Lunev ret = notifier_with_return_list_notify(&bs->before_write_notifiers, &req); 2498ec050f77SDenis V. Lunev if (ret < 0) { 2499ec050f77SDenis V. Lunev goto out; 2500ec050f77SDenis V. Lunev } 2501ec050f77SDenis V. Lunev 25029f1963b3SEric Blake max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT_MAX), 25039f1963b3SEric Blake align); 25043482b9bcSEric Blake assert(max_pdiscard >= bs->bl.request_alignment); 25059f1963b3SEric Blake 2506f5a5ca79SManos Pitsidianakis while (bytes > 0) { 2507f5a5ca79SManos Pitsidianakis int num = bytes; 25083482b9bcSEric Blake 25093482b9bcSEric Blake if (head) { 25103482b9bcSEric Blake /* Make small requests to get to alignment boundaries. */ 2511f5a5ca79SManos Pitsidianakis num = MIN(bytes, align - head); 25123482b9bcSEric Blake if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) { 25133482b9bcSEric Blake num %= bs->bl.request_alignment; 25143482b9bcSEric Blake } 25153482b9bcSEric Blake head = (head + num) % align; 25163482b9bcSEric Blake assert(num < max_pdiscard); 25173482b9bcSEric Blake } else if (tail) { 25183482b9bcSEric Blake if (num > align) { 25193482b9bcSEric Blake /* Shorten the request to the last aligned cluster. */ 25203482b9bcSEric Blake num -= tail; 25213482b9bcSEric Blake } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) && 25223482b9bcSEric Blake tail > bs->bl.request_alignment) { 25233482b9bcSEric Blake tail %= bs->bl.request_alignment; 25243482b9bcSEric Blake num -= tail; 25253482b9bcSEric Blake } 25263482b9bcSEric Blake } 25273482b9bcSEric Blake /* limit request size */ 25283482b9bcSEric Blake if (num > max_pdiscard) { 25293482b9bcSEric Blake num = max_pdiscard; 25303482b9bcSEric Blake } 253161007b31SStefan Hajnoczi 253247a5486dSEric Blake if (bs->drv->bdrv_co_pdiscard) { 253347a5486dSEric Blake ret = bs->drv->bdrv_co_pdiscard(bs, offset, num); 253461007b31SStefan Hajnoczi } else { 253561007b31SStefan Hajnoczi BlockAIOCB *acb; 253661007b31SStefan Hajnoczi CoroutineIOCompletion co = { 253761007b31SStefan Hajnoczi .coroutine = qemu_coroutine_self(), 253861007b31SStefan Hajnoczi }; 253961007b31SStefan Hajnoczi 25404da444a0SEric Blake acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num, 254161007b31SStefan Hajnoczi bdrv_co_io_em_complete, &co); 254261007b31SStefan Hajnoczi if (acb == NULL) { 2543b1066c87SFam Zheng ret = -EIO; 2544b1066c87SFam Zheng goto out; 254561007b31SStefan Hajnoczi } else { 254661007b31SStefan Hajnoczi qemu_coroutine_yield(); 254761007b31SStefan Hajnoczi ret = co.ret; 254861007b31SStefan Hajnoczi } 254961007b31SStefan Hajnoczi } 255061007b31SStefan Hajnoczi if (ret && ret != -ENOTSUP) { 2551b1066c87SFam Zheng goto out; 255261007b31SStefan Hajnoczi } 255361007b31SStefan Hajnoczi 25549f1963b3SEric Blake offset += num; 2555f5a5ca79SManos Pitsidianakis bytes -= num; 255661007b31SStefan Hajnoczi } 2557b1066c87SFam Zheng ret = 0; 2558b1066c87SFam Zheng out: 255947fec599SPaolo Bonzini atomic_inc(&bs->write_gen); 25600fdf1a4fSEric Blake bdrv_set_dirty(bs, req.offset, req.bytes); 2561b1066c87SFam Zheng tracked_request_end(&req); 256299723548SPaolo Bonzini bdrv_dec_in_flight(bs); 2563b1066c87SFam Zheng return ret; 256461007b31SStefan Hajnoczi } 256561007b31SStefan Hajnoczi 2566f5a5ca79SManos Pitsidianakis int bdrv_pdiscard(BlockDriverState *bs, int64_t offset, int bytes) 256761007b31SStefan Hajnoczi { 256861007b31SStefan Hajnoczi Coroutine *co; 256961007b31SStefan Hajnoczi DiscardCo rwco = { 257061007b31SStefan Hajnoczi .bs = bs, 25710c51a893SEric Blake .offset = offset, 2572f5a5ca79SManos Pitsidianakis .bytes = bytes, 257361007b31SStefan Hajnoczi .ret = NOT_DONE, 257461007b31SStefan Hajnoczi }; 257561007b31SStefan Hajnoczi 257661007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 257761007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 25780c51a893SEric Blake bdrv_pdiscard_co_entry(&rwco); 257961007b31SStefan Hajnoczi } else { 25800c51a893SEric Blake co = qemu_coroutine_create(bdrv_pdiscard_co_entry, &rwco); 2581e92f0e19SFam Zheng bdrv_coroutine_enter(bs, co); 258288b062c2SPaolo Bonzini BDRV_POLL_WHILE(bs, rwco.ret == NOT_DONE); 258361007b31SStefan Hajnoczi } 258461007b31SStefan Hajnoczi 258561007b31SStefan Hajnoczi return rwco.ret; 258661007b31SStefan Hajnoczi } 258761007b31SStefan Hajnoczi 258848af776aSKevin Wolf int bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf) 258961007b31SStefan Hajnoczi { 259061007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 25915c5ae76aSFam Zheng CoroutineIOCompletion co = { 25925c5ae76aSFam Zheng .coroutine = qemu_coroutine_self(), 25935c5ae76aSFam Zheng }; 25945c5ae76aSFam Zheng BlockAIOCB *acb; 259561007b31SStefan Hajnoczi 259699723548SPaolo Bonzini bdrv_inc_in_flight(bs); 259716a389dcSKevin Wolf if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) { 25985c5ae76aSFam Zheng co.ret = -ENOTSUP; 25995c5ae76aSFam Zheng goto out; 26005c5ae76aSFam Zheng } 26015c5ae76aSFam Zheng 260216a389dcSKevin Wolf if (drv->bdrv_co_ioctl) { 260316a389dcSKevin Wolf co.ret = drv->bdrv_co_ioctl(bs, req, buf); 260416a389dcSKevin Wolf } else { 26055c5ae76aSFam Zheng acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co); 26065c5ae76aSFam Zheng if (!acb) { 2607c8a9fd80SFam Zheng co.ret = -ENOTSUP; 2608c8a9fd80SFam Zheng goto out; 26095c5ae76aSFam Zheng } 26105c5ae76aSFam Zheng qemu_coroutine_yield(); 261116a389dcSKevin Wolf } 26125c5ae76aSFam Zheng out: 261399723548SPaolo Bonzini bdrv_dec_in_flight(bs); 26145c5ae76aSFam Zheng return co.ret; 26155c5ae76aSFam Zheng } 26165c5ae76aSFam Zheng 261761007b31SStefan Hajnoczi void *qemu_blockalign(BlockDriverState *bs, size_t size) 261861007b31SStefan Hajnoczi { 261961007b31SStefan Hajnoczi return qemu_memalign(bdrv_opt_mem_align(bs), size); 262061007b31SStefan Hajnoczi } 262161007b31SStefan Hajnoczi 262261007b31SStefan Hajnoczi void *qemu_blockalign0(BlockDriverState *bs, size_t size) 262361007b31SStefan Hajnoczi { 262461007b31SStefan Hajnoczi return memset(qemu_blockalign(bs, size), 0, size); 262561007b31SStefan Hajnoczi } 262661007b31SStefan Hajnoczi 262761007b31SStefan Hajnoczi void *qemu_try_blockalign(BlockDriverState *bs, size_t size) 262861007b31SStefan Hajnoczi { 262961007b31SStefan Hajnoczi size_t align = bdrv_opt_mem_align(bs); 263061007b31SStefan Hajnoczi 263161007b31SStefan Hajnoczi /* Ensure that NULL is never returned on success */ 263261007b31SStefan Hajnoczi assert(align > 0); 263361007b31SStefan Hajnoczi if (size == 0) { 263461007b31SStefan Hajnoczi size = align; 263561007b31SStefan Hajnoczi } 263661007b31SStefan Hajnoczi 263761007b31SStefan Hajnoczi return qemu_try_memalign(align, size); 263861007b31SStefan Hajnoczi } 263961007b31SStefan Hajnoczi 264061007b31SStefan Hajnoczi void *qemu_try_blockalign0(BlockDriverState *bs, size_t size) 264161007b31SStefan Hajnoczi { 264261007b31SStefan Hajnoczi void *mem = qemu_try_blockalign(bs, size); 264361007b31SStefan Hajnoczi 264461007b31SStefan Hajnoczi if (mem) { 264561007b31SStefan Hajnoczi memset(mem, 0, size); 264661007b31SStefan Hajnoczi } 264761007b31SStefan Hajnoczi 264861007b31SStefan Hajnoczi return mem; 264961007b31SStefan Hajnoczi } 265061007b31SStefan Hajnoczi 265161007b31SStefan Hajnoczi /* 265261007b31SStefan Hajnoczi * Check if all memory in this vector is sector aligned. 265361007b31SStefan Hajnoczi */ 265461007b31SStefan Hajnoczi bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov) 265561007b31SStefan Hajnoczi { 265661007b31SStefan Hajnoczi int i; 26574196d2f0SDenis V. Lunev size_t alignment = bdrv_min_mem_align(bs); 265861007b31SStefan Hajnoczi 265961007b31SStefan Hajnoczi for (i = 0; i < qiov->niov; i++) { 266061007b31SStefan Hajnoczi if ((uintptr_t) qiov->iov[i].iov_base % alignment) { 266161007b31SStefan Hajnoczi return false; 266261007b31SStefan Hajnoczi } 266361007b31SStefan Hajnoczi if (qiov->iov[i].iov_len % alignment) { 266461007b31SStefan Hajnoczi return false; 266561007b31SStefan Hajnoczi } 266661007b31SStefan Hajnoczi } 266761007b31SStefan Hajnoczi 266861007b31SStefan Hajnoczi return true; 266961007b31SStefan Hajnoczi } 267061007b31SStefan Hajnoczi 267161007b31SStefan Hajnoczi void bdrv_add_before_write_notifier(BlockDriverState *bs, 267261007b31SStefan Hajnoczi NotifierWithReturn *notifier) 267361007b31SStefan Hajnoczi { 267461007b31SStefan Hajnoczi notifier_with_return_list_add(&bs->before_write_notifiers, notifier); 267561007b31SStefan Hajnoczi } 267661007b31SStefan Hajnoczi 267761007b31SStefan Hajnoczi void bdrv_io_plug(BlockDriverState *bs) 267861007b31SStefan Hajnoczi { 26796b98bd64SPaolo Bonzini BdrvChild *child; 26806b98bd64SPaolo Bonzini 26816b98bd64SPaolo Bonzini QLIST_FOREACH(child, &bs->children, next) { 26826b98bd64SPaolo Bonzini bdrv_io_plug(child->bs); 26836b98bd64SPaolo Bonzini } 26846b98bd64SPaolo Bonzini 2685850d54a2SPaolo Bonzini if (atomic_fetch_inc(&bs->io_plugged) == 0) { 268661007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 268761007b31SStefan Hajnoczi if (drv && drv->bdrv_io_plug) { 268861007b31SStefan Hajnoczi drv->bdrv_io_plug(bs); 26896b98bd64SPaolo Bonzini } 269061007b31SStefan Hajnoczi } 269161007b31SStefan Hajnoczi } 269261007b31SStefan Hajnoczi 269361007b31SStefan Hajnoczi void bdrv_io_unplug(BlockDriverState *bs) 269461007b31SStefan Hajnoczi { 26956b98bd64SPaolo Bonzini BdrvChild *child; 26966b98bd64SPaolo Bonzini 26976b98bd64SPaolo Bonzini assert(bs->io_plugged); 2698850d54a2SPaolo Bonzini if (atomic_fetch_dec(&bs->io_plugged) == 1) { 269961007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 270061007b31SStefan Hajnoczi if (drv && drv->bdrv_io_unplug) { 270161007b31SStefan Hajnoczi drv->bdrv_io_unplug(bs); 270261007b31SStefan Hajnoczi } 270361007b31SStefan Hajnoczi } 270461007b31SStefan Hajnoczi 27056b98bd64SPaolo Bonzini QLIST_FOREACH(child, &bs->children, next) { 27066b98bd64SPaolo Bonzini bdrv_io_unplug(child->bs); 27076b98bd64SPaolo Bonzini } 27086b98bd64SPaolo Bonzini } 2709