161007b31SStefan Hajnoczi /* 261007b31SStefan Hajnoczi * Block layer I/O functions 361007b31SStefan Hajnoczi * 461007b31SStefan Hajnoczi * Copyright (c) 2003 Fabrice Bellard 561007b31SStefan Hajnoczi * 661007b31SStefan Hajnoczi * Permission is hereby granted, free of charge, to any person obtaining a copy 761007b31SStefan Hajnoczi * of this software and associated documentation files (the "Software"), to deal 861007b31SStefan Hajnoczi * in the Software without restriction, including without limitation the rights 961007b31SStefan Hajnoczi * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 1061007b31SStefan Hajnoczi * copies of the Software, and to permit persons to whom the Software is 1161007b31SStefan Hajnoczi * furnished to do so, subject to the following conditions: 1261007b31SStefan Hajnoczi * 1361007b31SStefan Hajnoczi * The above copyright notice and this permission notice shall be included in 1461007b31SStefan Hajnoczi * all copies or substantial portions of the Software. 1561007b31SStefan Hajnoczi * 1661007b31SStefan Hajnoczi * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 1761007b31SStefan Hajnoczi * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 1861007b31SStefan Hajnoczi * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 1961007b31SStefan Hajnoczi * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 2061007b31SStefan Hajnoczi * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 2161007b31SStefan Hajnoczi * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 2261007b31SStefan Hajnoczi * THE SOFTWARE. 2361007b31SStefan Hajnoczi */ 2461007b31SStefan Hajnoczi 2580c71a24SPeter Maydell #include "qemu/osdep.h" 2661007b31SStefan Hajnoczi #include "trace.h" 277f0e9da6SMax Reitz #include "sysemu/block-backend.h" 2861007b31SStefan Hajnoczi #include "block/blockjob.h" 2961007b31SStefan Hajnoczi #include "block/block_int.h" 30f348b6d1SVeronia Bahaa #include "qemu/cutils.h" 31da34e65cSMarkus Armbruster #include "qapi/error.h" 32d49b6836SMarkus Armbruster #include "qemu/error-report.h" 3361007b31SStefan Hajnoczi 3461007b31SStefan Hajnoczi #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */ 3561007b31SStefan Hajnoczi 36b15404e0SEric Blake static BlockAIOCB *bdrv_co_aio_prw_vector(BdrvChild *child, 37b15404e0SEric Blake int64_t offset, 3861007b31SStefan Hajnoczi QEMUIOVector *qiov, 3961007b31SStefan Hajnoczi BdrvRequestFlags flags, 4061007b31SStefan Hajnoczi BlockCompletionFunc *cb, 4161007b31SStefan Hajnoczi void *opaque, 4261007b31SStefan Hajnoczi bool is_write); 4361007b31SStefan Hajnoczi static void coroutine_fn bdrv_co_do_rw(void *opaque); 44d05aa8bbSEric Blake static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 45d05aa8bbSEric Blake int64_t offset, int count, BdrvRequestFlags flags); 4661007b31SStefan Hajnoczi 47c2066af0SKevin Wolf static void bdrv_parent_drained_begin(BlockDriverState *bs) 4861007b31SStefan Hajnoczi { 49c2066af0SKevin Wolf BdrvChild *c; 5027ccdd52SKevin Wolf 51c2066af0SKevin Wolf QLIST_FOREACH(c, &bs->parents, next_parent) { 52c2066af0SKevin Wolf if (c->role->drained_begin) { 53c2066af0SKevin Wolf c->role->drained_begin(c); 54c2066af0SKevin Wolf } 55ce0f1412SPaolo Bonzini } 56ce0f1412SPaolo Bonzini } 57ce0f1412SPaolo Bonzini 58c2066af0SKevin Wolf static void bdrv_parent_drained_end(BlockDriverState *bs) 59ce0f1412SPaolo Bonzini { 60c2066af0SKevin Wolf BdrvChild *c; 6127ccdd52SKevin Wolf 62c2066af0SKevin Wolf QLIST_FOREACH(c, &bs->parents, next_parent) { 63c2066af0SKevin Wolf if (c->role->drained_end) { 64c2066af0SKevin Wolf c->role->drained_end(c); 6527ccdd52SKevin Wolf } 66c2066af0SKevin Wolf } 6761007b31SStefan Hajnoczi } 6861007b31SStefan Hajnoczi 69d9e0dfa2SEric Blake static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src) 70d9e0dfa2SEric Blake { 71d9e0dfa2SEric Blake dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer); 72d9e0dfa2SEric Blake dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer); 73d9e0dfa2SEric Blake dst->opt_mem_alignment = MAX(dst->opt_mem_alignment, 74d9e0dfa2SEric Blake src->opt_mem_alignment); 75d9e0dfa2SEric Blake dst->min_mem_alignment = MAX(dst->min_mem_alignment, 76d9e0dfa2SEric Blake src->min_mem_alignment); 77d9e0dfa2SEric Blake dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov); 78d9e0dfa2SEric Blake } 79d9e0dfa2SEric Blake 8061007b31SStefan Hajnoczi void bdrv_refresh_limits(BlockDriverState *bs, Error **errp) 8161007b31SStefan Hajnoczi { 8261007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 8361007b31SStefan Hajnoczi Error *local_err = NULL; 8461007b31SStefan Hajnoczi 8561007b31SStefan Hajnoczi memset(&bs->bl, 0, sizeof(bs->bl)); 8661007b31SStefan Hajnoczi 8761007b31SStefan Hajnoczi if (!drv) { 8861007b31SStefan Hajnoczi return; 8961007b31SStefan Hajnoczi } 9061007b31SStefan Hajnoczi 9179ba8c98SEric Blake /* Default alignment based on whether driver has byte interface */ 92a5b8dd2cSEric Blake bs->bl.request_alignment = drv->bdrv_co_preadv ? 1 : 512; 9379ba8c98SEric Blake 9461007b31SStefan Hajnoczi /* Take some limits from the children as a default */ 9561007b31SStefan Hajnoczi if (bs->file) { 969a4f4c31SKevin Wolf bdrv_refresh_limits(bs->file->bs, &local_err); 9761007b31SStefan Hajnoczi if (local_err) { 9861007b31SStefan Hajnoczi error_propagate(errp, local_err); 9961007b31SStefan Hajnoczi return; 10061007b31SStefan Hajnoczi } 101d9e0dfa2SEric Blake bdrv_merge_limits(&bs->bl, &bs->file->bs->bl); 10261007b31SStefan Hajnoczi } else { 1034196d2f0SDenis V. Lunev bs->bl.min_mem_alignment = 512; 104459b4e66SDenis V. Lunev bs->bl.opt_mem_alignment = getpagesize(); 105bd44feb7SStefan Hajnoczi 106bd44feb7SStefan Hajnoczi /* Safe default since most protocols use readv()/writev()/etc */ 107bd44feb7SStefan Hajnoczi bs->bl.max_iov = IOV_MAX; 10861007b31SStefan Hajnoczi } 10961007b31SStefan Hajnoczi 110760e0063SKevin Wolf if (bs->backing) { 111760e0063SKevin Wolf bdrv_refresh_limits(bs->backing->bs, &local_err); 11261007b31SStefan Hajnoczi if (local_err) { 11361007b31SStefan Hajnoczi error_propagate(errp, local_err); 11461007b31SStefan Hajnoczi return; 11561007b31SStefan Hajnoczi } 116d9e0dfa2SEric Blake bdrv_merge_limits(&bs->bl, &bs->backing->bs->bl); 11761007b31SStefan Hajnoczi } 11861007b31SStefan Hajnoczi 11961007b31SStefan Hajnoczi /* Then let the driver override it */ 12061007b31SStefan Hajnoczi if (drv->bdrv_refresh_limits) { 12161007b31SStefan Hajnoczi drv->bdrv_refresh_limits(bs, errp); 12261007b31SStefan Hajnoczi } 12361007b31SStefan Hajnoczi } 12461007b31SStefan Hajnoczi 12561007b31SStefan Hajnoczi /** 12661007b31SStefan Hajnoczi * The copy-on-read flag is actually a reference count so multiple users may 12761007b31SStefan Hajnoczi * use the feature without worrying about clobbering its previous state. 12861007b31SStefan Hajnoczi * Copy-on-read stays enabled until all users have called to disable it. 12961007b31SStefan Hajnoczi */ 13061007b31SStefan Hajnoczi void bdrv_enable_copy_on_read(BlockDriverState *bs) 13161007b31SStefan Hajnoczi { 13261007b31SStefan Hajnoczi bs->copy_on_read++; 13361007b31SStefan Hajnoczi } 13461007b31SStefan Hajnoczi 13561007b31SStefan Hajnoczi void bdrv_disable_copy_on_read(BlockDriverState *bs) 13661007b31SStefan Hajnoczi { 13761007b31SStefan Hajnoczi assert(bs->copy_on_read > 0); 13861007b31SStefan Hajnoczi bs->copy_on_read--; 13961007b31SStefan Hajnoczi } 14061007b31SStefan Hajnoczi 14161007b31SStefan Hajnoczi /* Check if any requests are in-flight (including throttled requests) */ 142439db28cSKevin Wolf bool bdrv_requests_pending(BlockDriverState *bs) 14361007b31SStefan Hajnoczi { 14437a639a7SKevin Wolf BdrvChild *child; 14537a639a7SKevin Wolf 14699723548SPaolo Bonzini if (atomic_read(&bs->in_flight)) { 14761007b31SStefan Hajnoczi return true; 14861007b31SStefan Hajnoczi } 14937a639a7SKevin Wolf 15037a639a7SKevin Wolf QLIST_FOREACH(child, &bs->children, next) { 15137a639a7SKevin Wolf if (bdrv_requests_pending(child->bs)) { 15261007b31SStefan Hajnoczi return true; 15361007b31SStefan Hajnoczi } 15461007b31SStefan Hajnoczi } 15537a639a7SKevin Wolf 15661007b31SStefan Hajnoczi return false; 15761007b31SStefan Hajnoczi } 15861007b31SStefan Hajnoczi 159d42cf288SPaolo Bonzini static bool bdrv_drain_recurse(BlockDriverState *bs) 16067da1dc5SFam Zheng { 16167da1dc5SFam Zheng BdrvChild *child; 162d42cf288SPaolo Bonzini bool waited; 163d42cf288SPaolo Bonzini 16488b062c2SPaolo Bonzini waited = BDRV_POLL_WHILE(bs, atomic_read(&bs->in_flight) > 0); 16567da1dc5SFam Zheng 16667da1dc5SFam Zheng if (bs->drv && bs->drv->bdrv_drain) { 16767da1dc5SFam Zheng bs->drv->bdrv_drain(bs); 16867da1dc5SFam Zheng } 169d42cf288SPaolo Bonzini 17067da1dc5SFam Zheng QLIST_FOREACH(child, &bs->children, next) { 171d42cf288SPaolo Bonzini waited |= bdrv_drain_recurse(child->bs); 17267da1dc5SFam Zheng } 173d42cf288SPaolo Bonzini 174d42cf288SPaolo Bonzini return waited; 17567da1dc5SFam Zheng } 17667da1dc5SFam Zheng 177a77fd4bbSFam Zheng typedef struct { 178a77fd4bbSFam Zheng Coroutine *co; 179a77fd4bbSFam Zheng BlockDriverState *bs; 180a77fd4bbSFam Zheng bool done; 181a77fd4bbSFam Zheng } BdrvCoDrainData; 182a77fd4bbSFam Zheng 183a77fd4bbSFam Zheng static void bdrv_co_drain_bh_cb(void *opaque) 184a77fd4bbSFam Zheng { 185a77fd4bbSFam Zheng BdrvCoDrainData *data = opaque; 186a77fd4bbSFam Zheng Coroutine *co = data->co; 18799723548SPaolo Bonzini BlockDriverState *bs = data->bs; 188a77fd4bbSFam Zheng 18999723548SPaolo Bonzini bdrv_dec_in_flight(bs); 190d42cf288SPaolo Bonzini bdrv_drained_begin(bs); 191a77fd4bbSFam Zheng data->done = true; 1921919631eSPaolo Bonzini aio_co_wake(co); 193a77fd4bbSFam Zheng } 194a77fd4bbSFam Zheng 195b6e84c97SPaolo Bonzini static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs) 196a77fd4bbSFam Zheng { 197a77fd4bbSFam Zheng BdrvCoDrainData data; 198a77fd4bbSFam Zheng 199a77fd4bbSFam Zheng /* Calling bdrv_drain() from a BH ensures the current coroutine yields and 200a77fd4bbSFam Zheng * other coroutines run if they were queued from 201a77fd4bbSFam Zheng * qemu_co_queue_run_restart(). */ 202a77fd4bbSFam Zheng 203a77fd4bbSFam Zheng assert(qemu_in_coroutine()); 204a77fd4bbSFam Zheng data = (BdrvCoDrainData) { 205a77fd4bbSFam Zheng .co = qemu_coroutine_self(), 206a77fd4bbSFam Zheng .bs = bs, 207a77fd4bbSFam Zheng .done = false, 208a77fd4bbSFam Zheng }; 20999723548SPaolo Bonzini bdrv_inc_in_flight(bs); 210fffb6e12SPaolo Bonzini aio_bh_schedule_oneshot(bdrv_get_aio_context(bs), 211fffb6e12SPaolo Bonzini bdrv_co_drain_bh_cb, &data); 212a77fd4bbSFam Zheng 213a77fd4bbSFam Zheng qemu_coroutine_yield(); 214a77fd4bbSFam Zheng /* If we are resumed from some other event (such as an aio completion or a 215a77fd4bbSFam Zheng * timer callback), it is a bug in the caller that should be fixed. */ 216a77fd4bbSFam Zheng assert(data.done); 217a77fd4bbSFam Zheng } 218a77fd4bbSFam Zheng 2196820643fSKevin Wolf void bdrv_drained_begin(BlockDriverState *bs) 2206820643fSKevin Wolf { 221d42cf288SPaolo Bonzini if (qemu_in_coroutine()) { 222d42cf288SPaolo Bonzini bdrv_co_yield_to_drain(bs); 223d42cf288SPaolo Bonzini return; 224d42cf288SPaolo Bonzini } 225d42cf288SPaolo Bonzini 2266820643fSKevin Wolf if (!bs->quiesce_counter++) { 2276820643fSKevin Wolf aio_disable_external(bdrv_get_aio_context(bs)); 2286820643fSKevin Wolf bdrv_parent_drained_begin(bs); 2296820643fSKevin Wolf } 2306820643fSKevin Wolf 2316820643fSKevin Wolf bdrv_drain_recurse(bs); 2326820643fSKevin Wolf } 2336820643fSKevin Wolf 2346820643fSKevin Wolf void bdrv_drained_end(BlockDriverState *bs) 2356820643fSKevin Wolf { 2366820643fSKevin Wolf assert(bs->quiesce_counter > 0); 2376820643fSKevin Wolf if (--bs->quiesce_counter > 0) { 2386820643fSKevin Wolf return; 2396820643fSKevin Wolf } 2406820643fSKevin Wolf 2416820643fSKevin Wolf bdrv_parent_drained_end(bs); 2426820643fSKevin Wolf aio_enable_external(bdrv_get_aio_context(bs)); 2436820643fSKevin Wolf } 2446820643fSKevin Wolf 24561007b31SStefan Hajnoczi /* 24667da1dc5SFam Zheng * Wait for pending requests to complete on a single BlockDriverState subtree, 24767da1dc5SFam Zheng * and suspend block driver's internal I/O until next request arrives. 24861007b31SStefan Hajnoczi * 24961007b31SStefan Hajnoczi * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState 25061007b31SStefan Hajnoczi * AioContext. 2517a63f3cdSStefan Hajnoczi * 2527a63f3cdSStefan Hajnoczi * Only this BlockDriverState's AioContext is run, so in-flight requests must 2537a63f3cdSStefan Hajnoczi * not depend on events in other AioContexts. In that case, use 2547a63f3cdSStefan Hajnoczi * bdrv_drain_all() instead. 25561007b31SStefan Hajnoczi */ 256b6e84c97SPaolo Bonzini void coroutine_fn bdrv_co_drain(BlockDriverState *bs) 257b6e84c97SPaolo Bonzini { 2586820643fSKevin Wolf assert(qemu_in_coroutine()); 2596820643fSKevin Wolf bdrv_drained_begin(bs); 2606820643fSKevin Wolf bdrv_drained_end(bs); 261b6e84c97SPaolo Bonzini } 262b6e84c97SPaolo Bonzini 26361007b31SStefan Hajnoczi void bdrv_drain(BlockDriverState *bs) 26461007b31SStefan Hajnoczi { 2656820643fSKevin Wolf bdrv_drained_begin(bs); 2666820643fSKevin Wolf bdrv_drained_end(bs); 26761007b31SStefan Hajnoczi } 26861007b31SStefan Hajnoczi 26961007b31SStefan Hajnoczi /* 27061007b31SStefan Hajnoczi * Wait for pending requests to complete across all BlockDriverStates 27161007b31SStefan Hajnoczi * 27261007b31SStefan Hajnoczi * This function does not flush data to disk, use bdrv_flush_all() for that 27361007b31SStefan Hajnoczi * after calling this function. 274c0778f66SAlberto Garcia * 275c0778f66SAlberto Garcia * This pauses all block jobs and disables external clients. It must 276c0778f66SAlberto Garcia * be paired with bdrv_drain_all_end(). 277c0778f66SAlberto Garcia * 278c0778f66SAlberto Garcia * NOTE: no new block jobs or BlockDriverStates can be created between 279c0778f66SAlberto Garcia * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls. 28061007b31SStefan Hajnoczi */ 281c0778f66SAlberto Garcia void bdrv_drain_all_begin(void) 28261007b31SStefan Hajnoczi { 28361007b31SStefan Hajnoczi /* Always run first iteration so any pending completion BHs run */ 28499723548SPaolo Bonzini bool waited = true; 2857c8eece4SKevin Wolf BlockDriverState *bs; 28688be7b4bSKevin Wolf BdrvNextIterator it; 287eb1364ceSAlberto Garcia BlockJob *job = NULL; 288f406c03cSAlexander Yarygin GSList *aio_ctxs = NULL, *ctx; 28961007b31SStefan Hajnoczi 290eb1364ceSAlberto Garcia while ((job = block_job_next(job))) { 291eb1364ceSAlberto Garcia AioContext *aio_context = blk_get_aio_context(job->blk); 292eb1364ceSAlberto Garcia 293eb1364ceSAlberto Garcia aio_context_acquire(aio_context); 294eb1364ceSAlberto Garcia block_job_pause(job); 295eb1364ceSAlberto Garcia aio_context_release(aio_context); 296eb1364ceSAlberto Garcia } 297eb1364ceSAlberto Garcia 29888be7b4bSKevin Wolf for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 29961007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 30061007b31SStefan Hajnoczi 30161007b31SStefan Hajnoczi aio_context_acquire(aio_context); 302c2066af0SKevin Wolf bdrv_parent_drained_begin(bs); 303c0778f66SAlberto Garcia aio_disable_external(aio_context); 30461007b31SStefan Hajnoczi aio_context_release(aio_context); 305f406c03cSAlexander Yarygin 306764ba3aeSAlberto Garcia if (!g_slist_find(aio_ctxs, aio_context)) { 307f406c03cSAlexander Yarygin aio_ctxs = g_slist_prepend(aio_ctxs, aio_context); 308f406c03cSAlexander Yarygin } 30961007b31SStefan Hajnoczi } 31061007b31SStefan Hajnoczi 3117a63f3cdSStefan Hajnoczi /* Note that completion of an asynchronous I/O operation can trigger any 3127a63f3cdSStefan Hajnoczi * number of other I/O operations on other devices---for example a 3137a63f3cdSStefan Hajnoczi * coroutine can submit an I/O request to another device in response to 3147a63f3cdSStefan Hajnoczi * request completion. Therefore we must keep looping until there was no 3157a63f3cdSStefan Hajnoczi * more activity rather than simply draining each device independently. 3167a63f3cdSStefan Hajnoczi */ 31799723548SPaolo Bonzini while (waited) { 31899723548SPaolo Bonzini waited = false; 319f406c03cSAlexander Yarygin 320f406c03cSAlexander Yarygin for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) { 321f406c03cSAlexander Yarygin AioContext *aio_context = ctx->data; 32261007b31SStefan Hajnoczi 32361007b31SStefan Hajnoczi aio_context_acquire(aio_context); 32488be7b4bSKevin Wolf for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 325f406c03cSAlexander Yarygin if (aio_context == bdrv_get_aio_context(bs)) { 326d42cf288SPaolo Bonzini waited |= bdrv_drain_recurse(bs); 327f406c03cSAlexander Yarygin } 328f406c03cSAlexander Yarygin } 32961007b31SStefan Hajnoczi aio_context_release(aio_context); 33061007b31SStefan Hajnoczi } 33161007b31SStefan Hajnoczi } 33261007b31SStefan Hajnoczi 333c0778f66SAlberto Garcia g_slist_free(aio_ctxs); 334c0778f66SAlberto Garcia } 335c0778f66SAlberto Garcia 336c0778f66SAlberto Garcia void bdrv_drain_all_end(void) 337c0778f66SAlberto Garcia { 338c0778f66SAlberto Garcia BlockDriverState *bs; 339c0778f66SAlberto Garcia BdrvNextIterator it; 340c0778f66SAlberto Garcia BlockJob *job = NULL; 341c0778f66SAlberto Garcia 34288be7b4bSKevin Wolf for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 34361007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 34461007b31SStefan Hajnoczi 34561007b31SStefan Hajnoczi aio_context_acquire(aio_context); 346c0778f66SAlberto Garcia aio_enable_external(aio_context); 347c2066af0SKevin Wolf bdrv_parent_drained_end(bs); 34861007b31SStefan Hajnoczi aio_context_release(aio_context); 34961007b31SStefan Hajnoczi } 350eb1364ceSAlberto Garcia 351eb1364ceSAlberto Garcia while ((job = block_job_next(job))) { 352eb1364ceSAlberto Garcia AioContext *aio_context = blk_get_aio_context(job->blk); 353eb1364ceSAlberto Garcia 354eb1364ceSAlberto Garcia aio_context_acquire(aio_context); 355eb1364ceSAlberto Garcia block_job_resume(job); 356eb1364ceSAlberto Garcia aio_context_release(aio_context); 357eb1364ceSAlberto Garcia } 35861007b31SStefan Hajnoczi } 35961007b31SStefan Hajnoczi 360c0778f66SAlberto Garcia void bdrv_drain_all(void) 361c0778f66SAlberto Garcia { 362c0778f66SAlberto Garcia bdrv_drain_all_begin(); 363c0778f66SAlberto Garcia bdrv_drain_all_end(); 364c0778f66SAlberto Garcia } 365c0778f66SAlberto Garcia 36661007b31SStefan Hajnoczi /** 36761007b31SStefan Hajnoczi * Remove an active request from the tracked requests list 36861007b31SStefan Hajnoczi * 36961007b31SStefan Hajnoczi * This function should be called when a tracked request is completing. 37061007b31SStefan Hajnoczi */ 37161007b31SStefan Hajnoczi static void tracked_request_end(BdrvTrackedRequest *req) 37261007b31SStefan Hajnoczi { 37361007b31SStefan Hajnoczi if (req->serialising) { 37461007b31SStefan Hajnoczi req->bs->serialising_in_flight--; 37561007b31SStefan Hajnoczi } 37661007b31SStefan Hajnoczi 37761007b31SStefan Hajnoczi QLIST_REMOVE(req, list); 37861007b31SStefan Hajnoczi qemu_co_queue_restart_all(&req->wait_queue); 37961007b31SStefan Hajnoczi } 38061007b31SStefan Hajnoczi 38161007b31SStefan Hajnoczi /** 38261007b31SStefan Hajnoczi * Add an active request to the tracked requests list 38361007b31SStefan Hajnoczi */ 38461007b31SStefan Hajnoczi static void tracked_request_begin(BdrvTrackedRequest *req, 38561007b31SStefan Hajnoczi BlockDriverState *bs, 38661007b31SStefan Hajnoczi int64_t offset, 387ebde595cSFam Zheng unsigned int bytes, 388ebde595cSFam Zheng enum BdrvTrackedRequestType type) 38961007b31SStefan Hajnoczi { 39061007b31SStefan Hajnoczi *req = (BdrvTrackedRequest){ 39161007b31SStefan Hajnoczi .bs = bs, 39261007b31SStefan Hajnoczi .offset = offset, 39361007b31SStefan Hajnoczi .bytes = bytes, 394ebde595cSFam Zheng .type = type, 39561007b31SStefan Hajnoczi .co = qemu_coroutine_self(), 39661007b31SStefan Hajnoczi .serialising = false, 39761007b31SStefan Hajnoczi .overlap_offset = offset, 39861007b31SStefan Hajnoczi .overlap_bytes = bytes, 39961007b31SStefan Hajnoczi }; 40061007b31SStefan Hajnoczi 40161007b31SStefan Hajnoczi qemu_co_queue_init(&req->wait_queue); 40261007b31SStefan Hajnoczi 40361007b31SStefan Hajnoczi QLIST_INSERT_HEAD(&bs->tracked_requests, req, list); 40461007b31SStefan Hajnoczi } 40561007b31SStefan Hajnoczi 40661007b31SStefan Hajnoczi static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align) 40761007b31SStefan Hajnoczi { 40861007b31SStefan Hajnoczi int64_t overlap_offset = req->offset & ~(align - 1); 40961007b31SStefan Hajnoczi unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align) 41061007b31SStefan Hajnoczi - overlap_offset; 41161007b31SStefan Hajnoczi 41261007b31SStefan Hajnoczi if (!req->serialising) { 41361007b31SStefan Hajnoczi req->bs->serialising_in_flight++; 41461007b31SStefan Hajnoczi req->serialising = true; 41561007b31SStefan Hajnoczi } 41661007b31SStefan Hajnoczi 41761007b31SStefan Hajnoczi req->overlap_offset = MIN(req->overlap_offset, overlap_offset); 41861007b31SStefan Hajnoczi req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes); 41961007b31SStefan Hajnoczi } 42061007b31SStefan Hajnoczi 42161007b31SStefan Hajnoczi /** 422244483e6SKevin Wolf * Round a region to cluster boundaries (sector-based) 42361007b31SStefan Hajnoczi */ 424244483e6SKevin Wolf void bdrv_round_sectors_to_clusters(BlockDriverState *bs, 42561007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, 42661007b31SStefan Hajnoczi int64_t *cluster_sector_num, 42761007b31SStefan Hajnoczi int *cluster_nb_sectors) 42861007b31SStefan Hajnoczi { 42961007b31SStefan Hajnoczi BlockDriverInfo bdi; 43061007b31SStefan Hajnoczi 43161007b31SStefan Hajnoczi if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 43261007b31SStefan Hajnoczi *cluster_sector_num = sector_num; 43361007b31SStefan Hajnoczi *cluster_nb_sectors = nb_sectors; 43461007b31SStefan Hajnoczi } else { 43561007b31SStefan Hajnoczi int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE; 43661007b31SStefan Hajnoczi *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c); 43761007b31SStefan Hajnoczi *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num + 43861007b31SStefan Hajnoczi nb_sectors, c); 43961007b31SStefan Hajnoczi } 44061007b31SStefan Hajnoczi } 44161007b31SStefan Hajnoczi 442244483e6SKevin Wolf /** 443244483e6SKevin Wolf * Round a region to cluster boundaries 444244483e6SKevin Wolf */ 445244483e6SKevin Wolf void bdrv_round_to_clusters(BlockDriverState *bs, 446244483e6SKevin Wolf int64_t offset, unsigned int bytes, 447244483e6SKevin Wolf int64_t *cluster_offset, 448244483e6SKevin Wolf unsigned int *cluster_bytes) 449244483e6SKevin Wolf { 450244483e6SKevin Wolf BlockDriverInfo bdi; 451244483e6SKevin Wolf 452244483e6SKevin Wolf if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 453244483e6SKevin Wolf *cluster_offset = offset; 454244483e6SKevin Wolf *cluster_bytes = bytes; 455244483e6SKevin Wolf } else { 456244483e6SKevin Wolf int64_t c = bdi.cluster_size; 457244483e6SKevin Wolf *cluster_offset = QEMU_ALIGN_DOWN(offset, c); 458244483e6SKevin Wolf *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c); 459244483e6SKevin Wolf } 460244483e6SKevin Wolf } 461244483e6SKevin Wolf 46261007b31SStefan Hajnoczi static int bdrv_get_cluster_size(BlockDriverState *bs) 46361007b31SStefan Hajnoczi { 46461007b31SStefan Hajnoczi BlockDriverInfo bdi; 46561007b31SStefan Hajnoczi int ret; 46661007b31SStefan Hajnoczi 46761007b31SStefan Hajnoczi ret = bdrv_get_info(bs, &bdi); 46861007b31SStefan Hajnoczi if (ret < 0 || bdi.cluster_size == 0) { 469a5b8dd2cSEric Blake return bs->bl.request_alignment; 47061007b31SStefan Hajnoczi } else { 47161007b31SStefan Hajnoczi return bdi.cluster_size; 47261007b31SStefan Hajnoczi } 47361007b31SStefan Hajnoczi } 47461007b31SStefan Hajnoczi 47561007b31SStefan Hajnoczi static bool tracked_request_overlaps(BdrvTrackedRequest *req, 47661007b31SStefan Hajnoczi int64_t offset, unsigned int bytes) 47761007b31SStefan Hajnoczi { 47861007b31SStefan Hajnoczi /* aaaa bbbb */ 47961007b31SStefan Hajnoczi if (offset >= req->overlap_offset + req->overlap_bytes) { 48061007b31SStefan Hajnoczi return false; 48161007b31SStefan Hajnoczi } 48261007b31SStefan Hajnoczi /* bbbb aaaa */ 48361007b31SStefan Hajnoczi if (req->overlap_offset >= offset + bytes) { 48461007b31SStefan Hajnoczi return false; 48561007b31SStefan Hajnoczi } 48661007b31SStefan Hajnoczi return true; 48761007b31SStefan Hajnoczi } 48861007b31SStefan Hajnoczi 48999723548SPaolo Bonzini void bdrv_inc_in_flight(BlockDriverState *bs) 49099723548SPaolo Bonzini { 49199723548SPaolo Bonzini atomic_inc(&bs->in_flight); 49299723548SPaolo Bonzini } 49399723548SPaolo Bonzini 494c9d1a561SPaolo Bonzini static void dummy_bh_cb(void *opaque) 495c9d1a561SPaolo Bonzini { 496c9d1a561SPaolo Bonzini } 497c9d1a561SPaolo Bonzini 498c9d1a561SPaolo Bonzini void bdrv_wakeup(BlockDriverState *bs) 499c9d1a561SPaolo Bonzini { 500c9d1a561SPaolo Bonzini if (bs->wakeup) { 501c9d1a561SPaolo Bonzini aio_bh_schedule_oneshot(qemu_get_aio_context(), dummy_bh_cb, NULL); 502c9d1a561SPaolo Bonzini } 503c9d1a561SPaolo Bonzini } 504c9d1a561SPaolo Bonzini 50599723548SPaolo Bonzini void bdrv_dec_in_flight(BlockDriverState *bs) 50699723548SPaolo Bonzini { 50799723548SPaolo Bonzini atomic_dec(&bs->in_flight); 508c9d1a561SPaolo Bonzini bdrv_wakeup(bs); 50999723548SPaolo Bonzini } 51099723548SPaolo Bonzini 51161007b31SStefan Hajnoczi static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self) 51261007b31SStefan Hajnoczi { 51361007b31SStefan Hajnoczi BlockDriverState *bs = self->bs; 51461007b31SStefan Hajnoczi BdrvTrackedRequest *req; 51561007b31SStefan Hajnoczi bool retry; 51661007b31SStefan Hajnoczi bool waited = false; 51761007b31SStefan Hajnoczi 51861007b31SStefan Hajnoczi if (!bs->serialising_in_flight) { 51961007b31SStefan Hajnoczi return false; 52061007b31SStefan Hajnoczi } 52161007b31SStefan Hajnoczi 52261007b31SStefan Hajnoczi do { 52361007b31SStefan Hajnoczi retry = false; 52461007b31SStefan Hajnoczi QLIST_FOREACH(req, &bs->tracked_requests, list) { 52561007b31SStefan Hajnoczi if (req == self || (!req->serialising && !self->serialising)) { 52661007b31SStefan Hajnoczi continue; 52761007b31SStefan Hajnoczi } 52861007b31SStefan Hajnoczi if (tracked_request_overlaps(req, self->overlap_offset, 52961007b31SStefan Hajnoczi self->overlap_bytes)) 53061007b31SStefan Hajnoczi { 53161007b31SStefan Hajnoczi /* Hitting this means there was a reentrant request, for 53261007b31SStefan Hajnoczi * example, a block driver issuing nested requests. This must 53361007b31SStefan Hajnoczi * never happen since it means deadlock. 53461007b31SStefan Hajnoczi */ 53561007b31SStefan Hajnoczi assert(qemu_coroutine_self() != req->co); 53661007b31SStefan Hajnoczi 53761007b31SStefan Hajnoczi /* If the request is already (indirectly) waiting for us, or 53861007b31SStefan Hajnoczi * will wait for us as soon as it wakes up, then just go on 53961007b31SStefan Hajnoczi * (instead of producing a deadlock in the former case). */ 54061007b31SStefan Hajnoczi if (!req->waiting_for) { 54161007b31SStefan Hajnoczi self->waiting_for = req; 5421ace7ceaSPaolo Bonzini qemu_co_queue_wait(&req->wait_queue, NULL); 54361007b31SStefan Hajnoczi self->waiting_for = NULL; 54461007b31SStefan Hajnoczi retry = true; 54561007b31SStefan Hajnoczi waited = true; 54661007b31SStefan Hajnoczi break; 54761007b31SStefan Hajnoczi } 54861007b31SStefan Hajnoczi } 54961007b31SStefan Hajnoczi } 55061007b31SStefan Hajnoczi } while (retry); 55161007b31SStefan Hajnoczi 55261007b31SStefan Hajnoczi return waited; 55361007b31SStefan Hajnoczi } 55461007b31SStefan Hajnoczi 55561007b31SStefan Hajnoczi static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset, 55661007b31SStefan Hajnoczi size_t size) 55761007b31SStefan Hajnoczi { 55861007b31SStefan Hajnoczi if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) { 55961007b31SStefan Hajnoczi return -EIO; 56061007b31SStefan Hajnoczi } 56161007b31SStefan Hajnoczi 56261007b31SStefan Hajnoczi if (!bdrv_is_inserted(bs)) { 56361007b31SStefan Hajnoczi return -ENOMEDIUM; 56461007b31SStefan Hajnoczi } 56561007b31SStefan Hajnoczi 56661007b31SStefan Hajnoczi if (offset < 0) { 56761007b31SStefan Hajnoczi return -EIO; 56861007b31SStefan Hajnoczi } 56961007b31SStefan Hajnoczi 57061007b31SStefan Hajnoczi return 0; 57161007b31SStefan Hajnoczi } 57261007b31SStefan Hajnoczi 57361007b31SStefan Hajnoczi typedef struct RwCo { 574e293b7a3SKevin Wolf BdrvChild *child; 57561007b31SStefan Hajnoczi int64_t offset; 57661007b31SStefan Hajnoczi QEMUIOVector *qiov; 57761007b31SStefan Hajnoczi bool is_write; 57861007b31SStefan Hajnoczi int ret; 57961007b31SStefan Hajnoczi BdrvRequestFlags flags; 58061007b31SStefan Hajnoczi } RwCo; 58161007b31SStefan Hajnoczi 58261007b31SStefan Hajnoczi static void coroutine_fn bdrv_rw_co_entry(void *opaque) 58361007b31SStefan Hajnoczi { 58461007b31SStefan Hajnoczi RwCo *rwco = opaque; 58561007b31SStefan Hajnoczi 58661007b31SStefan Hajnoczi if (!rwco->is_write) { 587a03ef88fSKevin Wolf rwco->ret = bdrv_co_preadv(rwco->child, rwco->offset, 58861007b31SStefan Hajnoczi rwco->qiov->size, rwco->qiov, 58961007b31SStefan Hajnoczi rwco->flags); 59061007b31SStefan Hajnoczi } else { 591a03ef88fSKevin Wolf rwco->ret = bdrv_co_pwritev(rwco->child, rwco->offset, 59261007b31SStefan Hajnoczi rwco->qiov->size, rwco->qiov, 59361007b31SStefan Hajnoczi rwco->flags); 59461007b31SStefan Hajnoczi } 59561007b31SStefan Hajnoczi } 59661007b31SStefan Hajnoczi 59761007b31SStefan Hajnoczi /* 59861007b31SStefan Hajnoczi * Process a vectored synchronous request using coroutines 59961007b31SStefan Hajnoczi */ 600e293b7a3SKevin Wolf static int bdrv_prwv_co(BdrvChild *child, int64_t offset, 60161007b31SStefan Hajnoczi QEMUIOVector *qiov, bool is_write, 60261007b31SStefan Hajnoczi BdrvRequestFlags flags) 60361007b31SStefan Hajnoczi { 60461007b31SStefan Hajnoczi Coroutine *co; 60561007b31SStefan Hajnoczi RwCo rwco = { 606e293b7a3SKevin Wolf .child = child, 60761007b31SStefan Hajnoczi .offset = offset, 60861007b31SStefan Hajnoczi .qiov = qiov, 60961007b31SStefan Hajnoczi .is_write = is_write, 61061007b31SStefan Hajnoczi .ret = NOT_DONE, 61161007b31SStefan Hajnoczi .flags = flags, 61261007b31SStefan Hajnoczi }; 61361007b31SStefan Hajnoczi 61461007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 61561007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 61661007b31SStefan Hajnoczi bdrv_rw_co_entry(&rwco); 61761007b31SStefan Hajnoczi } else { 6180b8b8753SPaolo Bonzini co = qemu_coroutine_create(bdrv_rw_co_entry, &rwco); 6190b8b8753SPaolo Bonzini qemu_coroutine_enter(co); 62088b062c2SPaolo Bonzini BDRV_POLL_WHILE(child->bs, rwco.ret == NOT_DONE); 62161007b31SStefan Hajnoczi } 62261007b31SStefan Hajnoczi return rwco.ret; 62361007b31SStefan Hajnoczi } 62461007b31SStefan Hajnoczi 62561007b31SStefan Hajnoczi /* 62661007b31SStefan Hajnoczi * Process a synchronous request using coroutines 62761007b31SStefan Hajnoczi */ 628e293b7a3SKevin Wolf static int bdrv_rw_co(BdrvChild *child, int64_t sector_num, uint8_t *buf, 62961007b31SStefan Hajnoczi int nb_sectors, bool is_write, BdrvRequestFlags flags) 63061007b31SStefan Hajnoczi { 63161007b31SStefan Hajnoczi QEMUIOVector qiov; 63261007b31SStefan Hajnoczi struct iovec iov = { 63361007b31SStefan Hajnoczi .iov_base = (void *)buf, 63461007b31SStefan Hajnoczi .iov_len = nb_sectors * BDRV_SECTOR_SIZE, 63561007b31SStefan Hajnoczi }; 63661007b31SStefan Hajnoczi 63761007b31SStefan Hajnoczi if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 63861007b31SStefan Hajnoczi return -EINVAL; 63961007b31SStefan Hajnoczi } 64061007b31SStefan Hajnoczi 64161007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 642e293b7a3SKevin Wolf return bdrv_prwv_co(child, sector_num << BDRV_SECTOR_BITS, 64361007b31SStefan Hajnoczi &qiov, is_write, flags); 64461007b31SStefan Hajnoczi } 64561007b31SStefan Hajnoczi 64661007b31SStefan Hajnoczi /* return < 0 if error. See bdrv_write() for the return codes */ 647fbcbbf4eSKevin Wolf int bdrv_read(BdrvChild *child, int64_t sector_num, 64861007b31SStefan Hajnoczi uint8_t *buf, int nb_sectors) 64961007b31SStefan Hajnoczi { 650e293b7a3SKevin Wolf return bdrv_rw_co(child, sector_num, buf, nb_sectors, false, 0); 65161007b31SStefan Hajnoczi } 65261007b31SStefan Hajnoczi 65361007b31SStefan Hajnoczi /* Return < 0 if error. Important errors are: 65461007b31SStefan Hajnoczi -EIO generic I/O error (may happen for all errors) 65561007b31SStefan Hajnoczi -ENOMEDIUM No media inserted. 65661007b31SStefan Hajnoczi -EINVAL Invalid sector number or nb_sectors 65761007b31SStefan Hajnoczi -EACCES Trying to write a read-only device 65861007b31SStefan Hajnoczi */ 65918d51c4bSKevin Wolf int bdrv_write(BdrvChild *child, int64_t sector_num, 66061007b31SStefan Hajnoczi const uint8_t *buf, int nb_sectors) 66161007b31SStefan Hajnoczi { 662e293b7a3SKevin Wolf return bdrv_rw_co(child, sector_num, (uint8_t *)buf, nb_sectors, true, 0); 66361007b31SStefan Hajnoczi } 66461007b31SStefan Hajnoczi 665720ff280SKevin Wolf int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset, 66674021bc4SEric Blake int count, BdrvRequestFlags flags) 66761007b31SStefan Hajnoczi { 66874021bc4SEric Blake QEMUIOVector qiov; 66974021bc4SEric Blake struct iovec iov = { 67074021bc4SEric Blake .iov_base = NULL, 67174021bc4SEric Blake .iov_len = count, 67274021bc4SEric Blake }; 67374021bc4SEric Blake 67474021bc4SEric Blake qemu_iovec_init_external(&qiov, &iov, 1); 675e293b7a3SKevin Wolf return bdrv_prwv_co(child, offset, &qiov, true, 67661007b31SStefan Hajnoczi BDRV_REQ_ZERO_WRITE | flags); 67761007b31SStefan Hajnoczi } 67861007b31SStefan Hajnoczi 67961007b31SStefan Hajnoczi /* 68074021bc4SEric Blake * Completely zero out a block device with the help of bdrv_pwrite_zeroes. 68161007b31SStefan Hajnoczi * The operation is sped up by checking the block status and only writing 68261007b31SStefan Hajnoczi * zeroes to the device if they currently do not return zeroes. Optional 68374021bc4SEric Blake * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP, 684465fe887SEric Blake * BDRV_REQ_FUA). 68561007b31SStefan Hajnoczi * 68661007b31SStefan Hajnoczi * Returns < 0 on error, 0 on success. For error codes see bdrv_write(). 68761007b31SStefan Hajnoczi */ 688720ff280SKevin Wolf int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags) 68961007b31SStefan Hajnoczi { 69061007b31SStefan Hajnoczi int64_t target_sectors, ret, nb_sectors, sector_num = 0; 691720ff280SKevin Wolf BlockDriverState *bs = child->bs; 69267a0fd2aSFam Zheng BlockDriverState *file; 69361007b31SStefan Hajnoczi int n; 69461007b31SStefan Hajnoczi 69561007b31SStefan Hajnoczi target_sectors = bdrv_nb_sectors(bs); 69661007b31SStefan Hajnoczi if (target_sectors < 0) { 69761007b31SStefan Hajnoczi return target_sectors; 69861007b31SStefan Hajnoczi } 69961007b31SStefan Hajnoczi 70061007b31SStefan Hajnoczi for (;;) { 70161007b31SStefan Hajnoczi nb_sectors = MIN(target_sectors - sector_num, BDRV_REQUEST_MAX_SECTORS); 70261007b31SStefan Hajnoczi if (nb_sectors <= 0) { 70361007b31SStefan Hajnoczi return 0; 70461007b31SStefan Hajnoczi } 70567a0fd2aSFam Zheng ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n, &file); 70661007b31SStefan Hajnoczi if (ret < 0) { 70761007b31SStefan Hajnoczi error_report("error getting block status at sector %" PRId64 ": %s", 70861007b31SStefan Hajnoczi sector_num, strerror(-ret)); 70961007b31SStefan Hajnoczi return ret; 71061007b31SStefan Hajnoczi } 71161007b31SStefan Hajnoczi if (ret & BDRV_BLOCK_ZERO) { 71261007b31SStefan Hajnoczi sector_num += n; 71361007b31SStefan Hajnoczi continue; 71461007b31SStefan Hajnoczi } 715720ff280SKevin Wolf ret = bdrv_pwrite_zeroes(child, sector_num << BDRV_SECTOR_BITS, 71674021bc4SEric Blake n << BDRV_SECTOR_BITS, flags); 71761007b31SStefan Hajnoczi if (ret < 0) { 71861007b31SStefan Hajnoczi error_report("error writing zeroes at sector %" PRId64 ": %s", 71961007b31SStefan Hajnoczi sector_num, strerror(-ret)); 72061007b31SStefan Hajnoczi return ret; 72161007b31SStefan Hajnoczi } 72261007b31SStefan Hajnoczi sector_num += n; 72361007b31SStefan Hajnoczi } 72461007b31SStefan Hajnoczi } 72561007b31SStefan Hajnoczi 726cf2ab8fcSKevin Wolf int bdrv_preadv(BdrvChild *child, int64_t offset, QEMUIOVector *qiov) 727f1e84741SKevin Wolf { 728f1e84741SKevin Wolf int ret; 729f1e84741SKevin Wolf 730e293b7a3SKevin Wolf ret = bdrv_prwv_co(child, offset, qiov, false, 0); 731f1e84741SKevin Wolf if (ret < 0) { 732f1e84741SKevin Wolf return ret; 733f1e84741SKevin Wolf } 734f1e84741SKevin Wolf 735f1e84741SKevin Wolf return qiov->size; 736f1e84741SKevin Wolf } 737f1e84741SKevin Wolf 738cf2ab8fcSKevin Wolf int bdrv_pread(BdrvChild *child, int64_t offset, void *buf, int bytes) 73961007b31SStefan Hajnoczi { 74061007b31SStefan Hajnoczi QEMUIOVector qiov; 74161007b31SStefan Hajnoczi struct iovec iov = { 74261007b31SStefan Hajnoczi .iov_base = (void *)buf, 74361007b31SStefan Hajnoczi .iov_len = bytes, 74461007b31SStefan Hajnoczi }; 74561007b31SStefan Hajnoczi 74661007b31SStefan Hajnoczi if (bytes < 0) { 74761007b31SStefan Hajnoczi return -EINVAL; 74861007b31SStefan Hajnoczi } 74961007b31SStefan Hajnoczi 75061007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 751cf2ab8fcSKevin Wolf return bdrv_preadv(child, offset, &qiov); 75261007b31SStefan Hajnoczi } 75361007b31SStefan Hajnoczi 754d9ca2ea2SKevin Wolf int bdrv_pwritev(BdrvChild *child, int64_t offset, QEMUIOVector *qiov) 75561007b31SStefan Hajnoczi { 75661007b31SStefan Hajnoczi int ret; 75761007b31SStefan Hajnoczi 758e293b7a3SKevin Wolf ret = bdrv_prwv_co(child, offset, qiov, true, 0); 75961007b31SStefan Hajnoczi if (ret < 0) { 76061007b31SStefan Hajnoczi return ret; 76161007b31SStefan Hajnoczi } 76261007b31SStefan Hajnoczi 76361007b31SStefan Hajnoczi return qiov->size; 76461007b31SStefan Hajnoczi } 76561007b31SStefan Hajnoczi 766d9ca2ea2SKevin Wolf int bdrv_pwrite(BdrvChild *child, int64_t offset, const void *buf, int bytes) 76761007b31SStefan Hajnoczi { 76861007b31SStefan Hajnoczi QEMUIOVector qiov; 76961007b31SStefan Hajnoczi struct iovec iov = { 77061007b31SStefan Hajnoczi .iov_base = (void *) buf, 77161007b31SStefan Hajnoczi .iov_len = bytes, 77261007b31SStefan Hajnoczi }; 77361007b31SStefan Hajnoczi 77461007b31SStefan Hajnoczi if (bytes < 0) { 77561007b31SStefan Hajnoczi return -EINVAL; 77661007b31SStefan Hajnoczi } 77761007b31SStefan Hajnoczi 77861007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 779d9ca2ea2SKevin Wolf return bdrv_pwritev(child, offset, &qiov); 78061007b31SStefan Hajnoczi } 78161007b31SStefan Hajnoczi 78261007b31SStefan Hajnoczi /* 78361007b31SStefan Hajnoczi * Writes to the file and ensures that no writes are reordered across this 78461007b31SStefan Hajnoczi * request (acts as a barrier) 78561007b31SStefan Hajnoczi * 78661007b31SStefan Hajnoczi * Returns 0 on success, -errno in error cases. 78761007b31SStefan Hajnoczi */ 788d9ca2ea2SKevin Wolf int bdrv_pwrite_sync(BdrvChild *child, int64_t offset, 78961007b31SStefan Hajnoczi const void *buf, int count) 79061007b31SStefan Hajnoczi { 79161007b31SStefan Hajnoczi int ret; 79261007b31SStefan Hajnoczi 793d9ca2ea2SKevin Wolf ret = bdrv_pwrite(child, offset, buf, count); 79461007b31SStefan Hajnoczi if (ret < 0) { 79561007b31SStefan Hajnoczi return ret; 79661007b31SStefan Hajnoczi } 79761007b31SStefan Hajnoczi 798d9ca2ea2SKevin Wolf ret = bdrv_flush(child->bs); 799855a6a93SKevin Wolf if (ret < 0) { 800855a6a93SKevin Wolf return ret; 80161007b31SStefan Hajnoczi } 80261007b31SStefan Hajnoczi 80361007b31SStefan Hajnoczi return 0; 80461007b31SStefan Hajnoczi } 80561007b31SStefan Hajnoczi 80608844473SKevin Wolf typedef struct CoroutineIOCompletion { 80708844473SKevin Wolf Coroutine *coroutine; 80808844473SKevin Wolf int ret; 80908844473SKevin Wolf } CoroutineIOCompletion; 81008844473SKevin Wolf 81108844473SKevin Wolf static void bdrv_co_io_em_complete(void *opaque, int ret) 81208844473SKevin Wolf { 81308844473SKevin Wolf CoroutineIOCompletion *co = opaque; 81408844473SKevin Wolf 81508844473SKevin Wolf co->ret = ret; 816b9e413ddSPaolo Bonzini aio_co_wake(co->coroutine); 81708844473SKevin Wolf } 81808844473SKevin Wolf 819166fe960SKevin Wolf static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs, 820166fe960SKevin Wolf uint64_t offset, uint64_t bytes, 821166fe960SKevin Wolf QEMUIOVector *qiov, int flags) 822166fe960SKevin Wolf { 823166fe960SKevin Wolf BlockDriver *drv = bs->drv; 8243fb06697SKevin Wolf int64_t sector_num; 8253fb06697SKevin Wolf unsigned int nb_sectors; 8263fb06697SKevin Wolf 827fa166538SEric Blake assert(!(flags & ~BDRV_REQ_MASK)); 828fa166538SEric Blake 8293fb06697SKevin Wolf if (drv->bdrv_co_preadv) { 8303fb06697SKevin Wolf return drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags); 8313fb06697SKevin Wolf } 8323fb06697SKevin Wolf 8333fb06697SKevin Wolf sector_num = offset >> BDRV_SECTOR_BITS; 8343fb06697SKevin Wolf nb_sectors = bytes >> BDRV_SECTOR_BITS; 835166fe960SKevin Wolf 836166fe960SKevin Wolf assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 837166fe960SKevin Wolf assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 838166fe960SKevin Wolf assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS); 839166fe960SKevin Wolf 84008844473SKevin Wolf if (drv->bdrv_co_readv) { 841166fe960SKevin Wolf return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov); 84208844473SKevin Wolf } else { 84308844473SKevin Wolf BlockAIOCB *acb; 84408844473SKevin Wolf CoroutineIOCompletion co = { 84508844473SKevin Wolf .coroutine = qemu_coroutine_self(), 84608844473SKevin Wolf }; 84708844473SKevin Wolf 84808844473SKevin Wolf acb = bs->drv->bdrv_aio_readv(bs, sector_num, qiov, nb_sectors, 84908844473SKevin Wolf bdrv_co_io_em_complete, &co); 85008844473SKevin Wolf if (acb == NULL) { 85108844473SKevin Wolf return -EIO; 85208844473SKevin Wolf } else { 85308844473SKevin Wolf qemu_coroutine_yield(); 85408844473SKevin Wolf return co.ret; 85508844473SKevin Wolf } 85608844473SKevin Wolf } 857166fe960SKevin Wolf } 858166fe960SKevin Wolf 85978a07294SKevin Wolf static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs, 86078a07294SKevin Wolf uint64_t offset, uint64_t bytes, 86178a07294SKevin Wolf QEMUIOVector *qiov, int flags) 86278a07294SKevin Wolf { 86378a07294SKevin Wolf BlockDriver *drv = bs->drv; 8643fb06697SKevin Wolf int64_t sector_num; 8653fb06697SKevin Wolf unsigned int nb_sectors; 86678a07294SKevin Wolf int ret; 86778a07294SKevin Wolf 868fa166538SEric Blake assert(!(flags & ~BDRV_REQ_MASK)); 869fa166538SEric Blake 8703fb06697SKevin Wolf if (drv->bdrv_co_pwritev) { 871515c2f43SKevin Wolf ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, 872515c2f43SKevin Wolf flags & bs->supported_write_flags); 873515c2f43SKevin Wolf flags &= ~bs->supported_write_flags; 8743fb06697SKevin Wolf goto emulate_flags; 8753fb06697SKevin Wolf } 8763fb06697SKevin Wolf 8773fb06697SKevin Wolf sector_num = offset >> BDRV_SECTOR_BITS; 8783fb06697SKevin Wolf nb_sectors = bytes >> BDRV_SECTOR_BITS; 8793fb06697SKevin Wolf 88078a07294SKevin Wolf assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 88178a07294SKevin Wolf assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 88278a07294SKevin Wolf assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS); 88378a07294SKevin Wolf 88478a07294SKevin Wolf if (drv->bdrv_co_writev_flags) { 88578a07294SKevin Wolf ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov, 8864df863f3SEric Blake flags & bs->supported_write_flags); 8874df863f3SEric Blake flags &= ~bs->supported_write_flags; 88808844473SKevin Wolf } else if (drv->bdrv_co_writev) { 8894df863f3SEric Blake assert(!bs->supported_write_flags); 89078a07294SKevin Wolf ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov); 89108844473SKevin Wolf } else { 89208844473SKevin Wolf BlockAIOCB *acb; 89308844473SKevin Wolf CoroutineIOCompletion co = { 89408844473SKevin Wolf .coroutine = qemu_coroutine_self(), 89508844473SKevin Wolf }; 89608844473SKevin Wolf 89708844473SKevin Wolf acb = bs->drv->bdrv_aio_writev(bs, sector_num, qiov, nb_sectors, 89808844473SKevin Wolf bdrv_co_io_em_complete, &co); 89908844473SKevin Wolf if (acb == NULL) { 9003fb06697SKevin Wolf ret = -EIO; 90108844473SKevin Wolf } else { 90208844473SKevin Wolf qemu_coroutine_yield(); 9033fb06697SKevin Wolf ret = co.ret; 90408844473SKevin Wolf } 90578a07294SKevin Wolf } 90678a07294SKevin Wolf 9073fb06697SKevin Wolf emulate_flags: 9084df863f3SEric Blake if (ret == 0 && (flags & BDRV_REQ_FUA)) { 90978a07294SKevin Wolf ret = bdrv_co_flush(bs); 91078a07294SKevin Wolf } 91178a07294SKevin Wolf 91278a07294SKevin Wolf return ret; 91378a07294SKevin Wolf } 91478a07294SKevin Wolf 91529a298afSPavel Butsykin static int coroutine_fn 91629a298afSPavel Butsykin bdrv_driver_pwritev_compressed(BlockDriverState *bs, uint64_t offset, 91729a298afSPavel Butsykin uint64_t bytes, QEMUIOVector *qiov) 91829a298afSPavel Butsykin { 91929a298afSPavel Butsykin BlockDriver *drv = bs->drv; 92029a298afSPavel Butsykin 92129a298afSPavel Butsykin if (!drv->bdrv_co_pwritev_compressed) { 92229a298afSPavel Butsykin return -ENOTSUP; 92329a298afSPavel Butsykin } 92429a298afSPavel Butsykin 92529a298afSPavel Butsykin return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov); 92629a298afSPavel Butsykin } 92729a298afSPavel Butsykin 928*85c97ca7SKevin Wolf static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child, 929244483e6SKevin Wolf int64_t offset, unsigned int bytes, QEMUIOVector *qiov) 93061007b31SStefan Hajnoczi { 931*85c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 932*85c97ca7SKevin Wolf 93361007b31SStefan Hajnoczi /* Perform I/O through a temporary buffer so that users who scribble over 93461007b31SStefan Hajnoczi * their read buffer while the operation is in progress do not end up 93561007b31SStefan Hajnoczi * modifying the image file. This is critical for zero-copy guest I/O 93661007b31SStefan Hajnoczi * where anything might happen inside guest memory. 93761007b31SStefan Hajnoczi */ 93861007b31SStefan Hajnoczi void *bounce_buffer; 93961007b31SStefan Hajnoczi 94061007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 94161007b31SStefan Hajnoczi struct iovec iov; 94261007b31SStefan Hajnoczi QEMUIOVector bounce_qiov; 943244483e6SKevin Wolf int64_t cluster_offset; 944244483e6SKevin Wolf unsigned int cluster_bytes; 94561007b31SStefan Hajnoczi size_t skip_bytes; 94661007b31SStefan Hajnoczi int ret; 94761007b31SStefan Hajnoczi 94861007b31SStefan Hajnoczi /* Cover entire cluster so no additional backing file I/O is required when 94961007b31SStefan Hajnoczi * allocating cluster in the image file. 95061007b31SStefan Hajnoczi */ 951244483e6SKevin Wolf bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes); 95261007b31SStefan Hajnoczi 953244483e6SKevin Wolf trace_bdrv_co_do_copy_on_readv(bs, offset, bytes, 954244483e6SKevin Wolf cluster_offset, cluster_bytes); 95561007b31SStefan Hajnoczi 956244483e6SKevin Wolf iov.iov_len = cluster_bytes; 95761007b31SStefan Hajnoczi iov.iov_base = bounce_buffer = qemu_try_blockalign(bs, iov.iov_len); 95861007b31SStefan Hajnoczi if (bounce_buffer == NULL) { 95961007b31SStefan Hajnoczi ret = -ENOMEM; 96061007b31SStefan Hajnoczi goto err; 96161007b31SStefan Hajnoczi } 96261007b31SStefan Hajnoczi 96361007b31SStefan Hajnoczi qemu_iovec_init_external(&bounce_qiov, &iov, 1); 96461007b31SStefan Hajnoczi 965244483e6SKevin Wolf ret = bdrv_driver_preadv(bs, cluster_offset, cluster_bytes, 966166fe960SKevin Wolf &bounce_qiov, 0); 96761007b31SStefan Hajnoczi if (ret < 0) { 96861007b31SStefan Hajnoczi goto err; 96961007b31SStefan Hajnoczi } 97061007b31SStefan Hajnoczi 971c1499a5eSEric Blake if (drv->bdrv_co_pwrite_zeroes && 97261007b31SStefan Hajnoczi buffer_is_zero(bounce_buffer, iov.iov_len)) { 973a604fa2bSEric Blake /* FIXME: Should we (perhaps conditionally) be setting 974a604fa2bSEric Blake * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy 975a604fa2bSEric Blake * that still correctly reads as zero? */ 976244483e6SKevin Wolf ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, cluster_bytes, 0); 97761007b31SStefan Hajnoczi } else { 97861007b31SStefan Hajnoczi /* This does not change the data on the disk, it is not necessary 97961007b31SStefan Hajnoczi * to flush even in cache=writethrough mode. 98061007b31SStefan Hajnoczi */ 981244483e6SKevin Wolf ret = bdrv_driver_pwritev(bs, cluster_offset, cluster_bytes, 98278a07294SKevin Wolf &bounce_qiov, 0); 98361007b31SStefan Hajnoczi } 98461007b31SStefan Hajnoczi 98561007b31SStefan Hajnoczi if (ret < 0) { 98661007b31SStefan Hajnoczi /* It might be okay to ignore write errors for guest requests. If this 98761007b31SStefan Hajnoczi * is a deliberate copy-on-read then we don't want to ignore the error. 98861007b31SStefan Hajnoczi * Simply report it in all cases. 98961007b31SStefan Hajnoczi */ 99061007b31SStefan Hajnoczi goto err; 99161007b31SStefan Hajnoczi } 99261007b31SStefan Hajnoczi 993244483e6SKevin Wolf skip_bytes = offset - cluster_offset; 994244483e6SKevin Wolf qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes, bytes); 99561007b31SStefan Hajnoczi 99661007b31SStefan Hajnoczi err: 99761007b31SStefan Hajnoczi qemu_vfree(bounce_buffer); 99861007b31SStefan Hajnoczi return ret; 99961007b31SStefan Hajnoczi } 100061007b31SStefan Hajnoczi 100161007b31SStefan Hajnoczi /* 100261007b31SStefan Hajnoczi * Forwards an already correctly aligned request to the BlockDriver. This 10031a62d0acSEric Blake * handles copy on read, zeroing after EOF, and fragmentation of large 10041a62d0acSEric Blake * reads; any other features must be implemented by the caller. 100561007b31SStefan Hajnoczi */ 1006*85c97ca7SKevin Wolf static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child, 100761007b31SStefan Hajnoczi BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 100861007b31SStefan Hajnoczi int64_t align, QEMUIOVector *qiov, int flags) 100961007b31SStefan Hajnoczi { 1010*85c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 1011c9d20029SKevin Wolf int64_t total_bytes, max_bytes; 10121a62d0acSEric Blake int ret = 0; 10131a62d0acSEric Blake uint64_t bytes_remaining = bytes; 10141a62d0acSEric Blake int max_transfer; 101561007b31SStefan Hajnoczi 101649c07526SKevin Wolf assert(is_power_of_2(align)); 101749c07526SKevin Wolf assert((offset & (align - 1)) == 0); 101849c07526SKevin Wolf assert((bytes & (align - 1)) == 0); 101961007b31SStefan Hajnoczi assert(!qiov || bytes == qiov->size); 1020abb06c5aSDaniel P. Berrange assert((bs->open_flags & BDRV_O_NO_IO) == 0); 10211a62d0acSEric Blake max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 10221a62d0acSEric Blake align); 1023a604fa2bSEric Blake 1024a604fa2bSEric Blake /* TODO: We would need a per-BDS .supported_read_flags and 1025a604fa2bSEric Blake * potential fallback support, if we ever implement any read flags 1026a604fa2bSEric Blake * to pass through to drivers. For now, there aren't any 1027a604fa2bSEric Blake * passthrough flags. */ 1028a604fa2bSEric Blake assert(!(flags & ~(BDRV_REQ_NO_SERIALISING | BDRV_REQ_COPY_ON_READ))); 102961007b31SStefan Hajnoczi 103061007b31SStefan Hajnoczi /* Handle Copy on Read and associated serialisation */ 103161007b31SStefan Hajnoczi if (flags & BDRV_REQ_COPY_ON_READ) { 103261007b31SStefan Hajnoczi /* If we touch the same cluster it counts as an overlap. This 103361007b31SStefan Hajnoczi * guarantees that allocating writes will be serialized and not race 103461007b31SStefan Hajnoczi * with each other for the same cluster. For example, in copy-on-read 103561007b31SStefan Hajnoczi * it ensures that the CoR read and write operations are atomic and 103661007b31SStefan Hajnoczi * guest writes cannot interleave between them. */ 103761007b31SStefan Hajnoczi mark_request_serialising(req, bdrv_get_cluster_size(bs)); 103861007b31SStefan Hajnoczi } 103961007b31SStefan Hajnoczi 104061408b25SFam Zheng if (!(flags & BDRV_REQ_NO_SERIALISING)) { 104161007b31SStefan Hajnoczi wait_serialising_requests(req); 104261408b25SFam Zheng } 104361007b31SStefan Hajnoczi 104461007b31SStefan Hajnoczi if (flags & BDRV_REQ_COPY_ON_READ) { 104549c07526SKevin Wolf int64_t start_sector = offset >> BDRV_SECTOR_BITS; 104649c07526SKevin Wolf int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 104749c07526SKevin Wolf unsigned int nb_sectors = end_sector - start_sector; 104861007b31SStefan Hajnoczi int pnum; 104961007b31SStefan Hajnoczi 105049c07526SKevin Wolf ret = bdrv_is_allocated(bs, start_sector, nb_sectors, &pnum); 105161007b31SStefan Hajnoczi if (ret < 0) { 105261007b31SStefan Hajnoczi goto out; 105361007b31SStefan Hajnoczi } 105461007b31SStefan Hajnoczi 105561007b31SStefan Hajnoczi if (!ret || pnum != nb_sectors) { 1056*85c97ca7SKevin Wolf ret = bdrv_co_do_copy_on_readv(child, offset, bytes, qiov); 105761007b31SStefan Hajnoczi goto out; 105861007b31SStefan Hajnoczi } 105961007b31SStefan Hajnoczi } 106061007b31SStefan Hajnoczi 10611a62d0acSEric Blake /* Forward the request to the BlockDriver, possibly fragmenting it */ 106249c07526SKevin Wolf total_bytes = bdrv_getlength(bs); 106349c07526SKevin Wolf if (total_bytes < 0) { 106449c07526SKevin Wolf ret = total_bytes; 106561007b31SStefan Hajnoczi goto out; 106661007b31SStefan Hajnoczi } 106761007b31SStefan Hajnoczi 106849c07526SKevin Wolf max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align); 10691a62d0acSEric Blake if (bytes <= max_bytes && bytes <= max_transfer) { 1070166fe960SKevin Wolf ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0); 10711a62d0acSEric Blake goto out; 107261007b31SStefan Hajnoczi } 107361007b31SStefan Hajnoczi 10741a62d0acSEric Blake while (bytes_remaining) { 10751a62d0acSEric Blake int num; 10761a62d0acSEric Blake 10771a62d0acSEric Blake if (max_bytes) { 10781a62d0acSEric Blake QEMUIOVector local_qiov; 10791a62d0acSEric Blake 10801a62d0acSEric Blake num = MIN(bytes_remaining, MIN(max_bytes, max_transfer)); 10811a62d0acSEric Blake assert(num); 10821a62d0acSEric Blake qemu_iovec_init(&local_qiov, qiov->niov); 10831a62d0acSEric Blake qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num); 10841a62d0acSEric Blake 10851a62d0acSEric Blake ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining, 10861a62d0acSEric Blake num, &local_qiov, 0); 10871a62d0acSEric Blake max_bytes -= num; 10881a62d0acSEric Blake qemu_iovec_destroy(&local_qiov); 10891a62d0acSEric Blake } else { 10901a62d0acSEric Blake num = bytes_remaining; 10911a62d0acSEric Blake ret = qemu_iovec_memset(qiov, bytes - bytes_remaining, 0, 10921a62d0acSEric Blake bytes_remaining); 10931a62d0acSEric Blake } 10941a62d0acSEric Blake if (ret < 0) { 10951a62d0acSEric Blake goto out; 10961a62d0acSEric Blake } 10971a62d0acSEric Blake bytes_remaining -= num; 109861007b31SStefan Hajnoczi } 109961007b31SStefan Hajnoczi 110061007b31SStefan Hajnoczi out: 11011a62d0acSEric Blake return ret < 0 ? ret : 0; 110261007b31SStefan Hajnoczi } 110361007b31SStefan Hajnoczi 110461007b31SStefan Hajnoczi /* 110561007b31SStefan Hajnoczi * Handle a read request in coroutine context 110661007b31SStefan Hajnoczi */ 1107a03ef88fSKevin Wolf int coroutine_fn bdrv_co_preadv(BdrvChild *child, 110861007b31SStefan Hajnoczi int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 110961007b31SStefan Hajnoczi BdrvRequestFlags flags) 111061007b31SStefan Hajnoczi { 1111a03ef88fSKevin Wolf BlockDriverState *bs = child->bs; 111261007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 111361007b31SStefan Hajnoczi BdrvTrackedRequest req; 111461007b31SStefan Hajnoczi 1115a5b8dd2cSEric Blake uint64_t align = bs->bl.request_alignment; 111661007b31SStefan Hajnoczi uint8_t *head_buf = NULL; 111761007b31SStefan Hajnoczi uint8_t *tail_buf = NULL; 111861007b31SStefan Hajnoczi QEMUIOVector local_qiov; 111961007b31SStefan Hajnoczi bool use_local_qiov = false; 112061007b31SStefan Hajnoczi int ret; 112161007b31SStefan Hajnoczi 112261007b31SStefan Hajnoczi if (!drv) { 112361007b31SStefan Hajnoczi return -ENOMEDIUM; 112461007b31SStefan Hajnoczi } 112561007b31SStefan Hajnoczi 112661007b31SStefan Hajnoczi ret = bdrv_check_byte_request(bs, offset, bytes); 112761007b31SStefan Hajnoczi if (ret < 0) { 112861007b31SStefan Hajnoczi return ret; 112961007b31SStefan Hajnoczi } 113061007b31SStefan Hajnoczi 113199723548SPaolo Bonzini bdrv_inc_in_flight(bs); 113299723548SPaolo Bonzini 11339568b511SWen Congyang /* Don't do copy-on-read if we read data before write operation */ 113461408b25SFam Zheng if (bs->copy_on_read && !(flags & BDRV_REQ_NO_SERIALISING)) { 113561007b31SStefan Hajnoczi flags |= BDRV_REQ_COPY_ON_READ; 113661007b31SStefan Hajnoczi } 113761007b31SStefan Hajnoczi 113861007b31SStefan Hajnoczi /* Align read if necessary by padding qiov */ 113961007b31SStefan Hajnoczi if (offset & (align - 1)) { 114061007b31SStefan Hajnoczi head_buf = qemu_blockalign(bs, align); 114161007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 2); 114261007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 114361007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 114461007b31SStefan Hajnoczi use_local_qiov = true; 114561007b31SStefan Hajnoczi 114661007b31SStefan Hajnoczi bytes += offset & (align - 1); 114761007b31SStefan Hajnoczi offset = offset & ~(align - 1); 114861007b31SStefan Hajnoczi } 114961007b31SStefan Hajnoczi 115061007b31SStefan Hajnoczi if ((offset + bytes) & (align - 1)) { 115161007b31SStefan Hajnoczi if (!use_local_qiov) { 115261007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 1); 115361007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 115461007b31SStefan Hajnoczi use_local_qiov = true; 115561007b31SStefan Hajnoczi } 115661007b31SStefan Hajnoczi tail_buf = qemu_blockalign(bs, align); 115761007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, tail_buf, 115861007b31SStefan Hajnoczi align - ((offset + bytes) & (align - 1))); 115961007b31SStefan Hajnoczi 116061007b31SStefan Hajnoczi bytes = ROUND_UP(bytes, align); 116161007b31SStefan Hajnoczi } 116261007b31SStefan Hajnoczi 1163ebde595cSFam Zheng tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ); 1164*85c97ca7SKevin Wolf ret = bdrv_aligned_preadv(child, &req, offset, bytes, align, 116561007b31SStefan Hajnoczi use_local_qiov ? &local_qiov : qiov, 116661007b31SStefan Hajnoczi flags); 116761007b31SStefan Hajnoczi tracked_request_end(&req); 116899723548SPaolo Bonzini bdrv_dec_in_flight(bs); 116961007b31SStefan Hajnoczi 117061007b31SStefan Hajnoczi if (use_local_qiov) { 117161007b31SStefan Hajnoczi qemu_iovec_destroy(&local_qiov); 117261007b31SStefan Hajnoczi qemu_vfree(head_buf); 117361007b31SStefan Hajnoczi qemu_vfree(tail_buf); 117461007b31SStefan Hajnoczi } 117561007b31SStefan Hajnoczi 117661007b31SStefan Hajnoczi return ret; 117761007b31SStefan Hajnoczi } 117861007b31SStefan Hajnoczi 1179adad6496SKevin Wolf static int coroutine_fn bdrv_co_do_readv(BdrvChild *child, 118061007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, 118161007b31SStefan Hajnoczi BdrvRequestFlags flags) 118261007b31SStefan Hajnoczi { 118361007b31SStefan Hajnoczi if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 118461007b31SStefan Hajnoczi return -EINVAL; 118561007b31SStefan Hajnoczi } 118661007b31SStefan Hajnoczi 1187a03ef88fSKevin Wolf return bdrv_co_preadv(child, sector_num << BDRV_SECTOR_BITS, 118861007b31SStefan Hajnoczi nb_sectors << BDRV_SECTOR_BITS, qiov, flags); 118961007b31SStefan Hajnoczi } 119061007b31SStefan Hajnoczi 119128b04a8fSKevin Wolf int coroutine_fn bdrv_co_readv(BdrvChild *child, int64_t sector_num, 119261007b31SStefan Hajnoczi int nb_sectors, QEMUIOVector *qiov) 119361007b31SStefan Hajnoczi { 119428b04a8fSKevin Wolf trace_bdrv_co_readv(child->bs, sector_num, nb_sectors); 119561007b31SStefan Hajnoczi 1196adad6496SKevin Wolf return bdrv_co_do_readv(child, sector_num, nb_sectors, qiov, 0); 119761007b31SStefan Hajnoczi } 119861007b31SStefan Hajnoczi 11995def6b80SEric Blake /* Maximum buffer for write zeroes fallback, in bytes */ 12005def6b80SEric Blake #define MAX_WRITE_ZEROES_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS) 120161007b31SStefan Hajnoczi 1202d05aa8bbSEric Blake static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 1203d05aa8bbSEric Blake int64_t offset, int count, BdrvRequestFlags flags) 120461007b31SStefan Hajnoczi { 120561007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 120661007b31SStefan Hajnoczi QEMUIOVector qiov; 120761007b31SStefan Hajnoczi struct iovec iov = {0}; 120861007b31SStefan Hajnoczi int ret = 0; 1209465fe887SEric Blake bool need_flush = false; 1210443668caSDenis V. Lunev int head = 0; 1211443668caSDenis V. Lunev int tail = 0; 121261007b31SStefan Hajnoczi 1213cf081fcaSEric Blake int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, INT_MAX); 1214a5b8dd2cSEric Blake int alignment = MAX(bs->bl.pwrite_zeroes_alignment, 1215a5b8dd2cSEric Blake bs->bl.request_alignment); 1216b2f95feeSEric Blake int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, 1217b2f95feeSEric Blake MAX_WRITE_ZEROES_BOUNCE_BUFFER); 1218cf081fcaSEric Blake 1219b8d0a980SEric Blake assert(alignment % bs->bl.request_alignment == 0); 1220b8d0a980SEric Blake head = offset % alignment; 1221b8d0a980SEric Blake tail = (offset + count) % alignment; 1222b8d0a980SEric Blake max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment); 1223b8d0a980SEric Blake assert(max_write_zeroes >= bs->bl.request_alignment); 122461007b31SStefan Hajnoczi 1225d05aa8bbSEric Blake while (count > 0 && !ret) { 1226d05aa8bbSEric Blake int num = count; 122761007b31SStefan Hajnoczi 122861007b31SStefan Hajnoczi /* Align request. Block drivers can expect the "bulk" of the request 1229443668caSDenis V. Lunev * to be aligned, and that unaligned requests do not cross cluster 1230443668caSDenis V. Lunev * boundaries. 123161007b31SStefan Hajnoczi */ 1232443668caSDenis V. Lunev if (head) { 1233b2f95feeSEric Blake /* Make a small request up to the first aligned sector. For 1234b2f95feeSEric Blake * convenience, limit this request to max_transfer even if 1235b2f95feeSEric Blake * we don't need to fall back to writes. */ 1236b2f95feeSEric Blake num = MIN(MIN(count, max_transfer), alignment - head); 1237b2f95feeSEric Blake head = (head + num) % alignment; 1238b2f95feeSEric Blake assert(num < max_write_zeroes); 1239d05aa8bbSEric Blake } else if (tail && num > alignment) { 1240443668caSDenis V. Lunev /* Shorten the request to the last aligned sector. */ 1241443668caSDenis V. Lunev num -= tail; 124261007b31SStefan Hajnoczi } 124361007b31SStefan Hajnoczi 124461007b31SStefan Hajnoczi /* limit request size */ 124561007b31SStefan Hajnoczi if (num > max_write_zeroes) { 124661007b31SStefan Hajnoczi num = max_write_zeroes; 124761007b31SStefan Hajnoczi } 124861007b31SStefan Hajnoczi 124961007b31SStefan Hajnoczi ret = -ENOTSUP; 125061007b31SStefan Hajnoczi /* First try the efficient write zeroes operation */ 1251d05aa8bbSEric Blake if (drv->bdrv_co_pwrite_zeroes) { 1252d05aa8bbSEric Blake ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num, 1253d05aa8bbSEric Blake flags & bs->supported_zero_flags); 1254d05aa8bbSEric Blake if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) && 1255d05aa8bbSEric Blake !(bs->supported_zero_flags & BDRV_REQ_FUA)) { 1256d05aa8bbSEric Blake need_flush = true; 1257d05aa8bbSEric Blake } 1258465fe887SEric Blake } else { 1259465fe887SEric Blake assert(!bs->supported_zero_flags); 126061007b31SStefan Hajnoczi } 126161007b31SStefan Hajnoczi 126261007b31SStefan Hajnoczi if (ret == -ENOTSUP) { 126361007b31SStefan Hajnoczi /* Fall back to bounce buffer if write zeroes is unsupported */ 1264465fe887SEric Blake BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE; 1265465fe887SEric Blake 1266465fe887SEric Blake if ((flags & BDRV_REQ_FUA) && 1267465fe887SEric Blake !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1268465fe887SEric Blake /* No need for bdrv_driver_pwrite() to do a fallback 1269465fe887SEric Blake * flush on each chunk; use just one at the end */ 1270465fe887SEric Blake write_flags &= ~BDRV_REQ_FUA; 1271465fe887SEric Blake need_flush = true; 1272465fe887SEric Blake } 12735def6b80SEric Blake num = MIN(num, max_transfer); 1274d05aa8bbSEric Blake iov.iov_len = num; 127561007b31SStefan Hajnoczi if (iov.iov_base == NULL) { 1276d05aa8bbSEric Blake iov.iov_base = qemu_try_blockalign(bs, num); 127761007b31SStefan Hajnoczi if (iov.iov_base == NULL) { 127861007b31SStefan Hajnoczi ret = -ENOMEM; 127961007b31SStefan Hajnoczi goto fail; 128061007b31SStefan Hajnoczi } 1281d05aa8bbSEric Blake memset(iov.iov_base, 0, num); 128261007b31SStefan Hajnoczi } 128361007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 128461007b31SStefan Hajnoczi 1285d05aa8bbSEric Blake ret = bdrv_driver_pwritev(bs, offset, num, &qiov, write_flags); 128661007b31SStefan Hajnoczi 128761007b31SStefan Hajnoczi /* Keep bounce buffer around if it is big enough for all 128861007b31SStefan Hajnoczi * all future requests. 128961007b31SStefan Hajnoczi */ 12905def6b80SEric Blake if (num < max_transfer) { 129161007b31SStefan Hajnoczi qemu_vfree(iov.iov_base); 129261007b31SStefan Hajnoczi iov.iov_base = NULL; 129361007b31SStefan Hajnoczi } 129461007b31SStefan Hajnoczi } 129561007b31SStefan Hajnoczi 1296d05aa8bbSEric Blake offset += num; 1297d05aa8bbSEric Blake count -= num; 129861007b31SStefan Hajnoczi } 129961007b31SStefan Hajnoczi 130061007b31SStefan Hajnoczi fail: 1301465fe887SEric Blake if (ret == 0 && need_flush) { 1302465fe887SEric Blake ret = bdrv_co_flush(bs); 1303465fe887SEric Blake } 130461007b31SStefan Hajnoczi qemu_vfree(iov.iov_base); 130561007b31SStefan Hajnoczi return ret; 130661007b31SStefan Hajnoczi } 130761007b31SStefan Hajnoczi 130861007b31SStefan Hajnoczi /* 130904ed95f4SEric Blake * Forwards an already correctly aligned write request to the BlockDriver, 131004ed95f4SEric Blake * after possibly fragmenting it. 131161007b31SStefan Hajnoczi */ 1312*85c97ca7SKevin Wolf static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child, 131361007b31SStefan Hajnoczi BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 1314cff86b38SEric Blake int64_t align, QEMUIOVector *qiov, int flags) 131561007b31SStefan Hajnoczi { 1316*85c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 131761007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 131861007b31SStefan Hajnoczi bool waited; 131961007b31SStefan Hajnoczi int ret; 132061007b31SStefan Hajnoczi 13219896c876SKevin Wolf int64_t start_sector = offset >> BDRV_SECTOR_BITS; 13229896c876SKevin Wolf int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 132304ed95f4SEric Blake uint64_t bytes_remaining = bytes; 132404ed95f4SEric Blake int max_transfer; 132561007b31SStefan Hajnoczi 1326cff86b38SEric Blake assert(is_power_of_2(align)); 1327cff86b38SEric Blake assert((offset & (align - 1)) == 0); 1328cff86b38SEric Blake assert((bytes & (align - 1)) == 0); 132961007b31SStefan Hajnoczi assert(!qiov || bytes == qiov->size); 1330abb06c5aSDaniel P. Berrange assert((bs->open_flags & BDRV_O_NO_IO) == 0); 1331fa166538SEric Blake assert(!(flags & ~BDRV_REQ_MASK)); 133204ed95f4SEric Blake max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 133304ed95f4SEric Blake align); 133461007b31SStefan Hajnoczi 133561007b31SStefan Hajnoczi waited = wait_serialising_requests(req); 133661007b31SStefan Hajnoczi assert(!waited || !req->serialising); 133761007b31SStefan Hajnoczi assert(req->overlap_offset <= offset); 133861007b31SStefan Hajnoczi assert(offset + bytes <= req->overlap_offset + req->overlap_bytes); 133961007b31SStefan Hajnoczi 134061007b31SStefan Hajnoczi ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req); 134161007b31SStefan Hajnoczi 134261007b31SStefan Hajnoczi if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF && 1343c1499a5eSEric Blake !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes && 134461007b31SStefan Hajnoczi qemu_iovec_is_zero(qiov)) { 134561007b31SStefan Hajnoczi flags |= BDRV_REQ_ZERO_WRITE; 134661007b31SStefan Hajnoczi if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) { 134761007b31SStefan Hajnoczi flags |= BDRV_REQ_MAY_UNMAP; 134861007b31SStefan Hajnoczi } 134961007b31SStefan Hajnoczi } 135061007b31SStefan Hajnoczi 135161007b31SStefan Hajnoczi if (ret < 0) { 135261007b31SStefan Hajnoczi /* Do nothing, write notifier decided to fail this request */ 135361007b31SStefan Hajnoczi } else if (flags & BDRV_REQ_ZERO_WRITE) { 13549a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO); 13559896c876SKevin Wolf ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags); 13563ea1a091SPavel Butsykin } else if (flags & BDRV_REQ_WRITE_COMPRESSED) { 13573ea1a091SPavel Butsykin ret = bdrv_driver_pwritev_compressed(bs, offset, bytes, qiov); 135804ed95f4SEric Blake } else if (bytes <= max_transfer) { 13599a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV); 136078a07294SKevin Wolf ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, flags); 136104ed95f4SEric Blake } else { 136204ed95f4SEric Blake bdrv_debug_event(bs, BLKDBG_PWRITEV); 136304ed95f4SEric Blake while (bytes_remaining) { 136404ed95f4SEric Blake int num = MIN(bytes_remaining, max_transfer); 136504ed95f4SEric Blake QEMUIOVector local_qiov; 136604ed95f4SEric Blake int local_flags = flags; 136704ed95f4SEric Blake 136804ed95f4SEric Blake assert(num); 136904ed95f4SEric Blake if (num < bytes_remaining && (flags & BDRV_REQ_FUA) && 137004ed95f4SEric Blake !(bs->supported_write_flags & BDRV_REQ_FUA)) { 137104ed95f4SEric Blake /* If FUA is going to be emulated by flush, we only 137204ed95f4SEric Blake * need to flush on the last iteration */ 137304ed95f4SEric Blake local_flags &= ~BDRV_REQ_FUA; 137404ed95f4SEric Blake } 137504ed95f4SEric Blake qemu_iovec_init(&local_qiov, qiov->niov); 137604ed95f4SEric Blake qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num); 137704ed95f4SEric Blake 137804ed95f4SEric Blake ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining, 137904ed95f4SEric Blake num, &local_qiov, local_flags); 138004ed95f4SEric Blake qemu_iovec_destroy(&local_qiov); 138104ed95f4SEric Blake if (ret < 0) { 138204ed95f4SEric Blake break; 138304ed95f4SEric Blake } 138404ed95f4SEric Blake bytes_remaining -= num; 138504ed95f4SEric Blake } 138661007b31SStefan Hajnoczi } 13879a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE); 138861007b31SStefan Hajnoczi 13893ff2f67aSEvgeny Yakovlev ++bs->write_gen; 13909896c876SKevin Wolf bdrv_set_dirty(bs, start_sector, end_sector - start_sector); 139161007b31SStefan Hajnoczi 139253d8f9d8SMax Reitz if (bs->wr_highest_offset < offset + bytes) { 139353d8f9d8SMax Reitz bs->wr_highest_offset = offset + bytes; 139453d8f9d8SMax Reitz } 139561007b31SStefan Hajnoczi 139661007b31SStefan Hajnoczi if (ret >= 0) { 13979896c876SKevin Wolf bs->total_sectors = MAX(bs->total_sectors, end_sector); 139804ed95f4SEric Blake ret = 0; 139961007b31SStefan Hajnoczi } 140061007b31SStefan Hajnoczi 140161007b31SStefan Hajnoczi return ret; 140261007b31SStefan Hajnoczi } 140361007b31SStefan Hajnoczi 1404*85c97ca7SKevin Wolf static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child, 14059eeb6dd1SFam Zheng int64_t offset, 14069eeb6dd1SFam Zheng unsigned int bytes, 14079eeb6dd1SFam Zheng BdrvRequestFlags flags, 14089eeb6dd1SFam Zheng BdrvTrackedRequest *req) 14099eeb6dd1SFam Zheng { 1410*85c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 14119eeb6dd1SFam Zheng uint8_t *buf = NULL; 14129eeb6dd1SFam Zheng QEMUIOVector local_qiov; 14139eeb6dd1SFam Zheng struct iovec iov; 1414a5b8dd2cSEric Blake uint64_t align = bs->bl.request_alignment; 14159eeb6dd1SFam Zheng unsigned int head_padding_bytes, tail_padding_bytes; 14169eeb6dd1SFam Zheng int ret = 0; 14179eeb6dd1SFam Zheng 14189eeb6dd1SFam Zheng head_padding_bytes = offset & (align - 1); 14199eeb6dd1SFam Zheng tail_padding_bytes = align - ((offset + bytes) & (align - 1)); 14209eeb6dd1SFam Zheng 14219eeb6dd1SFam Zheng 14229eeb6dd1SFam Zheng assert(flags & BDRV_REQ_ZERO_WRITE); 14239eeb6dd1SFam Zheng if (head_padding_bytes || tail_padding_bytes) { 14249eeb6dd1SFam Zheng buf = qemu_blockalign(bs, align); 14259eeb6dd1SFam Zheng iov = (struct iovec) { 14269eeb6dd1SFam Zheng .iov_base = buf, 14279eeb6dd1SFam Zheng .iov_len = align, 14289eeb6dd1SFam Zheng }; 14299eeb6dd1SFam Zheng qemu_iovec_init_external(&local_qiov, &iov, 1); 14309eeb6dd1SFam Zheng } 14319eeb6dd1SFam Zheng if (head_padding_bytes) { 14329eeb6dd1SFam Zheng uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes); 14339eeb6dd1SFam Zheng 14349eeb6dd1SFam Zheng /* RMW the unaligned part before head. */ 14359eeb6dd1SFam Zheng mark_request_serialising(req, align); 14369eeb6dd1SFam Zheng wait_serialising_requests(req); 14379a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 1438*85c97ca7SKevin Wolf ret = bdrv_aligned_preadv(child, req, offset & ~(align - 1), align, 14399eeb6dd1SFam Zheng align, &local_qiov, 0); 14409eeb6dd1SFam Zheng if (ret < 0) { 14419eeb6dd1SFam Zheng goto fail; 14429eeb6dd1SFam Zheng } 14439a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 14449eeb6dd1SFam Zheng 14459eeb6dd1SFam Zheng memset(buf + head_padding_bytes, 0, zero_bytes); 1446*85c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, req, offset & ~(align - 1), align, 1447cff86b38SEric Blake align, &local_qiov, 14489eeb6dd1SFam Zheng flags & ~BDRV_REQ_ZERO_WRITE); 14499eeb6dd1SFam Zheng if (ret < 0) { 14509eeb6dd1SFam Zheng goto fail; 14519eeb6dd1SFam Zheng } 14529eeb6dd1SFam Zheng offset += zero_bytes; 14539eeb6dd1SFam Zheng bytes -= zero_bytes; 14549eeb6dd1SFam Zheng } 14559eeb6dd1SFam Zheng 14569eeb6dd1SFam Zheng assert(!bytes || (offset & (align - 1)) == 0); 14579eeb6dd1SFam Zheng if (bytes >= align) { 14589eeb6dd1SFam Zheng /* Write the aligned part in the middle. */ 14599eeb6dd1SFam Zheng uint64_t aligned_bytes = bytes & ~(align - 1); 1460*85c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align, 14619eeb6dd1SFam Zheng NULL, flags); 14629eeb6dd1SFam Zheng if (ret < 0) { 14639eeb6dd1SFam Zheng goto fail; 14649eeb6dd1SFam Zheng } 14659eeb6dd1SFam Zheng bytes -= aligned_bytes; 14669eeb6dd1SFam Zheng offset += aligned_bytes; 14679eeb6dd1SFam Zheng } 14689eeb6dd1SFam Zheng 14699eeb6dd1SFam Zheng assert(!bytes || (offset & (align - 1)) == 0); 14709eeb6dd1SFam Zheng if (bytes) { 14719eeb6dd1SFam Zheng assert(align == tail_padding_bytes + bytes); 14729eeb6dd1SFam Zheng /* RMW the unaligned part after tail. */ 14739eeb6dd1SFam Zheng mark_request_serialising(req, align); 14749eeb6dd1SFam Zheng wait_serialising_requests(req); 14759a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1476*85c97ca7SKevin Wolf ret = bdrv_aligned_preadv(child, req, offset, align, 14779eeb6dd1SFam Zheng align, &local_qiov, 0); 14789eeb6dd1SFam Zheng if (ret < 0) { 14799eeb6dd1SFam Zheng goto fail; 14809eeb6dd1SFam Zheng } 14819a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 14829eeb6dd1SFam Zheng 14839eeb6dd1SFam Zheng memset(buf, 0, bytes); 1484*85c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, req, offset, align, align, 14859eeb6dd1SFam Zheng &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE); 14869eeb6dd1SFam Zheng } 14879eeb6dd1SFam Zheng fail: 14889eeb6dd1SFam Zheng qemu_vfree(buf); 14899eeb6dd1SFam Zheng return ret; 14909eeb6dd1SFam Zheng 14919eeb6dd1SFam Zheng } 14929eeb6dd1SFam Zheng 149361007b31SStefan Hajnoczi /* 149461007b31SStefan Hajnoczi * Handle a write request in coroutine context 149561007b31SStefan Hajnoczi */ 1496a03ef88fSKevin Wolf int coroutine_fn bdrv_co_pwritev(BdrvChild *child, 149761007b31SStefan Hajnoczi int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 149861007b31SStefan Hajnoczi BdrvRequestFlags flags) 149961007b31SStefan Hajnoczi { 1500a03ef88fSKevin Wolf BlockDriverState *bs = child->bs; 150161007b31SStefan Hajnoczi BdrvTrackedRequest req; 1502a5b8dd2cSEric Blake uint64_t align = bs->bl.request_alignment; 150361007b31SStefan Hajnoczi uint8_t *head_buf = NULL; 150461007b31SStefan Hajnoczi uint8_t *tail_buf = NULL; 150561007b31SStefan Hajnoczi QEMUIOVector local_qiov; 150661007b31SStefan Hajnoczi bool use_local_qiov = false; 150761007b31SStefan Hajnoczi int ret; 150861007b31SStefan Hajnoczi 150961007b31SStefan Hajnoczi if (!bs->drv) { 151061007b31SStefan Hajnoczi return -ENOMEDIUM; 151161007b31SStefan Hajnoczi } 151261007b31SStefan Hajnoczi if (bs->read_only) { 1513eaf5fe2dSPaolo Bonzini return -EPERM; 151461007b31SStefan Hajnoczi } 151504c01a5cSKevin Wolf assert(!(bs->open_flags & BDRV_O_INACTIVE)); 151661007b31SStefan Hajnoczi 151761007b31SStefan Hajnoczi ret = bdrv_check_byte_request(bs, offset, bytes); 151861007b31SStefan Hajnoczi if (ret < 0) { 151961007b31SStefan Hajnoczi return ret; 152061007b31SStefan Hajnoczi } 152161007b31SStefan Hajnoczi 152299723548SPaolo Bonzini bdrv_inc_in_flight(bs); 152361007b31SStefan Hajnoczi /* 152461007b31SStefan Hajnoczi * Align write if necessary by performing a read-modify-write cycle. 152561007b31SStefan Hajnoczi * Pad qiov with the read parts and be sure to have a tracked request not 152661007b31SStefan Hajnoczi * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle. 152761007b31SStefan Hajnoczi */ 1528ebde595cSFam Zheng tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE); 152961007b31SStefan Hajnoczi 15309eeb6dd1SFam Zheng if (!qiov) { 1531*85c97ca7SKevin Wolf ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req); 15329eeb6dd1SFam Zheng goto out; 15339eeb6dd1SFam Zheng } 15349eeb6dd1SFam Zheng 153561007b31SStefan Hajnoczi if (offset & (align - 1)) { 153661007b31SStefan Hajnoczi QEMUIOVector head_qiov; 153761007b31SStefan Hajnoczi struct iovec head_iov; 153861007b31SStefan Hajnoczi 153961007b31SStefan Hajnoczi mark_request_serialising(&req, align); 154061007b31SStefan Hajnoczi wait_serialising_requests(&req); 154161007b31SStefan Hajnoczi 154261007b31SStefan Hajnoczi head_buf = qemu_blockalign(bs, align); 154361007b31SStefan Hajnoczi head_iov = (struct iovec) { 154461007b31SStefan Hajnoczi .iov_base = head_buf, 154561007b31SStefan Hajnoczi .iov_len = align, 154661007b31SStefan Hajnoczi }; 154761007b31SStefan Hajnoczi qemu_iovec_init_external(&head_qiov, &head_iov, 1); 154861007b31SStefan Hajnoczi 15499a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 1550*85c97ca7SKevin Wolf ret = bdrv_aligned_preadv(child, &req, offset & ~(align - 1), align, 155161007b31SStefan Hajnoczi align, &head_qiov, 0); 155261007b31SStefan Hajnoczi if (ret < 0) { 155361007b31SStefan Hajnoczi goto fail; 155461007b31SStefan Hajnoczi } 15559a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 155661007b31SStefan Hajnoczi 155761007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 2); 155861007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 155961007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 156061007b31SStefan Hajnoczi use_local_qiov = true; 156161007b31SStefan Hajnoczi 156261007b31SStefan Hajnoczi bytes += offset & (align - 1); 156361007b31SStefan Hajnoczi offset = offset & ~(align - 1); 1564117bc3faSPeter Lieven 1565117bc3faSPeter Lieven /* We have read the tail already if the request is smaller 1566117bc3faSPeter Lieven * than one aligned block. 1567117bc3faSPeter Lieven */ 1568117bc3faSPeter Lieven if (bytes < align) { 1569117bc3faSPeter Lieven qemu_iovec_add(&local_qiov, head_buf + bytes, align - bytes); 1570117bc3faSPeter Lieven bytes = align; 1571117bc3faSPeter Lieven } 157261007b31SStefan Hajnoczi } 157361007b31SStefan Hajnoczi 157461007b31SStefan Hajnoczi if ((offset + bytes) & (align - 1)) { 157561007b31SStefan Hajnoczi QEMUIOVector tail_qiov; 157661007b31SStefan Hajnoczi struct iovec tail_iov; 157761007b31SStefan Hajnoczi size_t tail_bytes; 157861007b31SStefan Hajnoczi bool waited; 157961007b31SStefan Hajnoczi 158061007b31SStefan Hajnoczi mark_request_serialising(&req, align); 158161007b31SStefan Hajnoczi waited = wait_serialising_requests(&req); 158261007b31SStefan Hajnoczi assert(!waited || !use_local_qiov); 158361007b31SStefan Hajnoczi 158461007b31SStefan Hajnoczi tail_buf = qemu_blockalign(bs, align); 158561007b31SStefan Hajnoczi tail_iov = (struct iovec) { 158661007b31SStefan Hajnoczi .iov_base = tail_buf, 158761007b31SStefan Hajnoczi .iov_len = align, 158861007b31SStefan Hajnoczi }; 158961007b31SStefan Hajnoczi qemu_iovec_init_external(&tail_qiov, &tail_iov, 1); 159061007b31SStefan Hajnoczi 15919a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1592*85c97ca7SKevin Wolf ret = bdrv_aligned_preadv(child, &req, (offset + bytes) & ~(align - 1), 1593*85c97ca7SKevin Wolf align, align, &tail_qiov, 0); 159461007b31SStefan Hajnoczi if (ret < 0) { 159561007b31SStefan Hajnoczi goto fail; 159661007b31SStefan Hajnoczi } 15979a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 159861007b31SStefan Hajnoczi 159961007b31SStefan Hajnoczi if (!use_local_qiov) { 160061007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 1); 160161007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 160261007b31SStefan Hajnoczi use_local_qiov = true; 160361007b31SStefan Hajnoczi } 160461007b31SStefan Hajnoczi 160561007b31SStefan Hajnoczi tail_bytes = (offset + bytes) & (align - 1); 160661007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes); 160761007b31SStefan Hajnoczi 160861007b31SStefan Hajnoczi bytes = ROUND_UP(bytes, align); 160961007b31SStefan Hajnoczi } 161061007b31SStefan Hajnoczi 1611*85c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align, 161261007b31SStefan Hajnoczi use_local_qiov ? &local_qiov : qiov, 161361007b31SStefan Hajnoczi flags); 161461007b31SStefan Hajnoczi 161561007b31SStefan Hajnoczi fail: 161661007b31SStefan Hajnoczi 161761007b31SStefan Hajnoczi if (use_local_qiov) { 161861007b31SStefan Hajnoczi qemu_iovec_destroy(&local_qiov); 161961007b31SStefan Hajnoczi } 162061007b31SStefan Hajnoczi qemu_vfree(head_buf); 162161007b31SStefan Hajnoczi qemu_vfree(tail_buf); 16229eeb6dd1SFam Zheng out: 16239eeb6dd1SFam Zheng tracked_request_end(&req); 162499723548SPaolo Bonzini bdrv_dec_in_flight(bs); 162561007b31SStefan Hajnoczi return ret; 162661007b31SStefan Hajnoczi } 162761007b31SStefan Hajnoczi 1628adad6496SKevin Wolf static int coroutine_fn bdrv_co_do_writev(BdrvChild *child, 162961007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, 163061007b31SStefan Hajnoczi BdrvRequestFlags flags) 163161007b31SStefan Hajnoczi { 163261007b31SStefan Hajnoczi if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 163361007b31SStefan Hajnoczi return -EINVAL; 163461007b31SStefan Hajnoczi } 163561007b31SStefan Hajnoczi 1636a03ef88fSKevin Wolf return bdrv_co_pwritev(child, sector_num << BDRV_SECTOR_BITS, 163761007b31SStefan Hajnoczi nb_sectors << BDRV_SECTOR_BITS, qiov, flags); 163861007b31SStefan Hajnoczi } 163961007b31SStefan Hajnoczi 164025ec177dSKevin Wolf int coroutine_fn bdrv_co_writev(BdrvChild *child, int64_t sector_num, 164161007b31SStefan Hajnoczi int nb_sectors, QEMUIOVector *qiov) 164261007b31SStefan Hajnoczi { 164325ec177dSKevin Wolf trace_bdrv_co_writev(child->bs, sector_num, nb_sectors); 164461007b31SStefan Hajnoczi 1645adad6496SKevin Wolf return bdrv_co_do_writev(child, sector_num, nb_sectors, qiov, 0); 164661007b31SStefan Hajnoczi } 164761007b31SStefan Hajnoczi 1648a03ef88fSKevin Wolf int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset, 1649a03ef88fSKevin Wolf int count, BdrvRequestFlags flags) 165061007b31SStefan Hajnoczi { 1651a03ef88fSKevin Wolf trace_bdrv_co_pwrite_zeroes(child->bs, offset, count, flags); 165261007b31SStefan Hajnoczi 1653a03ef88fSKevin Wolf if (!(child->bs->open_flags & BDRV_O_UNMAP)) { 165461007b31SStefan Hajnoczi flags &= ~BDRV_REQ_MAY_UNMAP; 165561007b31SStefan Hajnoczi } 165661007b31SStefan Hajnoczi 1657a03ef88fSKevin Wolf return bdrv_co_pwritev(child, offset, count, NULL, 165861007b31SStefan Hajnoczi BDRV_REQ_ZERO_WRITE | flags); 165961007b31SStefan Hajnoczi } 166061007b31SStefan Hajnoczi 16614085f5c7SJohn Snow /* 16624085f5c7SJohn Snow * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not. 16634085f5c7SJohn Snow */ 16644085f5c7SJohn Snow int bdrv_flush_all(void) 16654085f5c7SJohn Snow { 16664085f5c7SJohn Snow BdrvNextIterator it; 16674085f5c7SJohn Snow BlockDriverState *bs = NULL; 16684085f5c7SJohn Snow int result = 0; 16694085f5c7SJohn Snow 16704085f5c7SJohn Snow for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 16714085f5c7SJohn Snow AioContext *aio_context = bdrv_get_aio_context(bs); 16724085f5c7SJohn Snow int ret; 16734085f5c7SJohn Snow 16744085f5c7SJohn Snow aio_context_acquire(aio_context); 16754085f5c7SJohn Snow ret = bdrv_flush(bs); 16764085f5c7SJohn Snow if (ret < 0 && !result) { 16774085f5c7SJohn Snow result = ret; 16784085f5c7SJohn Snow } 16794085f5c7SJohn Snow aio_context_release(aio_context); 16804085f5c7SJohn Snow } 16814085f5c7SJohn Snow 16824085f5c7SJohn Snow return result; 16834085f5c7SJohn Snow } 16844085f5c7SJohn Snow 16854085f5c7SJohn Snow 168661007b31SStefan Hajnoczi typedef struct BdrvCoGetBlockStatusData { 168761007b31SStefan Hajnoczi BlockDriverState *bs; 168861007b31SStefan Hajnoczi BlockDriverState *base; 168967a0fd2aSFam Zheng BlockDriverState **file; 169061007b31SStefan Hajnoczi int64_t sector_num; 169161007b31SStefan Hajnoczi int nb_sectors; 169261007b31SStefan Hajnoczi int *pnum; 169361007b31SStefan Hajnoczi int64_t ret; 169461007b31SStefan Hajnoczi bool done; 169561007b31SStefan Hajnoczi } BdrvCoGetBlockStatusData; 169661007b31SStefan Hajnoczi 169761007b31SStefan Hajnoczi /* 169861007b31SStefan Hajnoczi * Returns the allocation status of the specified sectors. 169961007b31SStefan Hajnoczi * Drivers not implementing the functionality are assumed to not support 170061007b31SStefan Hajnoczi * backing files, hence all their sectors are reported as allocated. 170161007b31SStefan Hajnoczi * 170261007b31SStefan Hajnoczi * If 'sector_num' is beyond the end of the disk image the return value is 0 170361007b31SStefan Hajnoczi * and 'pnum' is set to 0. 170461007b31SStefan Hajnoczi * 170561007b31SStefan Hajnoczi * 'pnum' is set to the number of sectors (including and immediately following 170661007b31SStefan Hajnoczi * the specified sector) that are known to be in the same 170761007b31SStefan Hajnoczi * allocated/unallocated state. 170861007b31SStefan Hajnoczi * 170961007b31SStefan Hajnoczi * 'nb_sectors' is the max value 'pnum' should be set to. If nb_sectors goes 171061007b31SStefan Hajnoczi * beyond the end of the disk image it will be clamped. 171167a0fd2aSFam Zheng * 171267a0fd2aSFam Zheng * If returned value is positive and BDRV_BLOCK_OFFSET_VALID bit is set, 'file' 171367a0fd2aSFam Zheng * points to the BDS which the sector range is allocated in. 171461007b31SStefan Hajnoczi */ 171561007b31SStefan Hajnoczi static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs, 171661007b31SStefan Hajnoczi int64_t sector_num, 171767a0fd2aSFam Zheng int nb_sectors, int *pnum, 171867a0fd2aSFam Zheng BlockDriverState **file) 171961007b31SStefan Hajnoczi { 172061007b31SStefan Hajnoczi int64_t total_sectors; 172161007b31SStefan Hajnoczi int64_t n; 172261007b31SStefan Hajnoczi int64_t ret, ret2; 172361007b31SStefan Hajnoczi 172461007b31SStefan Hajnoczi total_sectors = bdrv_nb_sectors(bs); 172561007b31SStefan Hajnoczi if (total_sectors < 0) { 172661007b31SStefan Hajnoczi return total_sectors; 172761007b31SStefan Hajnoczi } 172861007b31SStefan Hajnoczi 172961007b31SStefan Hajnoczi if (sector_num >= total_sectors) { 173061007b31SStefan Hajnoczi *pnum = 0; 173161007b31SStefan Hajnoczi return 0; 173261007b31SStefan Hajnoczi } 173361007b31SStefan Hajnoczi 173461007b31SStefan Hajnoczi n = total_sectors - sector_num; 173561007b31SStefan Hajnoczi if (n < nb_sectors) { 173661007b31SStefan Hajnoczi nb_sectors = n; 173761007b31SStefan Hajnoczi } 173861007b31SStefan Hajnoczi 173961007b31SStefan Hajnoczi if (!bs->drv->bdrv_co_get_block_status) { 174061007b31SStefan Hajnoczi *pnum = nb_sectors; 174161007b31SStefan Hajnoczi ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED; 174261007b31SStefan Hajnoczi if (bs->drv->protocol_name) { 174361007b31SStefan Hajnoczi ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE); 174461007b31SStefan Hajnoczi } 174561007b31SStefan Hajnoczi return ret; 174661007b31SStefan Hajnoczi } 174761007b31SStefan Hajnoczi 174867a0fd2aSFam Zheng *file = NULL; 174999723548SPaolo Bonzini bdrv_inc_in_flight(bs); 175067a0fd2aSFam Zheng ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum, 175167a0fd2aSFam Zheng file); 175261007b31SStefan Hajnoczi if (ret < 0) { 175361007b31SStefan Hajnoczi *pnum = 0; 175499723548SPaolo Bonzini goto out; 175561007b31SStefan Hajnoczi } 175661007b31SStefan Hajnoczi 175761007b31SStefan Hajnoczi if (ret & BDRV_BLOCK_RAW) { 175861007b31SStefan Hajnoczi assert(ret & BDRV_BLOCK_OFFSET_VALID); 175999723548SPaolo Bonzini ret = bdrv_get_block_status(bs->file->bs, ret >> BDRV_SECTOR_BITS, 176067a0fd2aSFam Zheng *pnum, pnum, file); 176199723548SPaolo Bonzini goto out; 176261007b31SStefan Hajnoczi } 176361007b31SStefan Hajnoczi 176461007b31SStefan Hajnoczi if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) { 176561007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ALLOCATED; 1766a53f1a95SPaolo Bonzini } else { 176761007b31SStefan Hajnoczi if (bdrv_unallocated_blocks_are_zero(bs)) { 176861007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 1769760e0063SKevin Wolf } else if (bs->backing) { 1770760e0063SKevin Wolf BlockDriverState *bs2 = bs->backing->bs; 177161007b31SStefan Hajnoczi int64_t nb_sectors2 = bdrv_nb_sectors(bs2); 177261007b31SStefan Hajnoczi if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) { 177361007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 177461007b31SStefan Hajnoczi } 177561007b31SStefan Hajnoczi } 177661007b31SStefan Hajnoczi } 177761007b31SStefan Hajnoczi 1778ac987b30SFam Zheng if (*file && *file != bs && 177961007b31SStefan Hajnoczi (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) && 178061007b31SStefan Hajnoczi (ret & BDRV_BLOCK_OFFSET_VALID)) { 178167a0fd2aSFam Zheng BlockDriverState *file2; 178261007b31SStefan Hajnoczi int file_pnum; 178361007b31SStefan Hajnoczi 1784ac987b30SFam Zheng ret2 = bdrv_co_get_block_status(*file, ret >> BDRV_SECTOR_BITS, 178567a0fd2aSFam Zheng *pnum, &file_pnum, &file2); 178661007b31SStefan Hajnoczi if (ret2 >= 0) { 178761007b31SStefan Hajnoczi /* Ignore errors. This is just providing extra information, it 178861007b31SStefan Hajnoczi * is useful but not necessary. 178961007b31SStefan Hajnoczi */ 179061007b31SStefan Hajnoczi if (!file_pnum) { 179161007b31SStefan Hajnoczi /* !file_pnum indicates an offset at or beyond the EOF; it is 179261007b31SStefan Hajnoczi * perfectly valid for the format block driver to point to such 179361007b31SStefan Hajnoczi * offsets, so catch it and mark everything as zero */ 179461007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 179561007b31SStefan Hajnoczi } else { 179661007b31SStefan Hajnoczi /* Limit request to the range reported by the protocol driver */ 179761007b31SStefan Hajnoczi *pnum = file_pnum; 179861007b31SStefan Hajnoczi ret |= (ret2 & BDRV_BLOCK_ZERO); 179961007b31SStefan Hajnoczi } 180061007b31SStefan Hajnoczi } 180161007b31SStefan Hajnoczi } 180261007b31SStefan Hajnoczi 180399723548SPaolo Bonzini out: 180499723548SPaolo Bonzini bdrv_dec_in_flight(bs); 180561007b31SStefan Hajnoczi return ret; 180661007b31SStefan Hajnoczi } 180761007b31SStefan Hajnoczi 1808ba3f0e25SFam Zheng static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs, 1809ba3f0e25SFam Zheng BlockDriverState *base, 1810ba3f0e25SFam Zheng int64_t sector_num, 1811ba3f0e25SFam Zheng int nb_sectors, 181267a0fd2aSFam Zheng int *pnum, 181367a0fd2aSFam Zheng BlockDriverState **file) 1814ba3f0e25SFam Zheng { 1815ba3f0e25SFam Zheng BlockDriverState *p; 1816ba3f0e25SFam Zheng int64_t ret = 0; 1817ba3f0e25SFam Zheng 1818ba3f0e25SFam Zheng assert(bs != base); 1819760e0063SKevin Wolf for (p = bs; p != base; p = backing_bs(p)) { 182067a0fd2aSFam Zheng ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum, file); 1821ba3f0e25SFam Zheng if (ret < 0 || ret & BDRV_BLOCK_ALLOCATED) { 1822ba3f0e25SFam Zheng break; 1823ba3f0e25SFam Zheng } 1824ba3f0e25SFam Zheng /* [sector_num, pnum] unallocated on this layer, which could be only 1825ba3f0e25SFam Zheng * the first part of [sector_num, nb_sectors]. */ 1826ba3f0e25SFam Zheng nb_sectors = MIN(nb_sectors, *pnum); 1827ba3f0e25SFam Zheng } 1828ba3f0e25SFam Zheng return ret; 1829ba3f0e25SFam Zheng } 1830ba3f0e25SFam Zheng 1831ba3f0e25SFam Zheng /* Coroutine wrapper for bdrv_get_block_status_above() */ 1832ba3f0e25SFam Zheng static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque) 183361007b31SStefan Hajnoczi { 183461007b31SStefan Hajnoczi BdrvCoGetBlockStatusData *data = opaque; 183561007b31SStefan Hajnoczi 1836ba3f0e25SFam Zheng data->ret = bdrv_co_get_block_status_above(data->bs, data->base, 1837ba3f0e25SFam Zheng data->sector_num, 1838ba3f0e25SFam Zheng data->nb_sectors, 183967a0fd2aSFam Zheng data->pnum, 184067a0fd2aSFam Zheng data->file); 184161007b31SStefan Hajnoczi data->done = true; 184261007b31SStefan Hajnoczi } 184361007b31SStefan Hajnoczi 184461007b31SStefan Hajnoczi /* 1845ba3f0e25SFam Zheng * Synchronous wrapper around bdrv_co_get_block_status_above(). 184661007b31SStefan Hajnoczi * 1847ba3f0e25SFam Zheng * See bdrv_co_get_block_status_above() for details. 184861007b31SStefan Hajnoczi */ 1849ba3f0e25SFam Zheng int64_t bdrv_get_block_status_above(BlockDriverState *bs, 1850ba3f0e25SFam Zheng BlockDriverState *base, 1851ba3f0e25SFam Zheng int64_t sector_num, 185267a0fd2aSFam Zheng int nb_sectors, int *pnum, 185367a0fd2aSFam Zheng BlockDriverState **file) 185461007b31SStefan Hajnoczi { 185561007b31SStefan Hajnoczi Coroutine *co; 185661007b31SStefan Hajnoczi BdrvCoGetBlockStatusData data = { 185761007b31SStefan Hajnoczi .bs = bs, 1858ba3f0e25SFam Zheng .base = base, 185967a0fd2aSFam Zheng .file = file, 186061007b31SStefan Hajnoczi .sector_num = sector_num, 186161007b31SStefan Hajnoczi .nb_sectors = nb_sectors, 186261007b31SStefan Hajnoczi .pnum = pnum, 186361007b31SStefan Hajnoczi .done = false, 186461007b31SStefan Hajnoczi }; 186561007b31SStefan Hajnoczi 186661007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 186761007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 1868ba3f0e25SFam Zheng bdrv_get_block_status_above_co_entry(&data); 186961007b31SStefan Hajnoczi } else { 18700b8b8753SPaolo Bonzini co = qemu_coroutine_create(bdrv_get_block_status_above_co_entry, 18710b8b8753SPaolo Bonzini &data); 18720b8b8753SPaolo Bonzini qemu_coroutine_enter(co); 187388b062c2SPaolo Bonzini BDRV_POLL_WHILE(bs, !data.done); 187461007b31SStefan Hajnoczi } 187561007b31SStefan Hajnoczi return data.ret; 187661007b31SStefan Hajnoczi } 187761007b31SStefan Hajnoczi 1878ba3f0e25SFam Zheng int64_t bdrv_get_block_status(BlockDriverState *bs, 1879ba3f0e25SFam Zheng int64_t sector_num, 188067a0fd2aSFam Zheng int nb_sectors, int *pnum, 188167a0fd2aSFam Zheng BlockDriverState **file) 1882ba3f0e25SFam Zheng { 1883760e0063SKevin Wolf return bdrv_get_block_status_above(bs, backing_bs(bs), 188467a0fd2aSFam Zheng sector_num, nb_sectors, pnum, file); 1885ba3f0e25SFam Zheng } 1886ba3f0e25SFam Zheng 188761007b31SStefan Hajnoczi int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num, 188861007b31SStefan Hajnoczi int nb_sectors, int *pnum) 188961007b31SStefan Hajnoczi { 189067a0fd2aSFam Zheng BlockDriverState *file; 189167a0fd2aSFam Zheng int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum, 189267a0fd2aSFam Zheng &file); 189361007b31SStefan Hajnoczi if (ret < 0) { 189461007b31SStefan Hajnoczi return ret; 189561007b31SStefan Hajnoczi } 189661007b31SStefan Hajnoczi return !!(ret & BDRV_BLOCK_ALLOCATED); 189761007b31SStefan Hajnoczi } 189861007b31SStefan Hajnoczi 189961007b31SStefan Hajnoczi /* 190061007b31SStefan Hajnoczi * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP] 190161007b31SStefan Hajnoczi * 190261007b31SStefan Hajnoczi * Return true if the given sector is allocated in any image between 190361007b31SStefan Hajnoczi * BASE and TOP (inclusive). BASE can be NULL to check if the given 190461007b31SStefan Hajnoczi * sector is allocated in any image of the chain. Return false otherwise. 190561007b31SStefan Hajnoczi * 190661007b31SStefan Hajnoczi * 'pnum' is set to the number of sectors (including and immediately following 190761007b31SStefan Hajnoczi * the specified sector) that are known to be in the same 190861007b31SStefan Hajnoczi * allocated/unallocated state. 190961007b31SStefan Hajnoczi * 191061007b31SStefan Hajnoczi */ 191161007b31SStefan Hajnoczi int bdrv_is_allocated_above(BlockDriverState *top, 191261007b31SStefan Hajnoczi BlockDriverState *base, 191361007b31SStefan Hajnoczi int64_t sector_num, 191461007b31SStefan Hajnoczi int nb_sectors, int *pnum) 191561007b31SStefan Hajnoczi { 191661007b31SStefan Hajnoczi BlockDriverState *intermediate; 191761007b31SStefan Hajnoczi int ret, n = nb_sectors; 191861007b31SStefan Hajnoczi 191961007b31SStefan Hajnoczi intermediate = top; 192061007b31SStefan Hajnoczi while (intermediate && intermediate != base) { 192161007b31SStefan Hajnoczi int pnum_inter; 192261007b31SStefan Hajnoczi ret = bdrv_is_allocated(intermediate, sector_num, nb_sectors, 192361007b31SStefan Hajnoczi &pnum_inter); 192461007b31SStefan Hajnoczi if (ret < 0) { 192561007b31SStefan Hajnoczi return ret; 192661007b31SStefan Hajnoczi } else if (ret) { 192761007b31SStefan Hajnoczi *pnum = pnum_inter; 192861007b31SStefan Hajnoczi return 1; 192961007b31SStefan Hajnoczi } 193061007b31SStefan Hajnoczi 193161007b31SStefan Hajnoczi /* 193261007b31SStefan Hajnoczi * [sector_num, nb_sectors] is unallocated on top but intermediate 193361007b31SStefan Hajnoczi * might have 193461007b31SStefan Hajnoczi * 193561007b31SStefan Hajnoczi * [sector_num+x, nr_sectors] allocated. 193661007b31SStefan Hajnoczi */ 193761007b31SStefan Hajnoczi if (n > pnum_inter && 193861007b31SStefan Hajnoczi (intermediate == top || 193961007b31SStefan Hajnoczi sector_num + pnum_inter < intermediate->total_sectors)) { 194061007b31SStefan Hajnoczi n = pnum_inter; 194161007b31SStefan Hajnoczi } 194261007b31SStefan Hajnoczi 1943760e0063SKevin Wolf intermediate = backing_bs(intermediate); 194461007b31SStefan Hajnoczi } 194561007b31SStefan Hajnoczi 194661007b31SStefan Hajnoczi *pnum = n; 194761007b31SStefan Hajnoczi return 0; 194861007b31SStefan Hajnoczi } 194961007b31SStefan Hajnoczi 19501a8ae822SKevin Wolf typedef struct BdrvVmstateCo { 19511a8ae822SKevin Wolf BlockDriverState *bs; 19521a8ae822SKevin Wolf QEMUIOVector *qiov; 19531a8ae822SKevin Wolf int64_t pos; 19541a8ae822SKevin Wolf bool is_read; 19551a8ae822SKevin Wolf int ret; 19561a8ae822SKevin Wolf } BdrvVmstateCo; 19571a8ae822SKevin Wolf 19581a8ae822SKevin Wolf static int coroutine_fn 19591a8ae822SKevin Wolf bdrv_co_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos, 19601a8ae822SKevin Wolf bool is_read) 19611a8ae822SKevin Wolf { 19621a8ae822SKevin Wolf BlockDriver *drv = bs->drv; 19631a8ae822SKevin Wolf 19641a8ae822SKevin Wolf if (!drv) { 19651a8ae822SKevin Wolf return -ENOMEDIUM; 19661a8ae822SKevin Wolf } else if (drv->bdrv_load_vmstate) { 19671a8ae822SKevin Wolf return is_read ? drv->bdrv_load_vmstate(bs, qiov, pos) 19681a8ae822SKevin Wolf : drv->bdrv_save_vmstate(bs, qiov, pos); 19691a8ae822SKevin Wolf } else if (bs->file) { 19701a8ae822SKevin Wolf return bdrv_co_rw_vmstate(bs->file->bs, qiov, pos, is_read); 19711a8ae822SKevin Wolf } 19721a8ae822SKevin Wolf 19731a8ae822SKevin Wolf return -ENOTSUP; 19741a8ae822SKevin Wolf } 19751a8ae822SKevin Wolf 19761a8ae822SKevin Wolf static void coroutine_fn bdrv_co_rw_vmstate_entry(void *opaque) 19771a8ae822SKevin Wolf { 19781a8ae822SKevin Wolf BdrvVmstateCo *co = opaque; 19791a8ae822SKevin Wolf co->ret = bdrv_co_rw_vmstate(co->bs, co->qiov, co->pos, co->is_read); 19801a8ae822SKevin Wolf } 19811a8ae822SKevin Wolf 19821a8ae822SKevin Wolf static inline int 19831a8ae822SKevin Wolf bdrv_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos, 19841a8ae822SKevin Wolf bool is_read) 19851a8ae822SKevin Wolf { 19861a8ae822SKevin Wolf if (qemu_in_coroutine()) { 19871a8ae822SKevin Wolf return bdrv_co_rw_vmstate(bs, qiov, pos, is_read); 19881a8ae822SKevin Wolf } else { 19891a8ae822SKevin Wolf BdrvVmstateCo data = { 19901a8ae822SKevin Wolf .bs = bs, 19911a8ae822SKevin Wolf .qiov = qiov, 19921a8ae822SKevin Wolf .pos = pos, 19931a8ae822SKevin Wolf .is_read = is_read, 19941a8ae822SKevin Wolf .ret = -EINPROGRESS, 19951a8ae822SKevin Wolf }; 19960b8b8753SPaolo Bonzini Coroutine *co = qemu_coroutine_create(bdrv_co_rw_vmstate_entry, &data); 19971a8ae822SKevin Wolf 19980b8b8753SPaolo Bonzini qemu_coroutine_enter(co); 19991a8ae822SKevin Wolf while (data.ret == -EINPROGRESS) { 20001a8ae822SKevin Wolf aio_poll(bdrv_get_aio_context(bs), true); 20011a8ae822SKevin Wolf } 20021a8ae822SKevin Wolf return data.ret; 20031a8ae822SKevin Wolf } 20041a8ae822SKevin Wolf } 20051a8ae822SKevin Wolf 200661007b31SStefan Hajnoczi int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf, 200761007b31SStefan Hajnoczi int64_t pos, int size) 200861007b31SStefan Hajnoczi { 200961007b31SStefan Hajnoczi QEMUIOVector qiov; 201061007b31SStefan Hajnoczi struct iovec iov = { 201161007b31SStefan Hajnoczi .iov_base = (void *) buf, 201261007b31SStefan Hajnoczi .iov_len = size, 201361007b31SStefan Hajnoczi }; 2014b433d942SKevin Wolf int ret; 201561007b31SStefan Hajnoczi 201661007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 2017b433d942SKevin Wolf 2018b433d942SKevin Wolf ret = bdrv_writev_vmstate(bs, &qiov, pos); 2019b433d942SKevin Wolf if (ret < 0) { 2020b433d942SKevin Wolf return ret; 2021b433d942SKevin Wolf } 2022b433d942SKevin Wolf 2023b433d942SKevin Wolf return size; 202461007b31SStefan Hajnoczi } 202561007b31SStefan Hajnoczi 202661007b31SStefan Hajnoczi int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 202761007b31SStefan Hajnoczi { 20281a8ae822SKevin Wolf return bdrv_rw_vmstate(bs, qiov, pos, false); 202961007b31SStefan Hajnoczi } 203061007b31SStefan Hajnoczi 203161007b31SStefan Hajnoczi int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf, 203261007b31SStefan Hajnoczi int64_t pos, int size) 203361007b31SStefan Hajnoczi { 20345ddda0b8SKevin Wolf QEMUIOVector qiov; 20355ddda0b8SKevin Wolf struct iovec iov = { 20365ddda0b8SKevin Wolf .iov_base = buf, 20375ddda0b8SKevin Wolf .iov_len = size, 20385ddda0b8SKevin Wolf }; 2039b433d942SKevin Wolf int ret; 20405ddda0b8SKevin Wolf 20415ddda0b8SKevin Wolf qemu_iovec_init_external(&qiov, &iov, 1); 2042b433d942SKevin Wolf ret = bdrv_readv_vmstate(bs, &qiov, pos); 2043b433d942SKevin Wolf if (ret < 0) { 2044b433d942SKevin Wolf return ret; 2045b433d942SKevin Wolf } 2046b433d942SKevin Wolf 2047b433d942SKevin Wolf return size; 20485ddda0b8SKevin Wolf } 20495ddda0b8SKevin Wolf 20505ddda0b8SKevin Wolf int bdrv_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 20515ddda0b8SKevin Wolf { 20521a8ae822SKevin Wolf return bdrv_rw_vmstate(bs, qiov, pos, true); 205361007b31SStefan Hajnoczi } 205461007b31SStefan Hajnoczi 205561007b31SStefan Hajnoczi /**************************************************************/ 205661007b31SStefan Hajnoczi /* async I/Os */ 205761007b31SStefan Hajnoczi 2058ebb7af21SKevin Wolf BlockAIOCB *bdrv_aio_readv(BdrvChild *child, int64_t sector_num, 205961007b31SStefan Hajnoczi QEMUIOVector *qiov, int nb_sectors, 206061007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque) 206161007b31SStefan Hajnoczi { 2062ebb7af21SKevin Wolf trace_bdrv_aio_readv(child->bs, sector_num, nb_sectors, opaque); 206361007b31SStefan Hajnoczi 2064b15404e0SEric Blake assert(nb_sectors << BDRV_SECTOR_BITS == qiov->size); 2065b15404e0SEric Blake return bdrv_co_aio_prw_vector(child, sector_num << BDRV_SECTOR_BITS, qiov, 2066b15404e0SEric Blake 0, cb, opaque, false); 206761007b31SStefan Hajnoczi } 206861007b31SStefan Hajnoczi 20690d1049c7SKevin Wolf BlockAIOCB *bdrv_aio_writev(BdrvChild *child, int64_t sector_num, 207061007b31SStefan Hajnoczi QEMUIOVector *qiov, int nb_sectors, 207161007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque) 207261007b31SStefan Hajnoczi { 20730d1049c7SKevin Wolf trace_bdrv_aio_writev(child->bs, sector_num, nb_sectors, opaque); 207461007b31SStefan Hajnoczi 2075b15404e0SEric Blake assert(nb_sectors << BDRV_SECTOR_BITS == qiov->size); 2076b15404e0SEric Blake return bdrv_co_aio_prw_vector(child, sector_num << BDRV_SECTOR_BITS, qiov, 2077b15404e0SEric Blake 0, cb, opaque, true); 207861007b31SStefan Hajnoczi } 207961007b31SStefan Hajnoczi 208061007b31SStefan Hajnoczi void bdrv_aio_cancel(BlockAIOCB *acb) 208161007b31SStefan Hajnoczi { 208261007b31SStefan Hajnoczi qemu_aio_ref(acb); 208361007b31SStefan Hajnoczi bdrv_aio_cancel_async(acb); 208461007b31SStefan Hajnoczi while (acb->refcnt > 1) { 208561007b31SStefan Hajnoczi if (acb->aiocb_info->get_aio_context) { 208661007b31SStefan Hajnoczi aio_poll(acb->aiocb_info->get_aio_context(acb), true); 208761007b31SStefan Hajnoczi } else if (acb->bs) { 20882f47da5fSPaolo Bonzini /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so 20892f47da5fSPaolo Bonzini * assert that we're not using an I/O thread. Thread-safe 20902f47da5fSPaolo Bonzini * code should use bdrv_aio_cancel_async exclusively. 20912f47da5fSPaolo Bonzini */ 20922f47da5fSPaolo Bonzini assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context()); 209361007b31SStefan Hajnoczi aio_poll(bdrv_get_aio_context(acb->bs), true); 209461007b31SStefan Hajnoczi } else { 209561007b31SStefan Hajnoczi abort(); 209661007b31SStefan Hajnoczi } 209761007b31SStefan Hajnoczi } 209861007b31SStefan Hajnoczi qemu_aio_unref(acb); 209961007b31SStefan Hajnoczi } 210061007b31SStefan Hajnoczi 210161007b31SStefan Hajnoczi /* Async version of aio cancel. The caller is not blocked if the acb implements 210261007b31SStefan Hajnoczi * cancel_async, otherwise we do nothing and let the request normally complete. 210361007b31SStefan Hajnoczi * In either case the completion callback must be called. */ 210461007b31SStefan Hajnoczi void bdrv_aio_cancel_async(BlockAIOCB *acb) 210561007b31SStefan Hajnoczi { 210661007b31SStefan Hajnoczi if (acb->aiocb_info->cancel_async) { 210761007b31SStefan Hajnoczi acb->aiocb_info->cancel_async(acb); 210861007b31SStefan Hajnoczi } 210961007b31SStefan Hajnoczi } 211061007b31SStefan Hajnoczi 211161007b31SStefan Hajnoczi /**************************************************************/ 211261007b31SStefan Hajnoczi /* async block device emulation */ 211361007b31SStefan Hajnoczi 211441574268SEric Blake typedef struct BlockRequest { 211541574268SEric Blake union { 211641574268SEric Blake /* Used during read, write, trim */ 211741574268SEric Blake struct { 2118b15404e0SEric Blake int64_t offset; 2119b15404e0SEric Blake int bytes; 212041574268SEric Blake int flags; 212141574268SEric Blake QEMUIOVector *qiov; 212241574268SEric Blake }; 212341574268SEric Blake /* Used during ioctl */ 212441574268SEric Blake struct { 212541574268SEric Blake int req; 212641574268SEric Blake void *buf; 212741574268SEric Blake }; 212841574268SEric Blake }; 212941574268SEric Blake BlockCompletionFunc *cb; 213041574268SEric Blake void *opaque; 213141574268SEric Blake 213241574268SEric Blake int error; 213341574268SEric Blake } BlockRequest; 213441574268SEric Blake 213561007b31SStefan Hajnoczi typedef struct BlockAIOCBCoroutine { 213661007b31SStefan Hajnoczi BlockAIOCB common; 2137adad6496SKevin Wolf BdrvChild *child; 213861007b31SStefan Hajnoczi BlockRequest req; 213961007b31SStefan Hajnoczi bool is_write; 214061007b31SStefan Hajnoczi bool need_bh; 214161007b31SStefan Hajnoczi bool *done; 214261007b31SStefan Hajnoczi } BlockAIOCBCoroutine; 214361007b31SStefan Hajnoczi 214461007b31SStefan Hajnoczi static const AIOCBInfo bdrv_em_co_aiocb_info = { 214561007b31SStefan Hajnoczi .aiocb_size = sizeof(BlockAIOCBCoroutine), 214661007b31SStefan Hajnoczi }; 214761007b31SStefan Hajnoczi 214861007b31SStefan Hajnoczi static void bdrv_co_complete(BlockAIOCBCoroutine *acb) 214961007b31SStefan Hajnoczi { 215061007b31SStefan Hajnoczi if (!acb->need_bh) { 215199723548SPaolo Bonzini bdrv_dec_in_flight(acb->common.bs); 215261007b31SStefan Hajnoczi acb->common.cb(acb->common.opaque, acb->req.error); 215361007b31SStefan Hajnoczi qemu_aio_unref(acb); 215461007b31SStefan Hajnoczi } 215561007b31SStefan Hajnoczi } 215661007b31SStefan Hajnoczi 215761007b31SStefan Hajnoczi static void bdrv_co_em_bh(void *opaque) 215861007b31SStefan Hajnoczi { 215961007b31SStefan Hajnoczi BlockAIOCBCoroutine *acb = opaque; 216061007b31SStefan Hajnoczi 216161007b31SStefan Hajnoczi assert(!acb->need_bh); 216261007b31SStefan Hajnoczi bdrv_co_complete(acb); 216361007b31SStefan Hajnoczi } 216461007b31SStefan Hajnoczi 216561007b31SStefan Hajnoczi static void bdrv_co_maybe_schedule_bh(BlockAIOCBCoroutine *acb) 216661007b31SStefan Hajnoczi { 216761007b31SStefan Hajnoczi acb->need_bh = false; 216861007b31SStefan Hajnoczi if (acb->req.error != -EINPROGRESS) { 216961007b31SStefan Hajnoczi BlockDriverState *bs = acb->common.bs; 217061007b31SStefan Hajnoczi 2171fffb6e12SPaolo Bonzini aio_bh_schedule_oneshot(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb); 217261007b31SStefan Hajnoczi } 217361007b31SStefan Hajnoczi } 217461007b31SStefan Hajnoczi 217561007b31SStefan Hajnoczi /* Invoke bdrv_co_do_readv/bdrv_co_do_writev */ 217661007b31SStefan Hajnoczi static void coroutine_fn bdrv_co_do_rw(void *opaque) 217761007b31SStefan Hajnoczi { 217861007b31SStefan Hajnoczi BlockAIOCBCoroutine *acb = opaque; 217961007b31SStefan Hajnoczi 218061007b31SStefan Hajnoczi if (!acb->is_write) { 2181b15404e0SEric Blake acb->req.error = bdrv_co_preadv(acb->child, acb->req.offset, 2182b15404e0SEric Blake acb->req.qiov->size, acb->req.qiov, acb->req.flags); 218361007b31SStefan Hajnoczi } else { 2184b15404e0SEric Blake acb->req.error = bdrv_co_pwritev(acb->child, acb->req.offset, 2185b15404e0SEric Blake acb->req.qiov->size, acb->req.qiov, acb->req.flags); 218661007b31SStefan Hajnoczi } 218761007b31SStefan Hajnoczi 218861007b31SStefan Hajnoczi bdrv_co_complete(acb); 218961007b31SStefan Hajnoczi } 219061007b31SStefan Hajnoczi 2191b15404e0SEric Blake static BlockAIOCB *bdrv_co_aio_prw_vector(BdrvChild *child, 2192b15404e0SEric Blake int64_t offset, 219361007b31SStefan Hajnoczi QEMUIOVector *qiov, 219461007b31SStefan Hajnoczi BdrvRequestFlags flags, 219561007b31SStefan Hajnoczi BlockCompletionFunc *cb, 219661007b31SStefan Hajnoczi void *opaque, 219761007b31SStefan Hajnoczi bool is_write) 219861007b31SStefan Hajnoczi { 219961007b31SStefan Hajnoczi Coroutine *co; 220061007b31SStefan Hajnoczi BlockAIOCBCoroutine *acb; 220161007b31SStefan Hajnoczi 220299723548SPaolo Bonzini /* Matched by bdrv_co_complete's bdrv_dec_in_flight. */ 220399723548SPaolo Bonzini bdrv_inc_in_flight(child->bs); 220499723548SPaolo Bonzini 2205adad6496SKevin Wolf acb = qemu_aio_get(&bdrv_em_co_aiocb_info, child->bs, cb, opaque); 2206adad6496SKevin Wolf acb->child = child; 220761007b31SStefan Hajnoczi acb->need_bh = true; 220861007b31SStefan Hajnoczi acb->req.error = -EINPROGRESS; 2209b15404e0SEric Blake acb->req.offset = offset; 221061007b31SStefan Hajnoczi acb->req.qiov = qiov; 221161007b31SStefan Hajnoczi acb->req.flags = flags; 221261007b31SStefan Hajnoczi acb->is_write = is_write; 221361007b31SStefan Hajnoczi 22140b8b8753SPaolo Bonzini co = qemu_coroutine_create(bdrv_co_do_rw, acb); 22150b8b8753SPaolo Bonzini qemu_coroutine_enter(co); 221661007b31SStefan Hajnoczi 221761007b31SStefan Hajnoczi bdrv_co_maybe_schedule_bh(acb); 221861007b31SStefan Hajnoczi return &acb->common; 221961007b31SStefan Hajnoczi } 222061007b31SStefan Hajnoczi 222161007b31SStefan Hajnoczi static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque) 222261007b31SStefan Hajnoczi { 222361007b31SStefan Hajnoczi BlockAIOCBCoroutine *acb = opaque; 222461007b31SStefan Hajnoczi BlockDriverState *bs = acb->common.bs; 222561007b31SStefan Hajnoczi 222661007b31SStefan Hajnoczi acb->req.error = bdrv_co_flush(bs); 222761007b31SStefan Hajnoczi bdrv_co_complete(acb); 222861007b31SStefan Hajnoczi } 222961007b31SStefan Hajnoczi 223061007b31SStefan Hajnoczi BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs, 223161007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque) 223261007b31SStefan Hajnoczi { 223361007b31SStefan Hajnoczi trace_bdrv_aio_flush(bs, opaque); 223461007b31SStefan Hajnoczi 223561007b31SStefan Hajnoczi Coroutine *co; 223661007b31SStefan Hajnoczi BlockAIOCBCoroutine *acb; 223761007b31SStefan Hajnoczi 223899723548SPaolo Bonzini /* Matched by bdrv_co_complete's bdrv_dec_in_flight. */ 223999723548SPaolo Bonzini bdrv_inc_in_flight(bs); 224099723548SPaolo Bonzini 224161007b31SStefan Hajnoczi acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque); 224261007b31SStefan Hajnoczi acb->need_bh = true; 224361007b31SStefan Hajnoczi acb->req.error = -EINPROGRESS; 224461007b31SStefan Hajnoczi 22450b8b8753SPaolo Bonzini co = qemu_coroutine_create(bdrv_aio_flush_co_entry, acb); 22460b8b8753SPaolo Bonzini qemu_coroutine_enter(co); 224761007b31SStefan Hajnoczi 224861007b31SStefan Hajnoczi bdrv_co_maybe_schedule_bh(acb); 224961007b31SStefan Hajnoczi return &acb->common; 225061007b31SStefan Hajnoczi } 225161007b31SStefan Hajnoczi 225261007b31SStefan Hajnoczi /**************************************************************/ 225361007b31SStefan Hajnoczi /* Coroutine block device emulation */ 225461007b31SStefan Hajnoczi 2255e293b7a3SKevin Wolf typedef struct FlushCo { 2256e293b7a3SKevin Wolf BlockDriverState *bs; 2257e293b7a3SKevin Wolf int ret; 2258e293b7a3SKevin Wolf } FlushCo; 2259e293b7a3SKevin Wolf 2260e293b7a3SKevin Wolf 226161007b31SStefan Hajnoczi static void coroutine_fn bdrv_flush_co_entry(void *opaque) 226261007b31SStefan Hajnoczi { 2263e293b7a3SKevin Wolf FlushCo *rwco = opaque; 226461007b31SStefan Hajnoczi 226561007b31SStefan Hajnoczi rwco->ret = bdrv_co_flush(rwco->bs); 226661007b31SStefan Hajnoczi } 226761007b31SStefan Hajnoczi 226861007b31SStefan Hajnoczi int coroutine_fn bdrv_co_flush(BlockDriverState *bs) 226961007b31SStefan Hajnoczi { 227061007b31SStefan Hajnoczi int ret; 227161007b31SStefan Hajnoczi 22721b6bc94dSDimitris Aragiorgis if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs) || 22731b6bc94dSDimitris Aragiorgis bdrv_is_sg(bs)) { 227461007b31SStefan Hajnoczi return 0; 227561007b31SStefan Hajnoczi } 227661007b31SStefan Hajnoczi 227799723548SPaolo Bonzini bdrv_inc_in_flight(bs); 2278c32b82afSPavel Dovgalyuk 22793ff2f67aSEvgeny Yakovlev int current_gen = bs->write_gen; 22803ff2f67aSEvgeny Yakovlev 22813ff2f67aSEvgeny Yakovlev /* Wait until any previous flushes are completed */ 228299723548SPaolo Bonzini while (bs->active_flush_req) { 22831ace7ceaSPaolo Bonzini qemu_co_queue_wait(&bs->flush_queue, NULL); 22843ff2f67aSEvgeny Yakovlev } 22853ff2f67aSEvgeny Yakovlev 228699723548SPaolo Bonzini bs->active_flush_req = true; 22873ff2f67aSEvgeny Yakovlev 2288c32b82afSPavel Dovgalyuk /* Write back all layers by calling one driver function */ 2289c32b82afSPavel Dovgalyuk if (bs->drv->bdrv_co_flush) { 2290c32b82afSPavel Dovgalyuk ret = bs->drv->bdrv_co_flush(bs); 2291c32b82afSPavel Dovgalyuk goto out; 2292c32b82afSPavel Dovgalyuk } 2293c32b82afSPavel Dovgalyuk 229461007b31SStefan Hajnoczi /* Write back cached data to the OS even with cache=unsafe */ 229561007b31SStefan Hajnoczi BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS); 229661007b31SStefan Hajnoczi if (bs->drv->bdrv_co_flush_to_os) { 229761007b31SStefan Hajnoczi ret = bs->drv->bdrv_co_flush_to_os(bs); 229861007b31SStefan Hajnoczi if (ret < 0) { 2299cdb5e315SFam Zheng goto out; 230061007b31SStefan Hajnoczi } 230161007b31SStefan Hajnoczi } 230261007b31SStefan Hajnoczi 230361007b31SStefan Hajnoczi /* But don't actually force it to the disk with cache=unsafe */ 230461007b31SStefan Hajnoczi if (bs->open_flags & BDRV_O_NO_FLUSH) { 230561007b31SStefan Hajnoczi goto flush_parent; 230661007b31SStefan Hajnoczi } 230761007b31SStefan Hajnoczi 23083ff2f67aSEvgeny Yakovlev /* Check if we really need to flush anything */ 23093ff2f67aSEvgeny Yakovlev if (bs->flushed_gen == current_gen) { 23103ff2f67aSEvgeny Yakovlev goto flush_parent; 23113ff2f67aSEvgeny Yakovlev } 23123ff2f67aSEvgeny Yakovlev 231361007b31SStefan Hajnoczi BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK); 231461007b31SStefan Hajnoczi if (bs->drv->bdrv_co_flush_to_disk) { 231561007b31SStefan Hajnoczi ret = bs->drv->bdrv_co_flush_to_disk(bs); 231661007b31SStefan Hajnoczi } else if (bs->drv->bdrv_aio_flush) { 231761007b31SStefan Hajnoczi BlockAIOCB *acb; 231861007b31SStefan Hajnoczi CoroutineIOCompletion co = { 231961007b31SStefan Hajnoczi .coroutine = qemu_coroutine_self(), 232061007b31SStefan Hajnoczi }; 232161007b31SStefan Hajnoczi 232261007b31SStefan Hajnoczi acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co); 232361007b31SStefan Hajnoczi if (acb == NULL) { 232461007b31SStefan Hajnoczi ret = -EIO; 232561007b31SStefan Hajnoczi } else { 232661007b31SStefan Hajnoczi qemu_coroutine_yield(); 232761007b31SStefan Hajnoczi ret = co.ret; 232861007b31SStefan Hajnoczi } 232961007b31SStefan Hajnoczi } else { 233061007b31SStefan Hajnoczi /* 233161007b31SStefan Hajnoczi * Some block drivers always operate in either writethrough or unsafe 233261007b31SStefan Hajnoczi * mode and don't support bdrv_flush therefore. Usually qemu doesn't 233361007b31SStefan Hajnoczi * know how the server works (because the behaviour is hardcoded or 233461007b31SStefan Hajnoczi * depends on server-side configuration), so we can't ensure that 233561007b31SStefan Hajnoczi * everything is safe on disk. Returning an error doesn't work because 233661007b31SStefan Hajnoczi * that would break guests even if the server operates in writethrough 233761007b31SStefan Hajnoczi * mode. 233861007b31SStefan Hajnoczi * 233961007b31SStefan Hajnoczi * Let's hope the user knows what he's doing. 234061007b31SStefan Hajnoczi */ 234161007b31SStefan Hajnoczi ret = 0; 234261007b31SStefan Hajnoczi } 23433ff2f67aSEvgeny Yakovlev 234461007b31SStefan Hajnoczi if (ret < 0) { 2345cdb5e315SFam Zheng goto out; 234661007b31SStefan Hajnoczi } 234761007b31SStefan Hajnoczi 234861007b31SStefan Hajnoczi /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH 234961007b31SStefan Hajnoczi * in the case of cache=unsafe, so there are no useless flushes. 235061007b31SStefan Hajnoczi */ 235161007b31SStefan Hajnoczi flush_parent: 2352cdb5e315SFam Zheng ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0; 2353cdb5e315SFam Zheng out: 23543ff2f67aSEvgeny Yakovlev /* Notify any pending flushes that we have completed */ 2355e6af1e08SKevin Wolf if (ret == 0) { 23563ff2f67aSEvgeny Yakovlev bs->flushed_gen = current_gen; 2357e6af1e08SKevin Wolf } 235899723548SPaolo Bonzini bs->active_flush_req = false; 2359156af3acSDenis V. Lunev /* Return value is ignored - it's ok if wait queue is empty */ 2360156af3acSDenis V. Lunev qemu_co_queue_next(&bs->flush_queue); 23613ff2f67aSEvgeny Yakovlev 236299723548SPaolo Bonzini bdrv_dec_in_flight(bs); 2363cdb5e315SFam Zheng return ret; 236461007b31SStefan Hajnoczi } 236561007b31SStefan Hajnoczi 236661007b31SStefan Hajnoczi int bdrv_flush(BlockDriverState *bs) 236761007b31SStefan Hajnoczi { 236861007b31SStefan Hajnoczi Coroutine *co; 2369e293b7a3SKevin Wolf FlushCo flush_co = { 237061007b31SStefan Hajnoczi .bs = bs, 237161007b31SStefan Hajnoczi .ret = NOT_DONE, 237261007b31SStefan Hajnoczi }; 237361007b31SStefan Hajnoczi 237461007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 237561007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 2376e293b7a3SKevin Wolf bdrv_flush_co_entry(&flush_co); 237761007b31SStefan Hajnoczi } else { 23780b8b8753SPaolo Bonzini co = qemu_coroutine_create(bdrv_flush_co_entry, &flush_co); 23790b8b8753SPaolo Bonzini qemu_coroutine_enter(co); 238088b062c2SPaolo Bonzini BDRV_POLL_WHILE(bs, flush_co.ret == NOT_DONE); 238161007b31SStefan Hajnoczi } 238261007b31SStefan Hajnoczi 2383e293b7a3SKevin Wolf return flush_co.ret; 238461007b31SStefan Hajnoczi } 238561007b31SStefan Hajnoczi 238661007b31SStefan Hajnoczi typedef struct DiscardCo { 238761007b31SStefan Hajnoczi BlockDriverState *bs; 23880c51a893SEric Blake int64_t offset; 23890c51a893SEric Blake int count; 239061007b31SStefan Hajnoczi int ret; 239161007b31SStefan Hajnoczi } DiscardCo; 23920c51a893SEric Blake static void coroutine_fn bdrv_pdiscard_co_entry(void *opaque) 239361007b31SStefan Hajnoczi { 239461007b31SStefan Hajnoczi DiscardCo *rwco = opaque; 239561007b31SStefan Hajnoczi 23960c51a893SEric Blake rwco->ret = bdrv_co_pdiscard(rwco->bs, rwco->offset, rwco->count); 239761007b31SStefan Hajnoczi } 239861007b31SStefan Hajnoczi 23999f1963b3SEric Blake int coroutine_fn bdrv_co_pdiscard(BlockDriverState *bs, int64_t offset, 24009f1963b3SEric Blake int count) 240161007b31SStefan Hajnoczi { 2402b1066c87SFam Zheng BdrvTrackedRequest req; 24039f1963b3SEric Blake int max_pdiscard, ret; 24043482b9bcSEric Blake int head, tail, align; 240561007b31SStefan Hajnoczi 240661007b31SStefan Hajnoczi if (!bs->drv) { 240761007b31SStefan Hajnoczi return -ENOMEDIUM; 240861007b31SStefan Hajnoczi } 240961007b31SStefan Hajnoczi 24109f1963b3SEric Blake ret = bdrv_check_byte_request(bs, offset, count); 241161007b31SStefan Hajnoczi if (ret < 0) { 241261007b31SStefan Hajnoczi return ret; 241361007b31SStefan Hajnoczi } else if (bs->read_only) { 2414eaf5fe2dSPaolo Bonzini return -EPERM; 241561007b31SStefan Hajnoczi } 241604c01a5cSKevin Wolf assert(!(bs->open_flags & BDRV_O_INACTIVE)); 241761007b31SStefan Hajnoczi 241861007b31SStefan Hajnoczi /* Do nothing if disabled. */ 241961007b31SStefan Hajnoczi if (!(bs->open_flags & BDRV_O_UNMAP)) { 242061007b31SStefan Hajnoczi return 0; 242161007b31SStefan Hajnoczi } 242261007b31SStefan Hajnoczi 242302aefe43SEric Blake if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) { 242461007b31SStefan Hajnoczi return 0; 242561007b31SStefan Hajnoczi } 242661007b31SStefan Hajnoczi 24273482b9bcSEric Blake /* Discard is advisory, but some devices track and coalesce 24283482b9bcSEric Blake * unaligned requests, so we must pass everything down rather than 24293482b9bcSEric Blake * round here. Still, most devices will just silently ignore 24303482b9bcSEric Blake * unaligned requests (by returning -ENOTSUP), so we must fragment 24313482b9bcSEric Blake * the request accordingly. */ 243202aefe43SEric Blake align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment); 2433b8d0a980SEric Blake assert(align % bs->bl.request_alignment == 0); 2434b8d0a980SEric Blake head = offset % align; 24353482b9bcSEric Blake tail = (offset + count) % align; 24369f1963b3SEric Blake 243799723548SPaolo Bonzini bdrv_inc_in_flight(bs); 24389f1963b3SEric Blake tracked_request_begin(&req, bs, offset, count, BDRV_TRACKED_DISCARD); 243950824995SFam Zheng 2440ec050f77SDenis V. Lunev ret = notifier_with_return_list_notify(&bs->before_write_notifiers, &req); 2441ec050f77SDenis V. Lunev if (ret < 0) { 2442ec050f77SDenis V. Lunev goto out; 2443ec050f77SDenis V. Lunev } 2444ec050f77SDenis V. Lunev 24459f1963b3SEric Blake max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT_MAX), 24469f1963b3SEric Blake align); 24473482b9bcSEric Blake assert(max_pdiscard >= bs->bl.request_alignment); 24489f1963b3SEric Blake 24499f1963b3SEric Blake while (count > 0) { 245061007b31SStefan Hajnoczi int ret; 24513482b9bcSEric Blake int num = count; 24523482b9bcSEric Blake 24533482b9bcSEric Blake if (head) { 24543482b9bcSEric Blake /* Make small requests to get to alignment boundaries. */ 24553482b9bcSEric Blake num = MIN(count, align - head); 24563482b9bcSEric Blake if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) { 24573482b9bcSEric Blake num %= bs->bl.request_alignment; 24583482b9bcSEric Blake } 24593482b9bcSEric Blake head = (head + num) % align; 24603482b9bcSEric Blake assert(num < max_pdiscard); 24613482b9bcSEric Blake } else if (tail) { 24623482b9bcSEric Blake if (num > align) { 24633482b9bcSEric Blake /* Shorten the request to the last aligned cluster. */ 24643482b9bcSEric Blake num -= tail; 24653482b9bcSEric Blake } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) && 24663482b9bcSEric Blake tail > bs->bl.request_alignment) { 24673482b9bcSEric Blake tail %= bs->bl.request_alignment; 24683482b9bcSEric Blake num -= tail; 24693482b9bcSEric Blake } 24703482b9bcSEric Blake } 24713482b9bcSEric Blake /* limit request size */ 24723482b9bcSEric Blake if (num > max_pdiscard) { 24733482b9bcSEric Blake num = max_pdiscard; 24743482b9bcSEric Blake } 247561007b31SStefan Hajnoczi 247647a5486dSEric Blake if (bs->drv->bdrv_co_pdiscard) { 247747a5486dSEric Blake ret = bs->drv->bdrv_co_pdiscard(bs, offset, num); 247861007b31SStefan Hajnoczi } else { 247961007b31SStefan Hajnoczi BlockAIOCB *acb; 248061007b31SStefan Hajnoczi CoroutineIOCompletion co = { 248161007b31SStefan Hajnoczi .coroutine = qemu_coroutine_self(), 248261007b31SStefan Hajnoczi }; 248361007b31SStefan Hajnoczi 24844da444a0SEric Blake acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num, 248561007b31SStefan Hajnoczi bdrv_co_io_em_complete, &co); 248661007b31SStefan Hajnoczi if (acb == NULL) { 2487b1066c87SFam Zheng ret = -EIO; 2488b1066c87SFam Zheng goto out; 248961007b31SStefan Hajnoczi } else { 249061007b31SStefan Hajnoczi qemu_coroutine_yield(); 249161007b31SStefan Hajnoczi ret = co.ret; 249261007b31SStefan Hajnoczi } 249361007b31SStefan Hajnoczi } 249461007b31SStefan Hajnoczi if (ret && ret != -ENOTSUP) { 2495b1066c87SFam Zheng goto out; 249661007b31SStefan Hajnoczi } 249761007b31SStefan Hajnoczi 24989f1963b3SEric Blake offset += num; 24999f1963b3SEric Blake count -= num; 250061007b31SStefan Hajnoczi } 2501b1066c87SFam Zheng ret = 0; 2502b1066c87SFam Zheng out: 25033ff2f67aSEvgeny Yakovlev ++bs->write_gen; 2504968d8b06SDenis V. Lunev bdrv_set_dirty(bs, req.offset >> BDRV_SECTOR_BITS, 2505968d8b06SDenis V. Lunev req.bytes >> BDRV_SECTOR_BITS); 2506b1066c87SFam Zheng tracked_request_end(&req); 250799723548SPaolo Bonzini bdrv_dec_in_flight(bs); 2508b1066c87SFam Zheng return ret; 250961007b31SStefan Hajnoczi } 251061007b31SStefan Hajnoczi 25110c51a893SEric Blake int bdrv_pdiscard(BlockDriverState *bs, int64_t offset, int count) 251261007b31SStefan Hajnoczi { 251361007b31SStefan Hajnoczi Coroutine *co; 251461007b31SStefan Hajnoczi DiscardCo rwco = { 251561007b31SStefan Hajnoczi .bs = bs, 25160c51a893SEric Blake .offset = offset, 25170c51a893SEric Blake .count = count, 251861007b31SStefan Hajnoczi .ret = NOT_DONE, 251961007b31SStefan Hajnoczi }; 252061007b31SStefan Hajnoczi 252161007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 252261007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 25230c51a893SEric Blake bdrv_pdiscard_co_entry(&rwco); 252461007b31SStefan Hajnoczi } else { 25250c51a893SEric Blake co = qemu_coroutine_create(bdrv_pdiscard_co_entry, &rwco); 25260b8b8753SPaolo Bonzini qemu_coroutine_enter(co); 252788b062c2SPaolo Bonzini BDRV_POLL_WHILE(bs, rwco.ret == NOT_DONE); 252861007b31SStefan Hajnoczi } 252961007b31SStefan Hajnoczi 253061007b31SStefan Hajnoczi return rwco.ret; 253161007b31SStefan Hajnoczi } 253261007b31SStefan Hajnoczi 253348af776aSKevin Wolf int bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf) 253461007b31SStefan Hajnoczi { 253561007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 25365c5ae76aSFam Zheng CoroutineIOCompletion co = { 25375c5ae76aSFam Zheng .coroutine = qemu_coroutine_self(), 25385c5ae76aSFam Zheng }; 25395c5ae76aSFam Zheng BlockAIOCB *acb; 254061007b31SStefan Hajnoczi 254199723548SPaolo Bonzini bdrv_inc_in_flight(bs); 254216a389dcSKevin Wolf if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) { 25435c5ae76aSFam Zheng co.ret = -ENOTSUP; 25445c5ae76aSFam Zheng goto out; 25455c5ae76aSFam Zheng } 25465c5ae76aSFam Zheng 254716a389dcSKevin Wolf if (drv->bdrv_co_ioctl) { 254816a389dcSKevin Wolf co.ret = drv->bdrv_co_ioctl(bs, req, buf); 254916a389dcSKevin Wolf } else { 25505c5ae76aSFam Zheng acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co); 25515c5ae76aSFam Zheng if (!acb) { 2552c8a9fd80SFam Zheng co.ret = -ENOTSUP; 2553c8a9fd80SFam Zheng goto out; 25545c5ae76aSFam Zheng } 25555c5ae76aSFam Zheng qemu_coroutine_yield(); 255616a389dcSKevin Wolf } 25575c5ae76aSFam Zheng out: 255899723548SPaolo Bonzini bdrv_dec_in_flight(bs); 25595c5ae76aSFam Zheng return co.ret; 25605c5ae76aSFam Zheng } 25615c5ae76aSFam Zheng 256261007b31SStefan Hajnoczi void *qemu_blockalign(BlockDriverState *bs, size_t size) 256361007b31SStefan Hajnoczi { 256461007b31SStefan Hajnoczi return qemu_memalign(bdrv_opt_mem_align(bs), size); 256561007b31SStefan Hajnoczi } 256661007b31SStefan Hajnoczi 256761007b31SStefan Hajnoczi void *qemu_blockalign0(BlockDriverState *bs, size_t size) 256861007b31SStefan Hajnoczi { 256961007b31SStefan Hajnoczi return memset(qemu_blockalign(bs, size), 0, size); 257061007b31SStefan Hajnoczi } 257161007b31SStefan Hajnoczi 257261007b31SStefan Hajnoczi void *qemu_try_blockalign(BlockDriverState *bs, size_t size) 257361007b31SStefan Hajnoczi { 257461007b31SStefan Hajnoczi size_t align = bdrv_opt_mem_align(bs); 257561007b31SStefan Hajnoczi 257661007b31SStefan Hajnoczi /* Ensure that NULL is never returned on success */ 257761007b31SStefan Hajnoczi assert(align > 0); 257861007b31SStefan Hajnoczi if (size == 0) { 257961007b31SStefan Hajnoczi size = align; 258061007b31SStefan Hajnoczi } 258161007b31SStefan Hajnoczi 258261007b31SStefan Hajnoczi return qemu_try_memalign(align, size); 258361007b31SStefan Hajnoczi } 258461007b31SStefan Hajnoczi 258561007b31SStefan Hajnoczi void *qemu_try_blockalign0(BlockDriverState *bs, size_t size) 258661007b31SStefan Hajnoczi { 258761007b31SStefan Hajnoczi void *mem = qemu_try_blockalign(bs, size); 258861007b31SStefan Hajnoczi 258961007b31SStefan Hajnoczi if (mem) { 259061007b31SStefan Hajnoczi memset(mem, 0, size); 259161007b31SStefan Hajnoczi } 259261007b31SStefan Hajnoczi 259361007b31SStefan Hajnoczi return mem; 259461007b31SStefan Hajnoczi } 259561007b31SStefan Hajnoczi 259661007b31SStefan Hajnoczi /* 259761007b31SStefan Hajnoczi * Check if all memory in this vector is sector aligned. 259861007b31SStefan Hajnoczi */ 259961007b31SStefan Hajnoczi bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov) 260061007b31SStefan Hajnoczi { 260161007b31SStefan Hajnoczi int i; 26024196d2f0SDenis V. Lunev size_t alignment = bdrv_min_mem_align(bs); 260361007b31SStefan Hajnoczi 260461007b31SStefan Hajnoczi for (i = 0; i < qiov->niov; i++) { 260561007b31SStefan Hajnoczi if ((uintptr_t) qiov->iov[i].iov_base % alignment) { 260661007b31SStefan Hajnoczi return false; 260761007b31SStefan Hajnoczi } 260861007b31SStefan Hajnoczi if (qiov->iov[i].iov_len % alignment) { 260961007b31SStefan Hajnoczi return false; 261061007b31SStefan Hajnoczi } 261161007b31SStefan Hajnoczi } 261261007b31SStefan Hajnoczi 261361007b31SStefan Hajnoczi return true; 261461007b31SStefan Hajnoczi } 261561007b31SStefan Hajnoczi 261661007b31SStefan Hajnoczi void bdrv_add_before_write_notifier(BlockDriverState *bs, 261761007b31SStefan Hajnoczi NotifierWithReturn *notifier) 261861007b31SStefan Hajnoczi { 261961007b31SStefan Hajnoczi notifier_with_return_list_add(&bs->before_write_notifiers, notifier); 262061007b31SStefan Hajnoczi } 262161007b31SStefan Hajnoczi 262261007b31SStefan Hajnoczi void bdrv_io_plug(BlockDriverState *bs) 262361007b31SStefan Hajnoczi { 26246b98bd64SPaolo Bonzini BdrvChild *child; 26256b98bd64SPaolo Bonzini 26266b98bd64SPaolo Bonzini QLIST_FOREACH(child, &bs->children, next) { 26276b98bd64SPaolo Bonzini bdrv_io_plug(child->bs); 26286b98bd64SPaolo Bonzini } 26296b98bd64SPaolo Bonzini 26308f90b5e9SPaolo Bonzini if (bs->io_plugged++ == 0) { 263161007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 263261007b31SStefan Hajnoczi if (drv && drv->bdrv_io_plug) { 263361007b31SStefan Hajnoczi drv->bdrv_io_plug(bs); 26346b98bd64SPaolo Bonzini } 263561007b31SStefan Hajnoczi } 263661007b31SStefan Hajnoczi } 263761007b31SStefan Hajnoczi 263861007b31SStefan Hajnoczi void bdrv_io_unplug(BlockDriverState *bs) 263961007b31SStefan Hajnoczi { 26406b98bd64SPaolo Bonzini BdrvChild *child; 26416b98bd64SPaolo Bonzini 26426b98bd64SPaolo Bonzini assert(bs->io_plugged); 26438f90b5e9SPaolo Bonzini if (--bs->io_plugged == 0) { 264461007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 264561007b31SStefan Hajnoczi if (drv && drv->bdrv_io_unplug) { 264661007b31SStefan Hajnoczi drv->bdrv_io_unplug(bs); 264761007b31SStefan Hajnoczi } 264861007b31SStefan Hajnoczi } 264961007b31SStefan Hajnoczi 26506b98bd64SPaolo Bonzini QLIST_FOREACH(child, &bs->children, next) { 26516b98bd64SPaolo Bonzini bdrv_io_unplug(child->bs); 26526b98bd64SPaolo Bonzini } 26536b98bd64SPaolo Bonzini } 2654