161007b31SStefan Hajnoczi /* 261007b31SStefan Hajnoczi * Block layer I/O functions 361007b31SStefan Hajnoczi * 461007b31SStefan Hajnoczi * Copyright (c) 2003 Fabrice Bellard 561007b31SStefan Hajnoczi * 661007b31SStefan Hajnoczi * Permission is hereby granted, free of charge, to any person obtaining a copy 761007b31SStefan Hajnoczi * of this software and associated documentation files (the "Software"), to deal 861007b31SStefan Hajnoczi * in the Software without restriction, including without limitation the rights 961007b31SStefan Hajnoczi * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 1061007b31SStefan Hajnoczi * copies of the Software, and to permit persons to whom the Software is 1161007b31SStefan Hajnoczi * furnished to do so, subject to the following conditions: 1261007b31SStefan Hajnoczi * 1361007b31SStefan Hajnoczi * The above copyright notice and this permission notice shall be included in 1461007b31SStefan Hajnoczi * all copies or substantial portions of the Software. 1561007b31SStefan Hajnoczi * 1661007b31SStefan Hajnoczi * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 1761007b31SStefan Hajnoczi * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 1861007b31SStefan Hajnoczi * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 1961007b31SStefan Hajnoczi * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 2061007b31SStefan Hajnoczi * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 2161007b31SStefan Hajnoczi * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 2261007b31SStefan Hajnoczi * THE SOFTWARE. 2361007b31SStefan Hajnoczi */ 2461007b31SStefan Hajnoczi 2580c71a24SPeter Maydell #include "qemu/osdep.h" 2661007b31SStefan Hajnoczi #include "trace.h" 277f0e9da6SMax Reitz #include "sysemu/block-backend.h" 2861007b31SStefan Hajnoczi #include "block/blockjob.h" 2961007b31SStefan Hajnoczi #include "block/block_int.h" 3076f4afb4SAlberto Garcia #include "block/throttle-groups.h" 31f348b6d1SVeronia Bahaa #include "qemu/cutils.h" 32da34e65cSMarkus Armbruster #include "qapi/error.h" 33d49b6836SMarkus Armbruster #include "qemu/error-report.h" 3461007b31SStefan Hajnoczi 3561007b31SStefan Hajnoczi #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */ 3661007b31SStefan Hajnoczi 3761007b31SStefan Hajnoczi static BlockAIOCB *bdrv_aio_readv_em(BlockDriverState *bs, 3861007b31SStefan Hajnoczi int64_t sector_num, QEMUIOVector *qiov, int nb_sectors, 3961007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque); 4061007b31SStefan Hajnoczi static BlockAIOCB *bdrv_aio_writev_em(BlockDriverState *bs, 4161007b31SStefan Hajnoczi int64_t sector_num, QEMUIOVector *qiov, int nb_sectors, 4261007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque); 4361007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_readv_em(BlockDriverState *bs, 4461007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, 4561007b31SStefan Hajnoczi QEMUIOVector *iov); 4661007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_writev_em(BlockDriverState *bs, 4761007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, 4861007b31SStefan Hajnoczi QEMUIOVector *iov); 4961007b31SStefan Hajnoczi static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs, 5061007b31SStefan Hajnoczi int64_t sector_num, 5161007b31SStefan Hajnoczi QEMUIOVector *qiov, 5261007b31SStefan Hajnoczi int nb_sectors, 5361007b31SStefan Hajnoczi BdrvRequestFlags flags, 5461007b31SStefan Hajnoczi BlockCompletionFunc *cb, 5561007b31SStefan Hajnoczi void *opaque, 5661007b31SStefan Hajnoczi bool is_write); 5761007b31SStefan Hajnoczi static void coroutine_fn bdrv_co_do_rw(void *opaque); 5861007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs, 5961007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, BdrvRequestFlags flags); 6061007b31SStefan Hajnoczi 6161007b31SStefan Hajnoczi /* throttling disk I/O limits */ 6261007b31SStefan Hajnoczi void bdrv_set_io_limits(BlockDriverState *bs, 6361007b31SStefan Hajnoczi ThrottleConfig *cfg) 6461007b31SStefan Hajnoczi { 6576f4afb4SAlberto Garcia throttle_group_config(bs, cfg); 6661007b31SStefan Hajnoczi } 6761007b31SStefan Hajnoczi 68ce0f1412SPaolo Bonzini void bdrv_no_throttling_begin(BlockDriverState *bs) 6961007b31SStefan Hajnoczi { 70ce0f1412SPaolo Bonzini if (bs->io_limits_disabled++ == 0) { 71a72f6414SPaolo Bonzini throttle_group_restart_bs(bs); 72ce0f1412SPaolo Bonzini } 73ce0f1412SPaolo Bonzini } 74ce0f1412SPaolo Bonzini 75ce0f1412SPaolo Bonzini void bdrv_no_throttling_end(BlockDriverState *bs) 76ce0f1412SPaolo Bonzini { 77ce0f1412SPaolo Bonzini assert(bs->io_limits_disabled); 78ce0f1412SPaolo Bonzini --bs->io_limits_disabled; 7961007b31SStefan Hajnoczi } 8061007b31SStefan Hajnoczi 8161007b31SStefan Hajnoczi void bdrv_io_limits_disable(BlockDriverState *bs) 8261007b31SStefan Hajnoczi { 83ce0f1412SPaolo Bonzini assert(bs->throttle_state); 84ce0f1412SPaolo Bonzini bdrv_no_throttling_begin(bs); 8576f4afb4SAlberto Garcia throttle_group_unregister_bs(bs); 86ce0f1412SPaolo Bonzini bdrv_no_throttling_end(bs); 8761007b31SStefan Hajnoczi } 8861007b31SStefan Hajnoczi 8961007b31SStefan Hajnoczi /* should be called before bdrv_set_io_limits if a limit is set */ 9076f4afb4SAlberto Garcia void bdrv_io_limits_enable(BlockDriverState *bs, const char *group) 9161007b31SStefan Hajnoczi { 92ce0f1412SPaolo Bonzini assert(!bs->throttle_state); 9376f4afb4SAlberto Garcia throttle_group_register_bs(bs, group); 9461007b31SStefan Hajnoczi } 9561007b31SStefan Hajnoczi 9676f4afb4SAlberto Garcia void bdrv_io_limits_update_group(BlockDriverState *bs, const char *group) 9761007b31SStefan Hajnoczi { 9876f4afb4SAlberto Garcia /* this bs is not part of any group */ 9976f4afb4SAlberto Garcia if (!bs->throttle_state) { 10061007b31SStefan Hajnoczi return; 10161007b31SStefan Hajnoczi } 10261007b31SStefan Hajnoczi 10376f4afb4SAlberto Garcia /* this bs is a part of the same group than the one we want */ 10476f4afb4SAlberto Garcia if (!g_strcmp0(throttle_group_get_name(bs), group)) { 10576f4afb4SAlberto Garcia return; 10676f4afb4SAlberto Garcia } 10776f4afb4SAlberto Garcia 10876f4afb4SAlberto Garcia /* need to change the group this bs belong to */ 10976f4afb4SAlberto Garcia bdrv_io_limits_disable(bs); 11076f4afb4SAlberto Garcia bdrv_io_limits_enable(bs, group); 11161007b31SStefan Hajnoczi } 11261007b31SStefan Hajnoczi 11361007b31SStefan Hajnoczi void bdrv_setup_io_funcs(BlockDriver *bdrv) 11461007b31SStefan Hajnoczi { 11561007b31SStefan Hajnoczi /* Block drivers without coroutine functions need emulation */ 11661007b31SStefan Hajnoczi if (!bdrv->bdrv_co_readv) { 11761007b31SStefan Hajnoczi bdrv->bdrv_co_readv = bdrv_co_readv_em; 11861007b31SStefan Hajnoczi bdrv->bdrv_co_writev = bdrv_co_writev_em; 11961007b31SStefan Hajnoczi 12061007b31SStefan Hajnoczi /* bdrv_co_readv_em()/brdv_co_writev_em() work in terms of aio, so if 12161007b31SStefan Hajnoczi * the block driver lacks aio we need to emulate that too. 12261007b31SStefan Hajnoczi */ 12361007b31SStefan Hajnoczi if (!bdrv->bdrv_aio_readv) { 12461007b31SStefan Hajnoczi /* add AIO emulation layer */ 12561007b31SStefan Hajnoczi bdrv->bdrv_aio_readv = bdrv_aio_readv_em; 12661007b31SStefan Hajnoczi bdrv->bdrv_aio_writev = bdrv_aio_writev_em; 12761007b31SStefan Hajnoczi } 12861007b31SStefan Hajnoczi } 12961007b31SStefan Hajnoczi } 13061007b31SStefan Hajnoczi 13161007b31SStefan Hajnoczi void bdrv_refresh_limits(BlockDriverState *bs, Error **errp) 13261007b31SStefan Hajnoczi { 13361007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 13461007b31SStefan Hajnoczi Error *local_err = NULL; 13561007b31SStefan Hajnoczi 13661007b31SStefan Hajnoczi memset(&bs->bl, 0, sizeof(bs->bl)); 13761007b31SStefan Hajnoczi 13861007b31SStefan Hajnoczi if (!drv) { 13961007b31SStefan Hajnoczi return; 14061007b31SStefan Hajnoczi } 14161007b31SStefan Hajnoczi 14261007b31SStefan Hajnoczi /* Take some limits from the children as a default */ 14361007b31SStefan Hajnoczi if (bs->file) { 1449a4f4c31SKevin Wolf bdrv_refresh_limits(bs->file->bs, &local_err); 14561007b31SStefan Hajnoczi if (local_err) { 14661007b31SStefan Hajnoczi error_propagate(errp, local_err); 14761007b31SStefan Hajnoczi return; 14861007b31SStefan Hajnoczi } 1499a4f4c31SKevin Wolf bs->bl.opt_transfer_length = bs->file->bs->bl.opt_transfer_length; 1509a4f4c31SKevin Wolf bs->bl.max_transfer_length = bs->file->bs->bl.max_transfer_length; 1519a4f4c31SKevin Wolf bs->bl.min_mem_alignment = bs->file->bs->bl.min_mem_alignment; 1529a4f4c31SKevin Wolf bs->bl.opt_mem_alignment = bs->file->bs->bl.opt_mem_alignment; 153bd44feb7SStefan Hajnoczi bs->bl.max_iov = bs->file->bs->bl.max_iov; 15461007b31SStefan Hajnoczi } else { 1554196d2f0SDenis V. Lunev bs->bl.min_mem_alignment = 512; 156459b4e66SDenis V. Lunev bs->bl.opt_mem_alignment = getpagesize(); 157bd44feb7SStefan Hajnoczi 158bd44feb7SStefan Hajnoczi /* Safe default since most protocols use readv()/writev()/etc */ 159bd44feb7SStefan Hajnoczi bs->bl.max_iov = IOV_MAX; 16061007b31SStefan Hajnoczi } 16161007b31SStefan Hajnoczi 162760e0063SKevin Wolf if (bs->backing) { 163760e0063SKevin Wolf bdrv_refresh_limits(bs->backing->bs, &local_err); 16461007b31SStefan Hajnoczi if (local_err) { 16561007b31SStefan Hajnoczi error_propagate(errp, local_err); 16661007b31SStefan Hajnoczi return; 16761007b31SStefan Hajnoczi } 16861007b31SStefan Hajnoczi bs->bl.opt_transfer_length = 16961007b31SStefan Hajnoczi MAX(bs->bl.opt_transfer_length, 170760e0063SKevin Wolf bs->backing->bs->bl.opt_transfer_length); 17161007b31SStefan Hajnoczi bs->bl.max_transfer_length = 17261007b31SStefan Hajnoczi MIN_NON_ZERO(bs->bl.max_transfer_length, 173760e0063SKevin Wolf bs->backing->bs->bl.max_transfer_length); 17461007b31SStefan Hajnoczi bs->bl.opt_mem_alignment = 17561007b31SStefan Hajnoczi MAX(bs->bl.opt_mem_alignment, 176760e0063SKevin Wolf bs->backing->bs->bl.opt_mem_alignment); 1774196d2f0SDenis V. Lunev bs->bl.min_mem_alignment = 1784196d2f0SDenis V. Lunev MAX(bs->bl.min_mem_alignment, 179760e0063SKevin Wolf bs->backing->bs->bl.min_mem_alignment); 180bd44feb7SStefan Hajnoczi bs->bl.max_iov = 181bd44feb7SStefan Hajnoczi MIN(bs->bl.max_iov, 182bd44feb7SStefan Hajnoczi bs->backing->bs->bl.max_iov); 18361007b31SStefan Hajnoczi } 18461007b31SStefan Hajnoczi 18561007b31SStefan Hajnoczi /* Then let the driver override it */ 18661007b31SStefan Hajnoczi if (drv->bdrv_refresh_limits) { 18761007b31SStefan Hajnoczi drv->bdrv_refresh_limits(bs, errp); 18861007b31SStefan Hajnoczi } 18961007b31SStefan Hajnoczi } 19061007b31SStefan Hajnoczi 19161007b31SStefan Hajnoczi /** 19261007b31SStefan Hajnoczi * The copy-on-read flag is actually a reference count so multiple users may 19361007b31SStefan Hajnoczi * use the feature without worrying about clobbering its previous state. 19461007b31SStefan Hajnoczi * Copy-on-read stays enabled until all users have called to disable it. 19561007b31SStefan Hajnoczi */ 19661007b31SStefan Hajnoczi void bdrv_enable_copy_on_read(BlockDriverState *bs) 19761007b31SStefan Hajnoczi { 19861007b31SStefan Hajnoczi bs->copy_on_read++; 19961007b31SStefan Hajnoczi } 20061007b31SStefan Hajnoczi 20161007b31SStefan Hajnoczi void bdrv_disable_copy_on_read(BlockDriverState *bs) 20261007b31SStefan Hajnoczi { 20361007b31SStefan Hajnoczi assert(bs->copy_on_read > 0); 20461007b31SStefan Hajnoczi bs->copy_on_read--; 20561007b31SStefan Hajnoczi } 20661007b31SStefan Hajnoczi 20761007b31SStefan Hajnoczi /* Check if any requests are in-flight (including throttled requests) */ 208439db28cSKevin Wolf bool bdrv_requests_pending(BlockDriverState *bs) 20961007b31SStefan Hajnoczi { 21037a639a7SKevin Wolf BdrvChild *child; 21137a639a7SKevin Wolf 21261007b31SStefan Hajnoczi if (!QLIST_EMPTY(&bs->tracked_requests)) { 21361007b31SStefan Hajnoczi return true; 21461007b31SStefan Hajnoczi } 21561007b31SStefan Hajnoczi if (!qemu_co_queue_empty(&bs->throttled_reqs[0])) { 21661007b31SStefan Hajnoczi return true; 21761007b31SStefan Hajnoczi } 21861007b31SStefan Hajnoczi if (!qemu_co_queue_empty(&bs->throttled_reqs[1])) { 21961007b31SStefan Hajnoczi return true; 22061007b31SStefan Hajnoczi } 22137a639a7SKevin Wolf 22237a639a7SKevin Wolf QLIST_FOREACH(child, &bs->children, next) { 22337a639a7SKevin Wolf if (bdrv_requests_pending(child->bs)) { 22461007b31SStefan Hajnoczi return true; 22561007b31SStefan Hajnoczi } 22661007b31SStefan Hajnoczi } 22737a639a7SKevin Wolf 22861007b31SStefan Hajnoczi return false; 22961007b31SStefan Hajnoczi } 23061007b31SStefan Hajnoczi 23167da1dc5SFam Zheng static void bdrv_drain_recurse(BlockDriverState *bs) 23267da1dc5SFam Zheng { 23367da1dc5SFam Zheng BdrvChild *child; 23467da1dc5SFam Zheng 23567da1dc5SFam Zheng if (bs->drv && bs->drv->bdrv_drain) { 23667da1dc5SFam Zheng bs->drv->bdrv_drain(bs); 23767da1dc5SFam Zheng } 23867da1dc5SFam Zheng QLIST_FOREACH(child, &bs->children, next) { 23967da1dc5SFam Zheng bdrv_drain_recurse(child->bs); 24067da1dc5SFam Zheng } 24167da1dc5SFam Zheng } 24267da1dc5SFam Zheng 243a77fd4bbSFam Zheng typedef struct { 244a77fd4bbSFam Zheng Coroutine *co; 245a77fd4bbSFam Zheng BlockDriverState *bs; 246a77fd4bbSFam Zheng QEMUBH *bh; 247a77fd4bbSFam Zheng bool done; 248a77fd4bbSFam Zheng } BdrvCoDrainData; 249a77fd4bbSFam Zheng 250b6e84c97SPaolo Bonzini static void bdrv_drain_poll(BlockDriverState *bs) 251b6e84c97SPaolo Bonzini { 252b6e84c97SPaolo Bonzini bool busy = true; 253b6e84c97SPaolo Bonzini 254b6e84c97SPaolo Bonzini while (busy) { 255b6e84c97SPaolo Bonzini /* Keep iterating */ 256b6e84c97SPaolo Bonzini busy = bdrv_requests_pending(bs); 257b6e84c97SPaolo Bonzini busy |= aio_poll(bdrv_get_aio_context(bs), busy); 258b6e84c97SPaolo Bonzini } 259b6e84c97SPaolo Bonzini } 260b6e84c97SPaolo Bonzini 261a77fd4bbSFam Zheng static void bdrv_co_drain_bh_cb(void *opaque) 262a77fd4bbSFam Zheng { 263a77fd4bbSFam Zheng BdrvCoDrainData *data = opaque; 264a77fd4bbSFam Zheng Coroutine *co = data->co; 265a77fd4bbSFam Zheng 266a77fd4bbSFam Zheng qemu_bh_delete(data->bh); 267b6e84c97SPaolo Bonzini bdrv_drain_poll(data->bs); 268a77fd4bbSFam Zheng data->done = true; 269a77fd4bbSFam Zheng qemu_coroutine_enter(co, NULL); 270a77fd4bbSFam Zheng } 271a77fd4bbSFam Zheng 272b6e84c97SPaolo Bonzini static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs) 273a77fd4bbSFam Zheng { 274a77fd4bbSFam Zheng BdrvCoDrainData data; 275a77fd4bbSFam Zheng 276a77fd4bbSFam Zheng /* Calling bdrv_drain() from a BH ensures the current coroutine yields and 277a77fd4bbSFam Zheng * other coroutines run if they were queued from 278a77fd4bbSFam Zheng * qemu_co_queue_run_restart(). */ 279a77fd4bbSFam Zheng 280a77fd4bbSFam Zheng assert(qemu_in_coroutine()); 281a77fd4bbSFam Zheng data = (BdrvCoDrainData) { 282a77fd4bbSFam Zheng .co = qemu_coroutine_self(), 283a77fd4bbSFam Zheng .bs = bs, 284a77fd4bbSFam Zheng .done = false, 285a77fd4bbSFam Zheng .bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_drain_bh_cb, &data), 286a77fd4bbSFam Zheng }; 287a77fd4bbSFam Zheng qemu_bh_schedule(data.bh); 288a77fd4bbSFam Zheng 289a77fd4bbSFam Zheng qemu_coroutine_yield(); 290a77fd4bbSFam Zheng /* If we are resumed from some other event (such as an aio completion or a 291a77fd4bbSFam Zheng * timer callback), it is a bug in the caller that should be fixed. */ 292a77fd4bbSFam Zheng assert(data.done); 293a77fd4bbSFam Zheng } 294a77fd4bbSFam Zheng 29561007b31SStefan Hajnoczi /* 29667da1dc5SFam Zheng * Wait for pending requests to complete on a single BlockDriverState subtree, 29767da1dc5SFam Zheng * and suspend block driver's internal I/O until next request arrives. 29861007b31SStefan Hajnoczi * 29961007b31SStefan Hajnoczi * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState 30061007b31SStefan Hajnoczi * AioContext. 3017a63f3cdSStefan Hajnoczi * 3027a63f3cdSStefan Hajnoczi * Only this BlockDriverState's AioContext is run, so in-flight requests must 3037a63f3cdSStefan Hajnoczi * not depend on events in other AioContexts. In that case, use 3047a63f3cdSStefan Hajnoczi * bdrv_drain_all() instead. 30561007b31SStefan Hajnoczi */ 306b6e84c97SPaolo Bonzini void coroutine_fn bdrv_co_drain(BlockDriverState *bs) 307b6e84c97SPaolo Bonzini { 308ce0f1412SPaolo Bonzini bdrv_no_throttling_begin(bs); 3096b98bd64SPaolo Bonzini bdrv_io_unplugged_begin(bs); 310b6e84c97SPaolo Bonzini bdrv_drain_recurse(bs); 311b6e84c97SPaolo Bonzini bdrv_co_yield_to_drain(bs); 3126b98bd64SPaolo Bonzini bdrv_io_unplugged_end(bs); 313ce0f1412SPaolo Bonzini bdrv_no_throttling_end(bs); 314b6e84c97SPaolo Bonzini } 315b6e84c97SPaolo Bonzini 31661007b31SStefan Hajnoczi void bdrv_drain(BlockDriverState *bs) 31761007b31SStefan Hajnoczi { 318ce0f1412SPaolo Bonzini bdrv_no_throttling_begin(bs); 3196b98bd64SPaolo Bonzini bdrv_io_unplugged_begin(bs); 32067da1dc5SFam Zheng bdrv_drain_recurse(bs); 321a77fd4bbSFam Zheng if (qemu_in_coroutine()) { 322b6e84c97SPaolo Bonzini bdrv_co_yield_to_drain(bs); 323b6e84c97SPaolo Bonzini } else { 324b6e84c97SPaolo Bonzini bdrv_drain_poll(bs); 32561007b31SStefan Hajnoczi } 3266b98bd64SPaolo Bonzini bdrv_io_unplugged_end(bs); 327ce0f1412SPaolo Bonzini bdrv_no_throttling_end(bs); 32861007b31SStefan Hajnoczi } 32961007b31SStefan Hajnoczi 33061007b31SStefan Hajnoczi /* 33161007b31SStefan Hajnoczi * Wait for pending requests to complete across all BlockDriverStates 33261007b31SStefan Hajnoczi * 33361007b31SStefan Hajnoczi * This function does not flush data to disk, use bdrv_flush_all() for that 33461007b31SStefan Hajnoczi * after calling this function. 33561007b31SStefan Hajnoczi */ 33661007b31SStefan Hajnoczi void bdrv_drain_all(void) 33761007b31SStefan Hajnoczi { 33861007b31SStefan Hajnoczi /* Always run first iteration so any pending completion BHs run */ 33961007b31SStefan Hajnoczi bool busy = true; 34061007b31SStefan Hajnoczi BlockDriverState *bs = NULL; 341f406c03cSAlexander Yarygin GSList *aio_ctxs = NULL, *ctx; 34261007b31SStefan Hajnoczi 34361007b31SStefan Hajnoczi while ((bs = bdrv_next(bs))) { 34461007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 34561007b31SStefan Hajnoczi 34661007b31SStefan Hajnoczi aio_context_acquire(aio_context); 34761007b31SStefan Hajnoczi if (bs->job) { 34861007b31SStefan Hajnoczi block_job_pause(bs->job); 34961007b31SStefan Hajnoczi } 350ce0f1412SPaolo Bonzini bdrv_no_throttling_begin(bs); 3516b98bd64SPaolo Bonzini bdrv_io_unplugged_begin(bs); 3529dcf8ecdSPaolo Bonzini bdrv_drain_recurse(bs); 35361007b31SStefan Hajnoczi aio_context_release(aio_context); 354f406c03cSAlexander Yarygin 355764ba3aeSAlberto Garcia if (!g_slist_find(aio_ctxs, aio_context)) { 356f406c03cSAlexander Yarygin aio_ctxs = g_slist_prepend(aio_ctxs, aio_context); 357f406c03cSAlexander Yarygin } 35861007b31SStefan Hajnoczi } 35961007b31SStefan Hajnoczi 3607a63f3cdSStefan Hajnoczi /* Note that completion of an asynchronous I/O operation can trigger any 3617a63f3cdSStefan Hajnoczi * number of other I/O operations on other devices---for example a 3627a63f3cdSStefan Hajnoczi * coroutine can submit an I/O request to another device in response to 3637a63f3cdSStefan Hajnoczi * request completion. Therefore we must keep looping until there was no 3647a63f3cdSStefan Hajnoczi * more activity rather than simply draining each device independently. 3657a63f3cdSStefan Hajnoczi */ 36661007b31SStefan Hajnoczi while (busy) { 36761007b31SStefan Hajnoczi busy = false; 368f406c03cSAlexander Yarygin 369f406c03cSAlexander Yarygin for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) { 370f406c03cSAlexander Yarygin AioContext *aio_context = ctx->data; 37161007b31SStefan Hajnoczi bs = NULL; 37261007b31SStefan Hajnoczi 37361007b31SStefan Hajnoczi aio_context_acquire(aio_context); 374f406c03cSAlexander Yarygin while ((bs = bdrv_next(bs))) { 375f406c03cSAlexander Yarygin if (aio_context == bdrv_get_aio_context(bs)) { 376f406c03cSAlexander Yarygin if (bdrv_requests_pending(bs)) { 377f406c03cSAlexander Yarygin busy = true; 378f406c03cSAlexander Yarygin aio_poll(aio_context, busy); 379f406c03cSAlexander Yarygin } 380f406c03cSAlexander Yarygin } 381f406c03cSAlexander Yarygin } 382f406c03cSAlexander Yarygin busy |= aio_poll(aio_context, false); 38361007b31SStefan Hajnoczi aio_context_release(aio_context); 38461007b31SStefan Hajnoczi } 38561007b31SStefan Hajnoczi } 38661007b31SStefan Hajnoczi 38761007b31SStefan Hajnoczi bs = NULL; 38861007b31SStefan Hajnoczi while ((bs = bdrv_next(bs))) { 38961007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 39061007b31SStefan Hajnoczi 39161007b31SStefan Hajnoczi aio_context_acquire(aio_context); 3926b98bd64SPaolo Bonzini bdrv_io_unplugged_end(bs); 393ce0f1412SPaolo Bonzini bdrv_no_throttling_end(bs); 39461007b31SStefan Hajnoczi if (bs->job) { 39561007b31SStefan Hajnoczi block_job_resume(bs->job); 39661007b31SStefan Hajnoczi } 39761007b31SStefan Hajnoczi aio_context_release(aio_context); 39861007b31SStefan Hajnoczi } 399f406c03cSAlexander Yarygin g_slist_free(aio_ctxs); 40061007b31SStefan Hajnoczi } 40161007b31SStefan Hajnoczi 40261007b31SStefan Hajnoczi /** 40361007b31SStefan Hajnoczi * Remove an active request from the tracked requests list 40461007b31SStefan Hajnoczi * 40561007b31SStefan Hajnoczi * This function should be called when a tracked request is completing. 40661007b31SStefan Hajnoczi */ 40761007b31SStefan Hajnoczi static void tracked_request_end(BdrvTrackedRequest *req) 40861007b31SStefan Hajnoczi { 40961007b31SStefan Hajnoczi if (req->serialising) { 41061007b31SStefan Hajnoczi req->bs->serialising_in_flight--; 41161007b31SStefan Hajnoczi } 41261007b31SStefan Hajnoczi 41361007b31SStefan Hajnoczi QLIST_REMOVE(req, list); 41461007b31SStefan Hajnoczi qemu_co_queue_restart_all(&req->wait_queue); 41561007b31SStefan Hajnoczi } 41661007b31SStefan Hajnoczi 41761007b31SStefan Hajnoczi /** 41861007b31SStefan Hajnoczi * Add an active request to the tracked requests list 41961007b31SStefan Hajnoczi */ 42061007b31SStefan Hajnoczi static void tracked_request_begin(BdrvTrackedRequest *req, 42161007b31SStefan Hajnoczi BlockDriverState *bs, 42261007b31SStefan Hajnoczi int64_t offset, 423ebde595cSFam Zheng unsigned int bytes, 424ebde595cSFam Zheng enum BdrvTrackedRequestType type) 42561007b31SStefan Hajnoczi { 42661007b31SStefan Hajnoczi *req = (BdrvTrackedRequest){ 42761007b31SStefan Hajnoczi .bs = bs, 42861007b31SStefan Hajnoczi .offset = offset, 42961007b31SStefan Hajnoczi .bytes = bytes, 430ebde595cSFam Zheng .type = type, 43161007b31SStefan Hajnoczi .co = qemu_coroutine_self(), 43261007b31SStefan Hajnoczi .serialising = false, 43361007b31SStefan Hajnoczi .overlap_offset = offset, 43461007b31SStefan Hajnoczi .overlap_bytes = bytes, 43561007b31SStefan Hajnoczi }; 43661007b31SStefan Hajnoczi 43761007b31SStefan Hajnoczi qemu_co_queue_init(&req->wait_queue); 43861007b31SStefan Hajnoczi 43961007b31SStefan Hajnoczi QLIST_INSERT_HEAD(&bs->tracked_requests, req, list); 44061007b31SStefan Hajnoczi } 44161007b31SStefan Hajnoczi 44261007b31SStefan Hajnoczi static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align) 44361007b31SStefan Hajnoczi { 44461007b31SStefan Hajnoczi int64_t overlap_offset = req->offset & ~(align - 1); 44561007b31SStefan Hajnoczi unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align) 44661007b31SStefan Hajnoczi - overlap_offset; 44761007b31SStefan Hajnoczi 44861007b31SStefan Hajnoczi if (!req->serialising) { 44961007b31SStefan Hajnoczi req->bs->serialising_in_flight++; 45061007b31SStefan Hajnoczi req->serialising = true; 45161007b31SStefan Hajnoczi } 45261007b31SStefan Hajnoczi 45361007b31SStefan Hajnoczi req->overlap_offset = MIN(req->overlap_offset, overlap_offset); 45461007b31SStefan Hajnoczi req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes); 45561007b31SStefan Hajnoczi } 45661007b31SStefan Hajnoczi 45761007b31SStefan Hajnoczi /** 45861007b31SStefan Hajnoczi * Round a region to cluster boundaries 45961007b31SStefan Hajnoczi */ 46061007b31SStefan Hajnoczi void bdrv_round_to_clusters(BlockDriverState *bs, 46161007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, 46261007b31SStefan Hajnoczi int64_t *cluster_sector_num, 46361007b31SStefan Hajnoczi int *cluster_nb_sectors) 46461007b31SStefan Hajnoczi { 46561007b31SStefan Hajnoczi BlockDriverInfo bdi; 46661007b31SStefan Hajnoczi 46761007b31SStefan Hajnoczi if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 46861007b31SStefan Hajnoczi *cluster_sector_num = sector_num; 46961007b31SStefan Hajnoczi *cluster_nb_sectors = nb_sectors; 47061007b31SStefan Hajnoczi } else { 47161007b31SStefan Hajnoczi int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE; 47261007b31SStefan Hajnoczi *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c); 47361007b31SStefan Hajnoczi *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num + 47461007b31SStefan Hajnoczi nb_sectors, c); 47561007b31SStefan Hajnoczi } 47661007b31SStefan Hajnoczi } 47761007b31SStefan Hajnoczi 47861007b31SStefan Hajnoczi static int bdrv_get_cluster_size(BlockDriverState *bs) 47961007b31SStefan Hajnoczi { 48061007b31SStefan Hajnoczi BlockDriverInfo bdi; 48161007b31SStefan Hajnoczi int ret; 48261007b31SStefan Hajnoczi 48361007b31SStefan Hajnoczi ret = bdrv_get_info(bs, &bdi); 48461007b31SStefan Hajnoczi if (ret < 0 || bdi.cluster_size == 0) { 48561007b31SStefan Hajnoczi return bs->request_alignment; 48661007b31SStefan Hajnoczi } else { 48761007b31SStefan Hajnoczi return bdi.cluster_size; 48861007b31SStefan Hajnoczi } 48961007b31SStefan Hajnoczi } 49061007b31SStefan Hajnoczi 49161007b31SStefan Hajnoczi static bool tracked_request_overlaps(BdrvTrackedRequest *req, 49261007b31SStefan Hajnoczi int64_t offset, unsigned int bytes) 49361007b31SStefan Hajnoczi { 49461007b31SStefan Hajnoczi /* aaaa bbbb */ 49561007b31SStefan Hajnoczi if (offset >= req->overlap_offset + req->overlap_bytes) { 49661007b31SStefan Hajnoczi return false; 49761007b31SStefan Hajnoczi } 49861007b31SStefan Hajnoczi /* bbbb aaaa */ 49961007b31SStefan Hajnoczi if (req->overlap_offset >= offset + bytes) { 50061007b31SStefan Hajnoczi return false; 50161007b31SStefan Hajnoczi } 50261007b31SStefan Hajnoczi return true; 50361007b31SStefan Hajnoczi } 50461007b31SStefan Hajnoczi 50561007b31SStefan Hajnoczi static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self) 50661007b31SStefan Hajnoczi { 50761007b31SStefan Hajnoczi BlockDriverState *bs = self->bs; 50861007b31SStefan Hajnoczi BdrvTrackedRequest *req; 50961007b31SStefan Hajnoczi bool retry; 51061007b31SStefan Hajnoczi bool waited = false; 51161007b31SStefan Hajnoczi 51261007b31SStefan Hajnoczi if (!bs->serialising_in_flight) { 51361007b31SStefan Hajnoczi return false; 51461007b31SStefan Hajnoczi } 51561007b31SStefan Hajnoczi 51661007b31SStefan Hajnoczi do { 51761007b31SStefan Hajnoczi retry = false; 51861007b31SStefan Hajnoczi QLIST_FOREACH(req, &bs->tracked_requests, list) { 51961007b31SStefan Hajnoczi if (req == self || (!req->serialising && !self->serialising)) { 52061007b31SStefan Hajnoczi continue; 52161007b31SStefan Hajnoczi } 52261007b31SStefan Hajnoczi if (tracked_request_overlaps(req, self->overlap_offset, 52361007b31SStefan Hajnoczi self->overlap_bytes)) 52461007b31SStefan Hajnoczi { 52561007b31SStefan Hajnoczi /* Hitting this means there was a reentrant request, for 52661007b31SStefan Hajnoczi * example, a block driver issuing nested requests. This must 52761007b31SStefan Hajnoczi * never happen since it means deadlock. 52861007b31SStefan Hajnoczi */ 52961007b31SStefan Hajnoczi assert(qemu_coroutine_self() != req->co); 53061007b31SStefan Hajnoczi 53161007b31SStefan Hajnoczi /* If the request is already (indirectly) waiting for us, or 53261007b31SStefan Hajnoczi * will wait for us as soon as it wakes up, then just go on 53361007b31SStefan Hajnoczi * (instead of producing a deadlock in the former case). */ 53461007b31SStefan Hajnoczi if (!req->waiting_for) { 53561007b31SStefan Hajnoczi self->waiting_for = req; 53661007b31SStefan Hajnoczi qemu_co_queue_wait(&req->wait_queue); 53761007b31SStefan Hajnoczi self->waiting_for = NULL; 53861007b31SStefan Hajnoczi retry = true; 53961007b31SStefan Hajnoczi waited = true; 54061007b31SStefan Hajnoczi break; 54161007b31SStefan Hajnoczi } 54261007b31SStefan Hajnoczi } 54361007b31SStefan Hajnoczi } 54461007b31SStefan Hajnoczi } while (retry); 54561007b31SStefan Hajnoczi 54661007b31SStefan Hajnoczi return waited; 54761007b31SStefan Hajnoczi } 54861007b31SStefan Hajnoczi 54961007b31SStefan Hajnoczi static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset, 55061007b31SStefan Hajnoczi size_t size) 55161007b31SStefan Hajnoczi { 55261007b31SStefan Hajnoczi if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) { 55361007b31SStefan Hajnoczi return -EIO; 55461007b31SStefan Hajnoczi } 55561007b31SStefan Hajnoczi 55661007b31SStefan Hajnoczi if (!bdrv_is_inserted(bs)) { 55761007b31SStefan Hajnoczi return -ENOMEDIUM; 55861007b31SStefan Hajnoczi } 55961007b31SStefan Hajnoczi 56061007b31SStefan Hajnoczi if (offset < 0) { 56161007b31SStefan Hajnoczi return -EIO; 56261007b31SStefan Hajnoczi } 56361007b31SStefan Hajnoczi 56461007b31SStefan Hajnoczi return 0; 56561007b31SStefan Hajnoczi } 56661007b31SStefan Hajnoczi 56761007b31SStefan Hajnoczi static int bdrv_check_request(BlockDriverState *bs, int64_t sector_num, 56861007b31SStefan Hajnoczi int nb_sectors) 56961007b31SStefan Hajnoczi { 57061007b31SStefan Hajnoczi if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 57161007b31SStefan Hajnoczi return -EIO; 57261007b31SStefan Hajnoczi } 57361007b31SStefan Hajnoczi 57461007b31SStefan Hajnoczi return bdrv_check_byte_request(bs, sector_num * BDRV_SECTOR_SIZE, 57561007b31SStefan Hajnoczi nb_sectors * BDRV_SECTOR_SIZE); 57661007b31SStefan Hajnoczi } 57761007b31SStefan Hajnoczi 57861007b31SStefan Hajnoczi typedef struct RwCo { 57961007b31SStefan Hajnoczi BlockDriverState *bs; 58061007b31SStefan Hajnoczi int64_t offset; 58161007b31SStefan Hajnoczi QEMUIOVector *qiov; 58261007b31SStefan Hajnoczi bool is_write; 58361007b31SStefan Hajnoczi int ret; 58461007b31SStefan Hajnoczi BdrvRequestFlags flags; 58561007b31SStefan Hajnoczi } RwCo; 58661007b31SStefan Hajnoczi 58761007b31SStefan Hajnoczi static void coroutine_fn bdrv_rw_co_entry(void *opaque) 58861007b31SStefan Hajnoczi { 58961007b31SStefan Hajnoczi RwCo *rwco = opaque; 59061007b31SStefan Hajnoczi 59161007b31SStefan Hajnoczi if (!rwco->is_write) { 59261007b31SStefan Hajnoczi rwco->ret = bdrv_co_do_preadv(rwco->bs, rwco->offset, 59361007b31SStefan Hajnoczi rwco->qiov->size, rwco->qiov, 59461007b31SStefan Hajnoczi rwco->flags); 59561007b31SStefan Hajnoczi } else { 59661007b31SStefan Hajnoczi rwco->ret = bdrv_co_do_pwritev(rwco->bs, rwco->offset, 59761007b31SStefan Hajnoczi rwco->qiov->size, rwco->qiov, 59861007b31SStefan Hajnoczi rwco->flags); 59961007b31SStefan Hajnoczi } 60061007b31SStefan Hajnoczi } 60161007b31SStefan Hajnoczi 60261007b31SStefan Hajnoczi /* 60361007b31SStefan Hajnoczi * Process a vectored synchronous request using coroutines 60461007b31SStefan Hajnoczi */ 60561007b31SStefan Hajnoczi static int bdrv_prwv_co(BlockDriverState *bs, int64_t offset, 60661007b31SStefan Hajnoczi QEMUIOVector *qiov, bool is_write, 60761007b31SStefan Hajnoczi BdrvRequestFlags flags) 60861007b31SStefan Hajnoczi { 60961007b31SStefan Hajnoczi Coroutine *co; 61061007b31SStefan Hajnoczi RwCo rwco = { 61161007b31SStefan Hajnoczi .bs = bs, 61261007b31SStefan Hajnoczi .offset = offset, 61361007b31SStefan Hajnoczi .qiov = qiov, 61461007b31SStefan Hajnoczi .is_write = is_write, 61561007b31SStefan Hajnoczi .ret = NOT_DONE, 61661007b31SStefan Hajnoczi .flags = flags, 61761007b31SStefan Hajnoczi }; 61861007b31SStefan Hajnoczi 61961007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 62061007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 62161007b31SStefan Hajnoczi bdrv_rw_co_entry(&rwco); 62261007b31SStefan Hajnoczi } else { 62361007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 62461007b31SStefan Hajnoczi 62561007b31SStefan Hajnoczi co = qemu_coroutine_create(bdrv_rw_co_entry); 62661007b31SStefan Hajnoczi qemu_coroutine_enter(co, &rwco); 62761007b31SStefan Hajnoczi while (rwco.ret == NOT_DONE) { 62861007b31SStefan Hajnoczi aio_poll(aio_context, true); 62961007b31SStefan Hajnoczi } 63061007b31SStefan Hajnoczi } 63161007b31SStefan Hajnoczi return rwco.ret; 63261007b31SStefan Hajnoczi } 63361007b31SStefan Hajnoczi 63461007b31SStefan Hajnoczi /* 63561007b31SStefan Hajnoczi * Process a synchronous request using coroutines 63661007b31SStefan Hajnoczi */ 63761007b31SStefan Hajnoczi static int bdrv_rw_co(BlockDriverState *bs, int64_t sector_num, uint8_t *buf, 63861007b31SStefan Hajnoczi int nb_sectors, bool is_write, BdrvRequestFlags flags) 63961007b31SStefan Hajnoczi { 64061007b31SStefan Hajnoczi QEMUIOVector qiov; 64161007b31SStefan Hajnoczi struct iovec iov = { 64261007b31SStefan Hajnoczi .iov_base = (void *)buf, 64361007b31SStefan Hajnoczi .iov_len = nb_sectors * BDRV_SECTOR_SIZE, 64461007b31SStefan Hajnoczi }; 64561007b31SStefan Hajnoczi 64661007b31SStefan Hajnoczi if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 64761007b31SStefan Hajnoczi return -EINVAL; 64861007b31SStefan Hajnoczi } 64961007b31SStefan Hajnoczi 65061007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 65161007b31SStefan Hajnoczi return bdrv_prwv_co(bs, sector_num << BDRV_SECTOR_BITS, 65261007b31SStefan Hajnoczi &qiov, is_write, flags); 65361007b31SStefan Hajnoczi } 65461007b31SStefan Hajnoczi 65561007b31SStefan Hajnoczi /* return < 0 if error. See bdrv_write() for the return codes */ 65661007b31SStefan Hajnoczi int bdrv_read(BlockDriverState *bs, int64_t sector_num, 65761007b31SStefan Hajnoczi uint8_t *buf, int nb_sectors) 65861007b31SStefan Hajnoczi { 65961007b31SStefan Hajnoczi return bdrv_rw_co(bs, sector_num, buf, nb_sectors, false, 0); 66061007b31SStefan Hajnoczi } 66161007b31SStefan Hajnoczi 66261007b31SStefan Hajnoczi /* Return < 0 if error. Important errors are: 66361007b31SStefan Hajnoczi -EIO generic I/O error (may happen for all errors) 66461007b31SStefan Hajnoczi -ENOMEDIUM No media inserted. 66561007b31SStefan Hajnoczi -EINVAL Invalid sector number or nb_sectors 66661007b31SStefan Hajnoczi -EACCES Trying to write a read-only device 66761007b31SStefan Hajnoczi */ 66861007b31SStefan Hajnoczi int bdrv_write(BlockDriverState *bs, int64_t sector_num, 66961007b31SStefan Hajnoczi const uint8_t *buf, int nb_sectors) 67061007b31SStefan Hajnoczi { 67161007b31SStefan Hajnoczi return bdrv_rw_co(bs, sector_num, (uint8_t *)buf, nb_sectors, true, 0); 67261007b31SStefan Hajnoczi } 67361007b31SStefan Hajnoczi 67461007b31SStefan Hajnoczi int bdrv_write_zeroes(BlockDriverState *bs, int64_t sector_num, 67561007b31SStefan Hajnoczi int nb_sectors, BdrvRequestFlags flags) 67661007b31SStefan Hajnoczi { 67761007b31SStefan Hajnoczi return bdrv_rw_co(bs, sector_num, NULL, nb_sectors, true, 67861007b31SStefan Hajnoczi BDRV_REQ_ZERO_WRITE | flags); 67961007b31SStefan Hajnoczi } 68061007b31SStefan Hajnoczi 68161007b31SStefan Hajnoczi /* 68261007b31SStefan Hajnoczi * Completely zero out a block device with the help of bdrv_write_zeroes. 68361007b31SStefan Hajnoczi * The operation is sped up by checking the block status and only writing 68461007b31SStefan Hajnoczi * zeroes to the device if they currently do not return zeroes. Optional 68561007b31SStefan Hajnoczi * flags are passed through to bdrv_write_zeroes (e.g. BDRV_REQ_MAY_UNMAP). 68661007b31SStefan Hajnoczi * 68761007b31SStefan Hajnoczi * Returns < 0 on error, 0 on success. For error codes see bdrv_write(). 68861007b31SStefan Hajnoczi */ 68961007b31SStefan Hajnoczi int bdrv_make_zero(BlockDriverState *bs, BdrvRequestFlags flags) 69061007b31SStefan Hajnoczi { 69161007b31SStefan Hajnoczi int64_t target_sectors, ret, nb_sectors, sector_num = 0; 69267a0fd2aSFam Zheng BlockDriverState *file; 69361007b31SStefan Hajnoczi int n; 69461007b31SStefan Hajnoczi 69561007b31SStefan Hajnoczi target_sectors = bdrv_nb_sectors(bs); 69661007b31SStefan Hajnoczi if (target_sectors < 0) { 69761007b31SStefan Hajnoczi return target_sectors; 69861007b31SStefan Hajnoczi } 69961007b31SStefan Hajnoczi 70061007b31SStefan Hajnoczi for (;;) { 70161007b31SStefan Hajnoczi nb_sectors = MIN(target_sectors - sector_num, BDRV_REQUEST_MAX_SECTORS); 70261007b31SStefan Hajnoczi if (nb_sectors <= 0) { 70361007b31SStefan Hajnoczi return 0; 70461007b31SStefan Hajnoczi } 70567a0fd2aSFam Zheng ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n, &file); 70661007b31SStefan Hajnoczi if (ret < 0) { 70761007b31SStefan Hajnoczi error_report("error getting block status at sector %" PRId64 ": %s", 70861007b31SStefan Hajnoczi sector_num, strerror(-ret)); 70961007b31SStefan Hajnoczi return ret; 71061007b31SStefan Hajnoczi } 71161007b31SStefan Hajnoczi if (ret & BDRV_BLOCK_ZERO) { 71261007b31SStefan Hajnoczi sector_num += n; 71361007b31SStefan Hajnoczi continue; 71461007b31SStefan Hajnoczi } 71561007b31SStefan Hajnoczi ret = bdrv_write_zeroes(bs, sector_num, n, flags); 71661007b31SStefan Hajnoczi if (ret < 0) { 71761007b31SStefan Hajnoczi error_report("error writing zeroes at sector %" PRId64 ": %s", 71861007b31SStefan Hajnoczi sector_num, strerror(-ret)); 71961007b31SStefan Hajnoczi return ret; 72061007b31SStefan Hajnoczi } 72161007b31SStefan Hajnoczi sector_num += n; 72261007b31SStefan Hajnoczi } 72361007b31SStefan Hajnoczi } 72461007b31SStefan Hajnoczi 72561007b31SStefan Hajnoczi int bdrv_pread(BlockDriverState *bs, int64_t offset, void *buf, int bytes) 72661007b31SStefan Hajnoczi { 72761007b31SStefan Hajnoczi QEMUIOVector qiov; 72861007b31SStefan Hajnoczi struct iovec iov = { 72961007b31SStefan Hajnoczi .iov_base = (void *)buf, 73061007b31SStefan Hajnoczi .iov_len = bytes, 73161007b31SStefan Hajnoczi }; 73261007b31SStefan Hajnoczi int ret; 73361007b31SStefan Hajnoczi 73461007b31SStefan Hajnoczi if (bytes < 0) { 73561007b31SStefan Hajnoczi return -EINVAL; 73661007b31SStefan Hajnoczi } 73761007b31SStefan Hajnoczi 73861007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 73961007b31SStefan Hajnoczi ret = bdrv_prwv_co(bs, offset, &qiov, false, 0); 74061007b31SStefan Hajnoczi if (ret < 0) { 74161007b31SStefan Hajnoczi return ret; 74261007b31SStefan Hajnoczi } 74361007b31SStefan Hajnoczi 74461007b31SStefan Hajnoczi return bytes; 74561007b31SStefan Hajnoczi } 74661007b31SStefan Hajnoczi 74761007b31SStefan Hajnoczi int bdrv_pwritev(BlockDriverState *bs, int64_t offset, QEMUIOVector *qiov) 74861007b31SStefan Hajnoczi { 74961007b31SStefan Hajnoczi int ret; 75061007b31SStefan Hajnoczi 75161007b31SStefan Hajnoczi ret = bdrv_prwv_co(bs, offset, qiov, true, 0); 75261007b31SStefan Hajnoczi if (ret < 0) { 75361007b31SStefan Hajnoczi return ret; 75461007b31SStefan Hajnoczi } 75561007b31SStefan Hajnoczi 75661007b31SStefan Hajnoczi return qiov->size; 75761007b31SStefan Hajnoczi } 75861007b31SStefan Hajnoczi 75961007b31SStefan Hajnoczi int bdrv_pwrite(BlockDriverState *bs, int64_t offset, 76061007b31SStefan Hajnoczi const void *buf, int bytes) 76161007b31SStefan Hajnoczi { 76261007b31SStefan Hajnoczi QEMUIOVector qiov; 76361007b31SStefan Hajnoczi struct iovec iov = { 76461007b31SStefan Hajnoczi .iov_base = (void *) buf, 76561007b31SStefan Hajnoczi .iov_len = bytes, 76661007b31SStefan Hajnoczi }; 76761007b31SStefan Hajnoczi 76861007b31SStefan Hajnoczi if (bytes < 0) { 76961007b31SStefan Hajnoczi return -EINVAL; 77061007b31SStefan Hajnoczi } 77161007b31SStefan Hajnoczi 77261007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 77361007b31SStefan Hajnoczi return bdrv_pwritev(bs, offset, &qiov); 77461007b31SStefan Hajnoczi } 77561007b31SStefan Hajnoczi 77661007b31SStefan Hajnoczi /* 77761007b31SStefan Hajnoczi * Writes to the file and ensures that no writes are reordered across this 77861007b31SStefan Hajnoczi * request (acts as a barrier) 77961007b31SStefan Hajnoczi * 78061007b31SStefan Hajnoczi * Returns 0 on success, -errno in error cases. 78161007b31SStefan Hajnoczi */ 78261007b31SStefan Hajnoczi int bdrv_pwrite_sync(BlockDriverState *bs, int64_t offset, 78361007b31SStefan Hajnoczi const void *buf, int count) 78461007b31SStefan Hajnoczi { 78561007b31SStefan Hajnoczi int ret; 78661007b31SStefan Hajnoczi 78761007b31SStefan Hajnoczi ret = bdrv_pwrite(bs, offset, buf, count); 78861007b31SStefan Hajnoczi if (ret < 0) { 78961007b31SStefan Hajnoczi return ret; 79061007b31SStefan Hajnoczi } 79161007b31SStefan Hajnoczi 792855a6a93SKevin Wolf ret = bdrv_flush(bs); 793855a6a93SKevin Wolf if (ret < 0) { 794855a6a93SKevin Wolf return ret; 79561007b31SStefan Hajnoczi } 79661007b31SStefan Hajnoczi 79761007b31SStefan Hajnoczi return 0; 79861007b31SStefan Hajnoczi } 79961007b31SStefan Hajnoczi 800*166fe960SKevin Wolf static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs, 801*166fe960SKevin Wolf uint64_t offset, uint64_t bytes, 802*166fe960SKevin Wolf QEMUIOVector *qiov, int flags) 803*166fe960SKevin Wolf { 804*166fe960SKevin Wolf BlockDriver *drv = bs->drv; 805*166fe960SKevin Wolf int64_t sector_num = offset >> BDRV_SECTOR_BITS; 806*166fe960SKevin Wolf unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS; 807*166fe960SKevin Wolf 808*166fe960SKevin Wolf assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 809*166fe960SKevin Wolf assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 810*166fe960SKevin Wolf assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS); 811*166fe960SKevin Wolf 812*166fe960SKevin Wolf return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov); 813*166fe960SKevin Wolf } 814*166fe960SKevin Wolf 81561007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs, 81661007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, QEMUIOVector *qiov) 81761007b31SStefan Hajnoczi { 81861007b31SStefan Hajnoczi /* Perform I/O through a temporary buffer so that users who scribble over 81961007b31SStefan Hajnoczi * their read buffer while the operation is in progress do not end up 82061007b31SStefan Hajnoczi * modifying the image file. This is critical for zero-copy guest I/O 82161007b31SStefan Hajnoczi * where anything might happen inside guest memory. 82261007b31SStefan Hajnoczi */ 82361007b31SStefan Hajnoczi void *bounce_buffer; 82461007b31SStefan Hajnoczi 82561007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 82661007b31SStefan Hajnoczi struct iovec iov; 82761007b31SStefan Hajnoczi QEMUIOVector bounce_qiov; 82861007b31SStefan Hajnoczi int64_t cluster_sector_num; 82961007b31SStefan Hajnoczi int cluster_nb_sectors; 83061007b31SStefan Hajnoczi size_t skip_bytes; 83161007b31SStefan Hajnoczi int ret; 83261007b31SStefan Hajnoczi 83361007b31SStefan Hajnoczi /* Cover entire cluster so no additional backing file I/O is required when 83461007b31SStefan Hajnoczi * allocating cluster in the image file. 83561007b31SStefan Hajnoczi */ 83661007b31SStefan Hajnoczi bdrv_round_to_clusters(bs, sector_num, nb_sectors, 83761007b31SStefan Hajnoczi &cluster_sector_num, &cluster_nb_sectors); 83861007b31SStefan Hajnoczi 83961007b31SStefan Hajnoczi trace_bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors, 84061007b31SStefan Hajnoczi cluster_sector_num, cluster_nb_sectors); 84161007b31SStefan Hajnoczi 84261007b31SStefan Hajnoczi iov.iov_len = cluster_nb_sectors * BDRV_SECTOR_SIZE; 84361007b31SStefan Hajnoczi iov.iov_base = bounce_buffer = qemu_try_blockalign(bs, iov.iov_len); 84461007b31SStefan Hajnoczi if (bounce_buffer == NULL) { 84561007b31SStefan Hajnoczi ret = -ENOMEM; 84661007b31SStefan Hajnoczi goto err; 84761007b31SStefan Hajnoczi } 84861007b31SStefan Hajnoczi 84961007b31SStefan Hajnoczi qemu_iovec_init_external(&bounce_qiov, &iov, 1); 85061007b31SStefan Hajnoczi 851*166fe960SKevin Wolf ret = bdrv_driver_preadv(bs, cluster_sector_num * BDRV_SECTOR_SIZE, 852*166fe960SKevin Wolf cluster_nb_sectors * BDRV_SECTOR_SIZE, 853*166fe960SKevin Wolf &bounce_qiov, 0); 85461007b31SStefan Hajnoczi if (ret < 0) { 85561007b31SStefan Hajnoczi goto err; 85661007b31SStefan Hajnoczi } 85761007b31SStefan Hajnoczi 85861007b31SStefan Hajnoczi if (drv->bdrv_co_write_zeroes && 85961007b31SStefan Hajnoczi buffer_is_zero(bounce_buffer, iov.iov_len)) { 86061007b31SStefan Hajnoczi ret = bdrv_co_do_write_zeroes(bs, cluster_sector_num, 86161007b31SStefan Hajnoczi cluster_nb_sectors, 0); 86261007b31SStefan Hajnoczi } else { 86361007b31SStefan Hajnoczi /* This does not change the data on the disk, it is not necessary 86461007b31SStefan Hajnoczi * to flush even in cache=writethrough mode. 86561007b31SStefan Hajnoczi */ 86661007b31SStefan Hajnoczi ret = drv->bdrv_co_writev(bs, cluster_sector_num, cluster_nb_sectors, 86761007b31SStefan Hajnoczi &bounce_qiov); 86861007b31SStefan Hajnoczi } 86961007b31SStefan Hajnoczi 87061007b31SStefan Hajnoczi if (ret < 0) { 87161007b31SStefan Hajnoczi /* It might be okay to ignore write errors for guest requests. If this 87261007b31SStefan Hajnoczi * is a deliberate copy-on-read then we don't want to ignore the error. 87361007b31SStefan Hajnoczi * Simply report it in all cases. 87461007b31SStefan Hajnoczi */ 87561007b31SStefan Hajnoczi goto err; 87661007b31SStefan Hajnoczi } 87761007b31SStefan Hajnoczi 87861007b31SStefan Hajnoczi skip_bytes = (sector_num - cluster_sector_num) * BDRV_SECTOR_SIZE; 87961007b31SStefan Hajnoczi qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes, 88061007b31SStefan Hajnoczi nb_sectors * BDRV_SECTOR_SIZE); 88161007b31SStefan Hajnoczi 88261007b31SStefan Hajnoczi err: 88361007b31SStefan Hajnoczi qemu_vfree(bounce_buffer); 88461007b31SStefan Hajnoczi return ret; 88561007b31SStefan Hajnoczi } 88661007b31SStefan Hajnoczi 88761007b31SStefan Hajnoczi /* 88861007b31SStefan Hajnoczi * Forwards an already correctly aligned request to the BlockDriver. This 88961007b31SStefan Hajnoczi * handles copy on read and zeroing after EOF; any other features must be 89061007b31SStefan Hajnoczi * implemented by the caller. 89161007b31SStefan Hajnoczi */ 89261007b31SStefan Hajnoczi static int coroutine_fn bdrv_aligned_preadv(BlockDriverState *bs, 89361007b31SStefan Hajnoczi BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 89461007b31SStefan Hajnoczi int64_t align, QEMUIOVector *qiov, int flags) 89561007b31SStefan Hajnoczi { 89661007b31SStefan Hajnoczi int ret; 89761007b31SStefan Hajnoczi 89861007b31SStefan Hajnoczi int64_t sector_num = offset >> BDRV_SECTOR_BITS; 89961007b31SStefan Hajnoczi unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS; 90061007b31SStefan Hajnoczi 90161007b31SStefan Hajnoczi assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 90261007b31SStefan Hajnoczi assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 90361007b31SStefan Hajnoczi assert(!qiov || bytes == qiov->size); 904abb06c5aSDaniel P. Berrange assert((bs->open_flags & BDRV_O_NO_IO) == 0); 90561007b31SStefan Hajnoczi 90661007b31SStefan Hajnoczi /* Handle Copy on Read and associated serialisation */ 90761007b31SStefan Hajnoczi if (flags & BDRV_REQ_COPY_ON_READ) { 90861007b31SStefan Hajnoczi /* If we touch the same cluster it counts as an overlap. This 90961007b31SStefan Hajnoczi * guarantees that allocating writes will be serialized and not race 91061007b31SStefan Hajnoczi * with each other for the same cluster. For example, in copy-on-read 91161007b31SStefan Hajnoczi * it ensures that the CoR read and write operations are atomic and 91261007b31SStefan Hajnoczi * guest writes cannot interleave between them. */ 91361007b31SStefan Hajnoczi mark_request_serialising(req, bdrv_get_cluster_size(bs)); 91461007b31SStefan Hajnoczi } 91561007b31SStefan Hajnoczi 91661408b25SFam Zheng if (!(flags & BDRV_REQ_NO_SERIALISING)) { 91761007b31SStefan Hajnoczi wait_serialising_requests(req); 91861408b25SFam Zheng } 91961007b31SStefan Hajnoczi 92061007b31SStefan Hajnoczi if (flags & BDRV_REQ_COPY_ON_READ) { 92161007b31SStefan Hajnoczi int pnum; 92261007b31SStefan Hajnoczi 92361007b31SStefan Hajnoczi ret = bdrv_is_allocated(bs, sector_num, nb_sectors, &pnum); 92461007b31SStefan Hajnoczi if (ret < 0) { 92561007b31SStefan Hajnoczi goto out; 92661007b31SStefan Hajnoczi } 92761007b31SStefan Hajnoczi 92861007b31SStefan Hajnoczi if (!ret || pnum != nb_sectors) { 92961007b31SStefan Hajnoczi ret = bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors, qiov); 93061007b31SStefan Hajnoczi goto out; 93161007b31SStefan Hajnoczi } 93261007b31SStefan Hajnoczi } 93361007b31SStefan Hajnoczi 93461007b31SStefan Hajnoczi /* Forward the request to the BlockDriver */ 93561007b31SStefan Hajnoczi if (!bs->zero_beyond_eof) { 936*166fe960SKevin Wolf ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0); 93761007b31SStefan Hajnoczi } else { 93861007b31SStefan Hajnoczi /* Read zeros after EOF */ 93961007b31SStefan Hajnoczi int64_t total_sectors, max_nb_sectors; 94061007b31SStefan Hajnoczi 94161007b31SStefan Hajnoczi total_sectors = bdrv_nb_sectors(bs); 94261007b31SStefan Hajnoczi if (total_sectors < 0) { 94361007b31SStefan Hajnoczi ret = total_sectors; 94461007b31SStefan Hajnoczi goto out; 94561007b31SStefan Hajnoczi } 94661007b31SStefan Hajnoczi 94761007b31SStefan Hajnoczi max_nb_sectors = ROUND_UP(MAX(0, total_sectors - sector_num), 94861007b31SStefan Hajnoczi align >> BDRV_SECTOR_BITS); 94961007b31SStefan Hajnoczi if (nb_sectors < max_nb_sectors) { 950*166fe960SKevin Wolf ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0); 95161007b31SStefan Hajnoczi } else if (max_nb_sectors > 0) { 95261007b31SStefan Hajnoczi QEMUIOVector local_qiov; 95361007b31SStefan Hajnoczi 95461007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov); 95561007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, 95661007b31SStefan Hajnoczi max_nb_sectors * BDRV_SECTOR_SIZE); 95761007b31SStefan Hajnoczi 958*166fe960SKevin Wolf ret = bdrv_driver_preadv(bs, offset, 959*166fe960SKevin Wolf max_nb_sectors * BDRV_SECTOR_SIZE, 960*166fe960SKevin Wolf &local_qiov, 0); 96161007b31SStefan Hajnoczi 96261007b31SStefan Hajnoczi qemu_iovec_destroy(&local_qiov); 96361007b31SStefan Hajnoczi } else { 96461007b31SStefan Hajnoczi ret = 0; 96561007b31SStefan Hajnoczi } 96661007b31SStefan Hajnoczi 96761007b31SStefan Hajnoczi /* Reading beyond end of file is supposed to produce zeroes */ 96861007b31SStefan Hajnoczi if (ret == 0 && total_sectors < sector_num + nb_sectors) { 96961007b31SStefan Hajnoczi uint64_t offset = MAX(0, total_sectors - sector_num); 97061007b31SStefan Hajnoczi uint64_t bytes = (sector_num + nb_sectors - offset) * 97161007b31SStefan Hajnoczi BDRV_SECTOR_SIZE; 97261007b31SStefan Hajnoczi qemu_iovec_memset(qiov, offset * BDRV_SECTOR_SIZE, 0, bytes); 97361007b31SStefan Hajnoczi } 97461007b31SStefan Hajnoczi } 97561007b31SStefan Hajnoczi 97661007b31SStefan Hajnoczi out: 97761007b31SStefan Hajnoczi return ret; 97861007b31SStefan Hajnoczi } 97961007b31SStefan Hajnoczi 98061007b31SStefan Hajnoczi /* 98161007b31SStefan Hajnoczi * Handle a read request in coroutine context 98261007b31SStefan Hajnoczi */ 9831bf1cbc9SKevin Wolf int coroutine_fn bdrv_co_do_preadv(BlockDriverState *bs, 98461007b31SStefan Hajnoczi int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 98561007b31SStefan Hajnoczi BdrvRequestFlags flags) 98661007b31SStefan Hajnoczi { 98761007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 98861007b31SStefan Hajnoczi BdrvTrackedRequest req; 98961007b31SStefan Hajnoczi 990d01c07f2SFam Zheng /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */ 991d01c07f2SFam Zheng uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment); 99261007b31SStefan Hajnoczi uint8_t *head_buf = NULL; 99361007b31SStefan Hajnoczi uint8_t *tail_buf = NULL; 99461007b31SStefan Hajnoczi QEMUIOVector local_qiov; 99561007b31SStefan Hajnoczi bool use_local_qiov = false; 99661007b31SStefan Hajnoczi int ret; 99761007b31SStefan Hajnoczi 99861007b31SStefan Hajnoczi if (!drv) { 99961007b31SStefan Hajnoczi return -ENOMEDIUM; 100061007b31SStefan Hajnoczi } 100161007b31SStefan Hajnoczi 100261007b31SStefan Hajnoczi ret = bdrv_check_byte_request(bs, offset, bytes); 100361007b31SStefan Hajnoczi if (ret < 0) { 100461007b31SStefan Hajnoczi return ret; 100561007b31SStefan Hajnoczi } 100661007b31SStefan Hajnoczi 10079568b511SWen Congyang /* Don't do copy-on-read if we read data before write operation */ 100861408b25SFam Zheng if (bs->copy_on_read && !(flags & BDRV_REQ_NO_SERIALISING)) { 100961007b31SStefan Hajnoczi flags |= BDRV_REQ_COPY_ON_READ; 101061007b31SStefan Hajnoczi } 101161007b31SStefan Hajnoczi 101261007b31SStefan Hajnoczi /* throttling disk I/O */ 1013ce0f1412SPaolo Bonzini if (bs->throttle_state) { 101476f4afb4SAlberto Garcia throttle_group_co_io_limits_intercept(bs, bytes, false); 101561007b31SStefan Hajnoczi } 101661007b31SStefan Hajnoczi 101761007b31SStefan Hajnoczi /* Align read if necessary by padding qiov */ 101861007b31SStefan Hajnoczi if (offset & (align - 1)) { 101961007b31SStefan Hajnoczi head_buf = qemu_blockalign(bs, align); 102061007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 2); 102161007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 102261007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 102361007b31SStefan Hajnoczi use_local_qiov = true; 102461007b31SStefan Hajnoczi 102561007b31SStefan Hajnoczi bytes += offset & (align - 1); 102661007b31SStefan Hajnoczi offset = offset & ~(align - 1); 102761007b31SStefan Hajnoczi } 102861007b31SStefan Hajnoczi 102961007b31SStefan Hajnoczi if ((offset + bytes) & (align - 1)) { 103061007b31SStefan Hajnoczi if (!use_local_qiov) { 103161007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 1); 103261007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 103361007b31SStefan Hajnoczi use_local_qiov = true; 103461007b31SStefan Hajnoczi } 103561007b31SStefan Hajnoczi tail_buf = qemu_blockalign(bs, align); 103661007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, tail_buf, 103761007b31SStefan Hajnoczi align - ((offset + bytes) & (align - 1))); 103861007b31SStefan Hajnoczi 103961007b31SStefan Hajnoczi bytes = ROUND_UP(bytes, align); 104061007b31SStefan Hajnoczi } 104161007b31SStefan Hajnoczi 1042ebde595cSFam Zheng tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ); 104361007b31SStefan Hajnoczi ret = bdrv_aligned_preadv(bs, &req, offset, bytes, align, 104461007b31SStefan Hajnoczi use_local_qiov ? &local_qiov : qiov, 104561007b31SStefan Hajnoczi flags); 104661007b31SStefan Hajnoczi tracked_request_end(&req); 104761007b31SStefan Hajnoczi 104861007b31SStefan Hajnoczi if (use_local_qiov) { 104961007b31SStefan Hajnoczi qemu_iovec_destroy(&local_qiov); 105061007b31SStefan Hajnoczi qemu_vfree(head_buf); 105161007b31SStefan Hajnoczi qemu_vfree(tail_buf); 105261007b31SStefan Hajnoczi } 105361007b31SStefan Hajnoczi 105461007b31SStefan Hajnoczi return ret; 105561007b31SStefan Hajnoczi } 105661007b31SStefan Hajnoczi 105761007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_do_readv(BlockDriverState *bs, 105861007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, 105961007b31SStefan Hajnoczi BdrvRequestFlags flags) 106061007b31SStefan Hajnoczi { 106161007b31SStefan Hajnoczi if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 106261007b31SStefan Hajnoczi return -EINVAL; 106361007b31SStefan Hajnoczi } 106461007b31SStefan Hajnoczi 106561007b31SStefan Hajnoczi return bdrv_co_do_preadv(bs, sector_num << BDRV_SECTOR_BITS, 106661007b31SStefan Hajnoczi nb_sectors << BDRV_SECTOR_BITS, qiov, flags); 106761007b31SStefan Hajnoczi } 106861007b31SStefan Hajnoczi 106961007b31SStefan Hajnoczi int coroutine_fn bdrv_co_readv(BlockDriverState *bs, int64_t sector_num, 107061007b31SStefan Hajnoczi int nb_sectors, QEMUIOVector *qiov) 107161007b31SStefan Hajnoczi { 107261007b31SStefan Hajnoczi trace_bdrv_co_readv(bs, sector_num, nb_sectors); 107361007b31SStefan Hajnoczi 107461007b31SStefan Hajnoczi return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 0); 107561007b31SStefan Hajnoczi } 107661007b31SStefan Hajnoczi 107761408b25SFam Zheng int coroutine_fn bdrv_co_readv_no_serialising(BlockDriverState *bs, 10789568b511SWen Congyang int64_t sector_num, int nb_sectors, QEMUIOVector *qiov) 10799568b511SWen Congyang { 108061408b25SFam Zheng trace_bdrv_co_readv_no_serialising(bs, sector_num, nb_sectors); 10819568b511SWen Congyang 10829568b511SWen Congyang return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 108361408b25SFam Zheng BDRV_REQ_NO_SERIALISING); 10849568b511SWen Congyang } 10859568b511SWen Congyang 108661007b31SStefan Hajnoczi int coroutine_fn bdrv_co_copy_on_readv(BlockDriverState *bs, 108761007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, QEMUIOVector *qiov) 108861007b31SStefan Hajnoczi { 108961007b31SStefan Hajnoczi trace_bdrv_co_copy_on_readv(bs, sector_num, nb_sectors); 109061007b31SStefan Hajnoczi 109161007b31SStefan Hajnoczi return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 109261007b31SStefan Hajnoczi BDRV_REQ_COPY_ON_READ); 109361007b31SStefan Hajnoczi } 109461007b31SStefan Hajnoczi 109561007b31SStefan Hajnoczi #define MAX_WRITE_ZEROES_BOUNCE_BUFFER 32768 109661007b31SStefan Hajnoczi 109761007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs, 109861007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, BdrvRequestFlags flags) 109961007b31SStefan Hajnoczi { 110061007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 110161007b31SStefan Hajnoczi QEMUIOVector qiov; 110261007b31SStefan Hajnoczi struct iovec iov = {0}; 110361007b31SStefan Hajnoczi int ret = 0; 110461007b31SStefan Hajnoczi 110561007b31SStefan Hajnoczi int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_write_zeroes, 110661007b31SStefan Hajnoczi BDRV_REQUEST_MAX_SECTORS); 110761007b31SStefan Hajnoczi 110861007b31SStefan Hajnoczi while (nb_sectors > 0 && !ret) { 110961007b31SStefan Hajnoczi int num = nb_sectors; 111061007b31SStefan Hajnoczi 111161007b31SStefan Hajnoczi /* Align request. Block drivers can expect the "bulk" of the request 111261007b31SStefan Hajnoczi * to be aligned. 111361007b31SStefan Hajnoczi */ 111461007b31SStefan Hajnoczi if (bs->bl.write_zeroes_alignment 111561007b31SStefan Hajnoczi && num > bs->bl.write_zeroes_alignment) { 111661007b31SStefan Hajnoczi if (sector_num % bs->bl.write_zeroes_alignment != 0) { 111761007b31SStefan Hajnoczi /* Make a small request up to the first aligned sector. */ 111861007b31SStefan Hajnoczi num = bs->bl.write_zeroes_alignment; 111961007b31SStefan Hajnoczi num -= sector_num % bs->bl.write_zeroes_alignment; 112061007b31SStefan Hajnoczi } else if ((sector_num + num) % bs->bl.write_zeroes_alignment != 0) { 112161007b31SStefan Hajnoczi /* Shorten the request to the last aligned sector. num cannot 112261007b31SStefan Hajnoczi * underflow because num > bs->bl.write_zeroes_alignment. 112361007b31SStefan Hajnoczi */ 112461007b31SStefan Hajnoczi num -= (sector_num + num) % bs->bl.write_zeroes_alignment; 112561007b31SStefan Hajnoczi } 112661007b31SStefan Hajnoczi } 112761007b31SStefan Hajnoczi 112861007b31SStefan Hajnoczi /* limit request size */ 112961007b31SStefan Hajnoczi if (num > max_write_zeroes) { 113061007b31SStefan Hajnoczi num = max_write_zeroes; 113161007b31SStefan Hajnoczi } 113261007b31SStefan Hajnoczi 113361007b31SStefan Hajnoczi ret = -ENOTSUP; 113461007b31SStefan Hajnoczi /* First try the efficient write zeroes operation */ 113561007b31SStefan Hajnoczi if (drv->bdrv_co_write_zeroes) { 113661007b31SStefan Hajnoczi ret = drv->bdrv_co_write_zeroes(bs, sector_num, num, flags); 113761007b31SStefan Hajnoczi } 113861007b31SStefan Hajnoczi 113961007b31SStefan Hajnoczi if (ret == -ENOTSUP) { 114061007b31SStefan Hajnoczi /* Fall back to bounce buffer if write zeroes is unsupported */ 114161007b31SStefan Hajnoczi int max_xfer_len = MIN_NON_ZERO(bs->bl.max_transfer_length, 114261007b31SStefan Hajnoczi MAX_WRITE_ZEROES_BOUNCE_BUFFER); 114361007b31SStefan Hajnoczi num = MIN(num, max_xfer_len); 114461007b31SStefan Hajnoczi iov.iov_len = num * BDRV_SECTOR_SIZE; 114561007b31SStefan Hajnoczi if (iov.iov_base == NULL) { 114661007b31SStefan Hajnoczi iov.iov_base = qemu_try_blockalign(bs, num * BDRV_SECTOR_SIZE); 114761007b31SStefan Hajnoczi if (iov.iov_base == NULL) { 114861007b31SStefan Hajnoczi ret = -ENOMEM; 114961007b31SStefan Hajnoczi goto fail; 115061007b31SStefan Hajnoczi } 115161007b31SStefan Hajnoczi memset(iov.iov_base, 0, num * BDRV_SECTOR_SIZE); 115261007b31SStefan Hajnoczi } 115361007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 115461007b31SStefan Hajnoczi 115561007b31SStefan Hajnoczi ret = drv->bdrv_co_writev(bs, sector_num, num, &qiov); 115661007b31SStefan Hajnoczi 115761007b31SStefan Hajnoczi /* Keep bounce buffer around if it is big enough for all 115861007b31SStefan Hajnoczi * all future requests. 115961007b31SStefan Hajnoczi */ 116061007b31SStefan Hajnoczi if (num < max_xfer_len) { 116161007b31SStefan Hajnoczi qemu_vfree(iov.iov_base); 116261007b31SStefan Hajnoczi iov.iov_base = NULL; 116361007b31SStefan Hajnoczi } 116461007b31SStefan Hajnoczi } 116561007b31SStefan Hajnoczi 116661007b31SStefan Hajnoczi sector_num += num; 116761007b31SStefan Hajnoczi nb_sectors -= num; 116861007b31SStefan Hajnoczi } 116961007b31SStefan Hajnoczi 117061007b31SStefan Hajnoczi fail: 117161007b31SStefan Hajnoczi qemu_vfree(iov.iov_base); 117261007b31SStefan Hajnoczi return ret; 117361007b31SStefan Hajnoczi } 117461007b31SStefan Hajnoczi 117561007b31SStefan Hajnoczi /* 117661007b31SStefan Hajnoczi * Forwards an already correctly aligned write request to the BlockDriver. 117761007b31SStefan Hajnoczi */ 117861007b31SStefan Hajnoczi static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs, 117961007b31SStefan Hajnoczi BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 118061007b31SStefan Hajnoczi QEMUIOVector *qiov, int flags) 118161007b31SStefan Hajnoczi { 118261007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 118361007b31SStefan Hajnoczi bool waited; 118461007b31SStefan Hajnoczi int ret; 118561007b31SStefan Hajnoczi 118661007b31SStefan Hajnoczi int64_t sector_num = offset >> BDRV_SECTOR_BITS; 118761007b31SStefan Hajnoczi unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS; 118861007b31SStefan Hajnoczi 118961007b31SStefan Hajnoczi assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 119061007b31SStefan Hajnoczi assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 119161007b31SStefan Hajnoczi assert(!qiov || bytes == qiov->size); 1192abb06c5aSDaniel P. Berrange assert((bs->open_flags & BDRV_O_NO_IO) == 0); 119361007b31SStefan Hajnoczi 119461007b31SStefan Hajnoczi waited = wait_serialising_requests(req); 119561007b31SStefan Hajnoczi assert(!waited || !req->serialising); 119661007b31SStefan Hajnoczi assert(req->overlap_offset <= offset); 119761007b31SStefan Hajnoczi assert(offset + bytes <= req->overlap_offset + req->overlap_bytes); 119861007b31SStefan Hajnoczi 119961007b31SStefan Hajnoczi ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req); 120061007b31SStefan Hajnoczi 120161007b31SStefan Hajnoczi if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF && 120261007b31SStefan Hajnoczi !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_write_zeroes && 120361007b31SStefan Hajnoczi qemu_iovec_is_zero(qiov)) { 120461007b31SStefan Hajnoczi flags |= BDRV_REQ_ZERO_WRITE; 120561007b31SStefan Hajnoczi if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) { 120661007b31SStefan Hajnoczi flags |= BDRV_REQ_MAY_UNMAP; 120761007b31SStefan Hajnoczi } 120861007b31SStefan Hajnoczi } 120961007b31SStefan Hajnoczi 121061007b31SStefan Hajnoczi if (ret < 0) { 121161007b31SStefan Hajnoczi /* Do nothing, write notifier decided to fail this request */ 121261007b31SStefan Hajnoczi } else if (flags & BDRV_REQ_ZERO_WRITE) { 12139a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO); 121461007b31SStefan Hajnoczi ret = bdrv_co_do_write_zeroes(bs, sector_num, nb_sectors, flags); 121593f5e6d8SKevin Wolf } else if (drv->bdrv_co_writev_flags) { 121693f5e6d8SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV); 121793f5e6d8SKevin Wolf ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov, 121893f5e6d8SKevin Wolf flags); 121961007b31SStefan Hajnoczi } else { 122093f5e6d8SKevin Wolf assert(drv->supported_write_flags == 0); 12219a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV); 122261007b31SStefan Hajnoczi ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov); 122361007b31SStefan Hajnoczi } 12249a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE); 122561007b31SStefan Hajnoczi 122693f5e6d8SKevin Wolf if (ret == 0 && (flags & BDRV_REQ_FUA) && 122793f5e6d8SKevin Wolf !(drv->supported_write_flags & BDRV_REQ_FUA)) 122893f5e6d8SKevin Wolf { 122961007b31SStefan Hajnoczi ret = bdrv_co_flush(bs); 123061007b31SStefan Hajnoczi } 123161007b31SStefan Hajnoczi 123261007b31SStefan Hajnoczi bdrv_set_dirty(bs, sector_num, nb_sectors); 123361007b31SStefan Hajnoczi 123453d8f9d8SMax Reitz if (bs->wr_highest_offset < offset + bytes) { 123553d8f9d8SMax Reitz bs->wr_highest_offset = offset + bytes; 123653d8f9d8SMax Reitz } 123761007b31SStefan Hajnoczi 123861007b31SStefan Hajnoczi if (ret >= 0) { 123961007b31SStefan Hajnoczi bs->total_sectors = MAX(bs->total_sectors, sector_num + nb_sectors); 124061007b31SStefan Hajnoczi } 124161007b31SStefan Hajnoczi 124261007b31SStefan Hajnoczi return ret; 124361007b31SStefan Hajnoczi } 124461007b31SStefan Hajnoczi 12459eeb6dd1SFam Zheng static int coroutine_fn bdrv_co_do_zero_pwritev(BlockDriverState *bs, 12469eeb6dd1SFam Zheng int64_t offset, 12479eeb6dd1SFam Zheng unsigned int bytes, 12489eeb6dd1SFam Zheng BdrvRequestFlags flags, 12499eeb6dd1SFam Zheng BdrvTrackedRequest *req) 12509eeb6dd1SFam Zheng { 12519eeb6dd1SFam Zheng uint8_t *buf = NULL; 12529eeb6dd1SFam Zheng QEMUIOVector local_qiov; 12539eeb6dd1SFam Zheng struct iovec iov; 12549eeb6dd1SFam Zheng uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment); 12559eeb6dd1SFam Zheng unsigned int head_padding_bytes, tail_padding_bytes; 12569eeb6dd1SFam Zheng int ret = 0; 12579eeb6dd1SFam Zheng 12589eeb6dd1SFam Zheng head_padding_bytes = offset & (align - 1); 12599eeb6dd1SFam Zheng tail_padding_bytes = align - ((offset + bytes) & (align - 1)); 12609eeb6dd1SFam Zheng 12619eeb6dd1SFam Zheng 12629eeb6dd1SFam Zheng assert(flags & BDRV_REQ_ZERO_WRITE); 12639eeb6dd1SFam Zheng if (head_padding_bytes || tail_padding_bytes) { 12649eeb6dd1SFam Zheng buf = qemu_blockalign(bs, align); 12659eeb6dd1SFam Zheng iov = (struct iovec) { 12669eeb6dd1SFam Zheng .iov_base = buf, 12679eeb6dd1SFam Zheng .iov_len = align, 12689eeb6dd1SFam Zheng }; 12699eeb6dd1SFam Zheng qemu_iovec_init_external(&local_qiov, &iov, 1); 12709eeb6dd1SFam Zheng } 12719eeb6dd1SFam Zheng if (head_padding_bytes) { 12729eeb6dd1SFam Zheng uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes); 12739eeb6dd1SFam Zheng 12749eeb6dd1SFam Zheng /* RMW the unaligned part before head. */ 12759eeb6dd1SFam Zheng mark_request_serialising(req, align); 12769eeb6dd1SFam Zheng wait_serialising_requests(req); 12779a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 12789eeb6dd1SFam Zheng ret = bdrv_aligned_preadv(bs, req, offset & ~(align - 1), align, 12799eeb6dd1SFam Zheng align, &local_qiov, 0); 12809eeb6dd1SFam Zheng if (ret < 0) { 12819eeb6dd1SFam Zheng goto fail; 12829eeb6dd1SFam Zheng } 12839a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 12849eeb6dd1SFam Zheng 12859eeb6dd1SFam Zheng memset(buf + head_padding_bytes, 0, zero_bytes); 12869eeb6dd1SFam Zheng ret = bdrv_aligned_pwritev(bs, req, offset & ~(align - 1), align, 12879eeb6dd1SFam Zheng &local_qiov, 12889eeb6dd1SFam Zheng flags & ~BDRV_REQ_ZERO_WRITE); 12899eeb6dd1SFam Zheng if (ret < 0) { 12909eeb6dd1SFam Zheng goto fail; 12919eeb6dd1SFam Zheng } 12929eeb6dd1SFam Zheng offset += zero_bytes; 12939eeb6dd1SFam Zheng bytes -= zero_bytes; 12949eeb6dd1SFam Zheng } 12959eeb6dd1SFam Zheng 12969eeb6dd1SFam Zheng assert(!bytes || (offset & (align - 1)) == 0); 12979eeb6dd1SFam Zheng if (bytes >= align) { 12989eeb6dd1SFam Zheng /* Write the aligned part in the middle. */ 12999eeb6dd1SFam Zheng uint64_t aligned_bytes = bytes & ~(align - 1); 13009eeb6dd1SFam Zheng ret = bdrv_aligned_pwritev(bs, req, offset, aligned_bytes, 13019eeb6dd1SFam Zheng NULL, flags); 13029eeb6dd1SFam Zheng if (ret < 0) { 13039eeb6dd1SFam Zheng goto fail; 13049eeb6dd1SFam Zheng } 13059eeb6dd1SFam Zheng bytes -= aligned_bytes; 13069eeb6dd1SFam Zheng offset += aligned_bytes; 13079eeb6dd1SFam Zheng } 13089eeb6dd1SFam Zheng 13099eeb6dd1SFam Zheng assert(!bytes || (offset & (align - 1)) == 0); 13109eeb6dd1SFam Zheng if (bytes) { 13119eeb6dd1SFam Zheng assert(align == tail_padding_bytes + bytes); 13129eeb6dd1SFam Zheng /* RMW the unaligned part after tail. */ 13139eeb6dd1SFam Zheng mark_request_serialising(req, align); 13149eeb6dd1SFam Zheng wait_serialising_requests(req); 13159a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 13169eeb6dd1SFam Zheng ret = bdrv_aligned_preadv(bs, req, offset, align, 13179eeb6dd1SFam Zheng align, &local_qiov, 0); 13189eeb6dd1SFam Zheng if (ret < 0) { 13199eeb6dd1SFam Zheng goto fail; 13209eeb6dd1SFam Zheng } 13219a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 13229eeb6dd1SFam Zheng 13239eeb6dd1SFam Zheng memset(buf, 0, bytes); 13249eeb6dd1SFam Zheng ret = bdrv_aligned_pwritev(bs, req, offset, align, 13259eeb6dd1SFam Zheng &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE); 13269eeb6dd1SFam Zheng } 13279eeb6dd1SFam Zheng fail: 13289eeb6dd1SFam Zheng qemu_vfree(buf); 13299eeb6dd1SFam Zheng return ret; 13309eeb6dd1SFam Zheng 13319eeb6dd1SFam Zheng } 13329eeb6dd1SFam Zheng 133361007b31SStefan Hajnoczi /* 133461007b31SStefan Hajnoczi * Handle a write request in coroutine context 133561007b31SStefan Hajnoczi */ 1336a8823a3bSKevin Wolf int coroutine_fn bdrv_co_do_pwritev(BlockDriverState *bs, 133761007b31SStefan Hajnoczi int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 133861007b31SStefan Hajnoczi BdrvRequestFlags flags) 133961007b31SStefan Hajnoczi { 134061007b31SStefan Hajnoczi BdrvTrackedRequest req; 1341d01c07f2SFam Zheng /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */ 1342d01c07f2SFam Zheng uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment); 134361007b31SStefan Hajnoczi uint8_t *head_buf = NULL; 134461007b31SStefan Hajnoczi uint8_t *tail_buf = NULL; 134561007b31SStefan Hajnoczi QEMUIOVector local_qiov; 134661007b31SStefan Hajnoczi bool use_local_qiov = false; 134761007b31SStefan Hajnoczi int ret; 134861007b31SStefan Hajnoczi 134961007b31SStefan Hajnoczi if (!bs->drv) { 135061007b31SStefan Hajnoczi return -ENOMEDIUM; 135161007b31SStefan Hajnoczi } 135261007b31SStefan Hajnoczi if (bs->read_only) { 1353eaf5fe2dSPaolo Bonzini return -EPERM; 135461007b31SStefan Hajnoczi } 135504c01a5cSKevin Wolf assert(!(bs->open_flags & BDRV_O_INACTIVE)); 135661007b31SStefan Hajnoczi 135761007b31SStefan Hajnoczi ret = bdrv_check_byte_request(bs, offset, bytes); 135861007b31SStefan Hajnoczi if (ret < 0) { 135961007b31SStefan Hajnoczi return ret; 136061007b31SStefan Hajnoczi } 136161007b31SStefan Hajnoczi 136261007b31SStefan Hajnoczi /* throttling disk I/O */ 1363ce0f1412SPaolo Bonzini if (bs->throttle_state) { 136476f4afb4SAlberto Garcia throttle_group_co_io_limits_intercept(bs, bytes, true); 136561007b31SStefan Hajnoczi } 136661007b31SStefan Hajnoczi 136761007b31SStefan Hajnoczi /* 136861007b31SStefan Hajnoczi * Align write if necessary by performing a read-modify-write cycle. 136961007b31SStefan Hajnoczi * Pad qiov with the read parts and be sure to have a tracked request not 137061007b31SStefan Hajnoczi * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle. 137161007b31SStefan Hajnoczi */ 1372ebde595cSFam Zheng tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE); 137361007b31SStefan Hajnoczi 13749eeb6dd1SFam Zheng if (!qiov) { 13759eeb6dd1SFam Zheng ret = bdrv_co_do_zero_pwritev(bs, offset, bytes, flags, &req); 13769eeb6dd1SFam Zheng goto out; 13779eeb6dd1SFam Zheng } 13789eeb6dd1SFam Zheng 137961007b31SStefan Hajnoczi if (offset & (align - 1)) { 138061007b31SStefan Hajnoczi QEMUIOVector head_qiov; 138161007b31SStefan Hajnoczi struct iovec head_iov; 138261007b31SStefan Hajnoczi 138361007b31SStefan Hajnoczi mark_request_serialising(&req, align); 138461007b31SStefan Hajnoczi wait_serialising_requests(&req); 138561007b31SStefan Hajnoczi 138661007b31SStefan Hajnoczi head_buf = qemu_blockalign(bs, align); 138761007b31SStefan Hajnoczi head_iov = (struct iovec) { 138861007b31SStefan Hajnoczi .iov_base = head_buf, 138961007b31SStefan Hajnoczi .iov_len = align, 139061007b31SStefan Hajnoczi }; 139161007b31SStefan Hajnoczi qemu_iovec_init_external(&head_qiov, &head_iov, 1); 139261007b31SStefan Hajnoczi 13939a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 139461007b31SStefan Hajnoczi ret = bdrv_aligned_preadv(bs, &req, offset & ~(align - 1), align, 139561007b31SStefan Hajnoczi align, &head_qiov, 0); 139661007b31SStefan Hajnoczi if (ret < 0) { 139761007b31SStefan Hajnoczi goto fail; 139861007b31SStefan Hajnoczi } 13999a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 140061007b31SStefan Hajnoczi 140161007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 2); 140261007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 140361007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 140461007b31SStefan Hajnoczi use_local_qiov = true; 140561007b31SStefan Hajnoczi 140661007b31SStefan Hajnoczi bytes += offset & (align - 1); 140761007b31SStefan Hajnoczi offset = offset & ~(align - 1); 140861007b31SStefan Hajnoczi } 140961007b31SStefan Hajnoczi 141061007b31SStefan Hajnoczi if ((offset + bytes) & (align - 1)) { 141161007b31SStefan Hajnoczi QEMUIOVector tail_qiov; 141261007b31SStefan Hajnoczi struct iovec tail_iov; 141361007b31SStefan Hajnoczi size_t tail_bytes; 141461007b31SStefan Hajnoczi bool waited; 141561007b31SStefan Hajnoczi 141661007b31SStefan Hajnoczi mark_request_serialising(&req, align); 141761007b31SStefan Hajnoczi waited = wait_serialising_requests(&req); 141861007b31SStefan Hajnoczi assert(!waited || !use_local_qiov); 141961007b31SStefan Hajnoczi 142061007b31SStefan Hajnoczi tail_buf = qemu_blockalign(bs, align); 142161007b31SStefan Hajnoczi tail_iov = (struct iovec) { 142261007b31SStefan Hajnoczi .iov_base = tail_buf, 142361007b31SStefan Hajnoczi .iov_len = align, 142461007b31SStefan Hajnoczi }; 142561007b31SStefan Hajnoczi qemu_iovec_init_external(&tail_qiov, &tail_iov, 1); 142661007b31SStefan Hajnoczi 14279a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 142861007b31SStefan Hajnoczi ret = bdrv_aligned_preadv(bs, &req, (offset + bytes) & ~(align - 1), align, 142961007b31SStefan Hajnoczi align, &tail_qiov, 0); 143061007b31SStefan Hajnoczi if (ret < 0) { 143161007b31SStefan Hajnoczi goto fail; 143261007b31SStefan Hajnoczi } 14339a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 143461007b31SStefan Hajnoczi 143561007b31SStefan Hajnoczi if (!use_local_qiov) { 143661007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 1); 143761007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 143861007b31SStefan Hajnoczi use_local_qiov = true; 143961007b31SStefan Hajnoczi } 144061007b31SStefan Hajnoczi 144161007b31SStefan Hajnoczi tail_bytes = (offset + bytes) & (align - 1); 144261007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes); 144361007b31SStefan Hajnoczi 144461007b31SStefan Hajnoczi bytes = ROUND_UP(bytes, align); 144561007b31SStefan Hajnoczi } 144661007b31SStefan Hajnoczi 144761007b31SStefan Hajnoczi ret = bdrv_aligned_pwritev(bs, &req, offset, bytes, 144861007b31SStefan Hajnoczi use_local_qiov ? &local_qiov : qiov, 144961007b31SStefan Hajnoczi flags); 145061007b31SStefan Hajnoczi 145161007b31SStefan Hajnoczi fail: 145261007b31SStefan Hajnoczi 145361007b31SStefan Hajnoczi if (use_local_qiov) { 145461007b31SStefan Hajnoczi qemu_iovec_destroy(&local_qiov); 145561007b31SStefan Hajnoczi } 145661007b31SStefan Hajnoczi qemu_vfree(head_buf); 145761007b31SStefan Hajnoczi qemu_vfree(tail_buf); 14589eeb6dd1SFam Zheng out: 14599eeb6dd1SFam Zheng tracked_request_end(&req); 146061007b31SStefan Hajnoczi return ret; 146161007b31SStefan Hajnoczi } 146261007b31SStefan Hajnoczi 146361007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_do_writev(BlockDriverState *bs, 146461007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, 146561007b31SStefan Hajnoczi BdrvRequestFlags flags) 146661007b31SStefan Hajnoczi { 146761007b31SStefan Hajnoczi if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 146861007b31SStefan Hajnoczi return -EINVAL; 146961007b31SStefan Hajnoczi } 147061007b31SStefan Hajnoczi 147161007b31SStefan Hajnoczi return bdrv_co_do_pwritev(bs, sector_num << BDRV_SECTOR_BITS, 147261007b31SStefan Hajnoczi nb_sectors << BDRV_SECTOR_BITS, qiov, flags); 147361007b31SStefan Hajnoczi } 147461007b31SStefan Hajnoczi 147561007b31SStefan Hajnoczi int coroutine_fn bdrv_co_writev(BlockDriverState *bs, int64_t sector_num, 147661007b31SStefan Hajnoczi int nb_sectors, QEMUIOVector *qiov) 147761007b31SStefan Hajnoczi { 147861007b31SStefan Hajnoczi trace_bdrv_co_writev(bs, sector_num, nb_sectors); 147961007b31SStefan Hajnoczi 148061007b31SStefan Hajnoczi return bdrv_co_do_writev(bs, sector_num, nb_sectors, qiov, 0); 148161007b31SStefan Hajnoczi } 148261007b31SStefan Hajnoczi 148361007b31SStefan Hajnoczi int coroutine_fn bdrv_co_write_zeroes(BlockDriverState *bs, 148461007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, 148561007b31SStefan Hajnoczi BdrvRequestFlags flags) 148661007b31SStefan Hajnoczi { 148761007b31SStefan Hajnoczi trace_bdrv_co_write_zeroes(bs, sector_num, nb_sectors, flags); 148861007b31SStefan Hajnoczi 148961007b31SStefan Hajnoczi if (!(bs->open_flags & BDRV_O_UNMAP)) { 149061007b31SStefan Hajnoczi flags &= ~BDRV_REQ_MAY_UNMAP; 149161007b31SStefan Hajnoczi } 149261007b31SStefan Hajnoczi 1493d01c07f2SFam Zheng return bdrv_co_do_writev(bs, sector_num, nb_sectors, NULL, 149461007b31SStefan Hajnoczi BDRV_REQ_ZERO_WRITE | flags); 149561007b31SStefan Hajnoczi } 149661007b31SStefan Hajnoczi 149761007b31SStefan Hajnoczi typedef struct BdrvCoGetBlockStatusData { 149861007b31SStefan Hajnoczi BlockDriverState *bs; 149961007b31SStefan Hajnoczi BlockDriverState *base; 150067a0fd2aSFam Zheng BlockDriverState **file; 150161007b31SStefan Hajnoczi int64_t sector_num; 150261007b31SStefan Hajnoczi int nb_sectors; 150361007b31SStefan Hajnoczi int *pnum; 150461007b31SStefan Hajnoczi int64_t ret; 150561007b31SStefan Hajnoczi bool done; 150661007b31SStefan Hajnoczi } BdrvCoGetBlockStatusData; 150761007b31SStefan Hajnoczi 150861007b31SStefan Hajnoczi /* 150961007b31SStefan Hajnoczi * Returns the allocation status of the specified sectors. 151061007b31SStefan Hajnoczi * Drivers not implementing the functionality are assumed to not support 151161007b31SStefan Hajnoczi * backing files, hence all their sectors are reported as allocated. 151261007b31SStefan Hajnoczi * 151361007b31SStefan Hajnoczi * If 'sector_num' is beyond the end of the disk image the return value is 0 151461007b31SStefan Hajnoczi * and 'pnum' is set to 0. 151561007b31SStefan Hajnoczi * 151661007b31SStefan Hajnoczi * 'pnum' is set to the number of sectors (including and immediately following 151761007b31SStefan Hajnoczi * the specified sector) that are known to be in the same 151861007b31SStefan Hajnoczi * allocated/unallocated state. 151961007b31SStefan Hajnoczi * 152061007b31SStefan Hajnoczi * 'nb_sectors' is the max value 'pnum' should be set to. If nb_sectors goes 152161007b31SStefan Hajnoczi * beyond the end of the disk image it will be clamped. 152267a0fd2aSFam Zheng * 152367a0fd2aSFam Zheng * If returned value is positive and BDRV_BLOCK_OFFSET_VALID bit is set, 'file' 152467a0fd2aSFam Zheng * points to the BDS which the sector range is allocated in. 152561007b31SStefan Hajnoczi */ 152661007b31SStefan Hajnoczi static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs, 152761007b31SStefan Hajnoczi int64_t sector_num, 152867a0fd2aSFam Zheng int nb_sectors, int *pnum, 152967a0fd2aSFam Zheng BlockDriverState **file) 153061007b31SStefan Hajnoczi { 153161007b31SStefan Hajnoczi int64_t total_sectors; 153261007b31SStefan Hajnoczi int64_t n; 153361007b31SStefan Hajnoczi int64_t ret, ret2; 153461007b31SStefan Hajnoczi 153561007b31SStefan Hajnoczi total_sectors = bdrv_nb_sectors(bs); 153661007b31SStefan Hajnoczi if (total_sectors < 0) { 153761007b31SStefan Hajnoczi return total_sectors; 153861007b31SStefan Hajnoczi } 153961007b31SStefan Hajnoczi 154061007b31SStefan Hajnoczi if (sector_num >= total_sectors) { 154161007b31SStefan Hajnoczi *pnum = 0; 154261007b31SStefan Hajnoczi return 0; 154361007b31SStefan Hajnoczi } 154461007b31SStefan Hajnoczi 154561007b31SStefan Hajnoczi n = total_sectors - sector_num; 154661007b31SStefan Hajnoczi if (n < nb_sectors) { 154761007b31SStefan Hajnoczi nb_sectors = n; 154861007b31SStefan Hajnoczi } 154961007b31SStefan Hajnoczi 155061007b31SStefan Hajnoczi if (!bs->drv->bdrv_co_get_block_status) { 155161007b31SStefan Hajnoczi *pnum = nb_sectors; 155261007b31SStefan Hajnoczi ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED; 155361007b31SStefan Hajnoczi if (bs->drv->protocol_name) { 155461007b31SStefan Hajnoczi ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE); 155561007b31SStefan Hajnoczi } 155661007b31SStefan Hajnoczi return ret; 155761007b31SStefan Hajnoczi } 155861007b31SStefan Hajnoczi 155967a0fd2aSFam Zheng *file = NULL; 156067a0fd2aSFam Zheng ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum, 156167a0fd2aSFam Zheng file); 156261007b31SStefan Hajnoczi if (ret < 0) { 156361007b31SStefan Hajnoczi *pnum = 0; 156461007b31SStefan Hajnoczi return ret; 156561007b31SStefan Hajnoczi } 156661007b31SStefan Hajnoczi 156761007b31SStefan Hajnoczi if (ret & BDRV_BLOCK_RAW) { 156861007b31SStefan Hajnoczi assert(ret & BDRV_BLOCK_OFFSET_VALID); 15699a4f4c31SKevin Wolf return bdrv_get_block_status(bs->file->bs, ret >> BDRV_SECTOR_BITS, 157067a0fd2aSFam Zheng *pnum, pnum, file); 157161007b31SStefan Hajnoczi } 157261007b31SStefan Hajnoczi 157361007b31SStefan Hajnoczi if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) { 157461007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ALLOCATED; 1575a53f1a95SPaolo Bonzini } else { 157661007b31SStefan Hajnoczi if (bdrv_unallocated_blocks_are_zero(bs)) { 157761007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 1578760e0063SKevin Wolf } else if (bs->backing) { 1579760e0063SKevin Wolf BlockDriverState *bs2 = bs->backing->bs; 158061007b31SStefan Hajnoczi int64_t nb_sectors2 = bdrv_nb_sectors(bs2); 158161007b31SStefan Hajnoczi if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) { 158261007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 158361007b31SStefan Hajnoczi } 158461007b31SStefan Hajnoczi } 158561007b31SStefan Hajnoczi } 158661007b31SStefan Hajnoczi 1587ac987b30SFam Zheng if (*file && *file != bs && 158861007b31SStefan Hajnoczi (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) && 158961007b31SStefan Hajnoczi (ret & BDRV_BLOCK_OFFSET_VALID)) { 159067a0fd2aSFam Zheng BlockDriverState *file2; 159161007b31SStefan Hajnoczi int file_pnum; 159261007b31SStefan Hajnoczi 1593ac987b30SFam Zheng ret2 = bdrv_co_get_block_status(*file, ret >> BDRV_SECTOR_BITS, 159467a0fd2aSFam Zheng *pnum, &file_pnum, &file2); 159561007b31SStefan Hajnoczi if (ret2 >= 0) { 159661007b31SStefan Hajnoczi /* Ignore errors. This is just providing extra information, it 159761007b31SStefan Hajnoczi * is useful but not necessary. 159861007b31SStefan Hajnoczi */ 159961007b31SStefan Hajnoczi if (!file_pnum) { 160061007b31SStefan Hajnoczi /* !file_pnum indicates an offset at or beyond the EOF; it is 160161007b31SStefan Hajnoczi * perfectly valid for the format block driver to point to such 160261007b31SStefan Hajnoczi * offsets, so catch it and mark everything as zero */ 160361007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 160461007b31SStefan Hajnoczi } else { 160561007b31SStefan Hajnoczi /* Limit request to the range reported by the protocol driver */ 160661007b31SStefan Hajnoczi *pnum = file_pnum; 160761007b31SStefan Hajnoczi ret |= (ret2 & BDRV_BLOCK_ZERO); 160861007b31SStefan Hajnoczi } 160961007b31SStefan Hajnoczi } 161061007b31SStefan Hajnoczi } 161161007b31SStefan Hajnoczi 161261007b31SStefan Hajnoczi return ret; 161361007b31SStefan Hajnoczi } 161461007b31SStefan Hajnoczi 1615ba3f0e25SFam Zheng static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs, 1616ba3f0e25SFam Zheng BlockDriverState *base, 1617ba3f0e25SFam Zheng int64_t sector_num, 1618ba3f0e25SFam Zheng int nb_sectors, 161967a0fd2aSFam Zheng int *pnum, 162067a0fd2aSFam Zheng BlockDriverState **file) 1621ba3f0e25SFam Zheng { 1622ba3f0e25SFam Zheng BlockDriverState *p; 1623ba3f0e25SFam Zheng int64_t ret = 0; 1624ba3f0e25SFam Zheng 1625ba3f0e25SFam Zheng assert(bs != base); 1626760e0063SKevin Wolf for (p = bs; p != base; p = backing_bs(p)) { 162767a0fd2aSFam Zheng ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum, file); 1628ba3f0e25SFam Zheng if (ret < 0 || ret & BDRV_BLOCK_ALLOCATED) { 1629ba3f0e25SFam Zheng break; 1630ba3f0e25SFam Zheng } 1631ba3f0e25SFam Zheng /* [sector_num, pnum] unallocated on this layer, which could be only 1632ba3f0e25SFam Zheng * the first part of [sector_num, nb_sectors]. */ 1633ba3f0e25SFam Zheng nb_sectors = MIN(nb_sectors, *pnum); 1634ba3f0e25SFam Zheng } 1635ba3f0e25SFam Zheng return ret; 1636ba3f0e25SFam Zheng } 1637ba3f0e25SFam Zheng 1638ba3f0e25SFam Zheng /* Coroutine wrapper for bdrv_get_block_status_above() */ 1639ba3f0e25SFam Zheng static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque) 164061007b31SStefan Hajnoczi { 164161007b31SStefan Hajnoczi BdrvCoGetBlockStatusData *data = opaque; 164261007b31SStefan Hajnoczi 1643ba3f0e25SFam Zheng data->ret = bdrv_co_get_block_status_above(data->bs, data->base, 1644ba3f0e25SFam Zheng data->sector_num, 1645ba3f0e25SFam Zheng data->nb_sectors, 164667a0fd2aSFam Zheng data->pnum, 164767a0fd2aSFam Zheng data->file); 164861007b31SStefan Hajnoczi data->done = true; 164961007b31SStefan Hajnoczi } 165061007b31SStefan Hajnoczi 165161007b31SStefan Hajnoczi /* 1652ba3f0e25SFam Zheng * Synchronous wrapper around bdrv_co_get_block_status_above(). 165361007b31SStefan Hajnoczi * 1654ba3f0e25SFam Zheng * See bdrv_co_get_block_status_above() for details. 165561007b31SStefan Hajnoczi */ 1656ba3f0e25SFam Zheng int64_t bdrv_get_block_status_above(BlockDriverState *bs, 1657ba3f0e25SFam Zheng BlockDriverState *base, 1658ba3f0e25SFam Zheng int64_t sector_num, 165967a0fd2aSFam Zheng int nb_sectors, int *pnum, 166067a0fd2aSFam Zheng BlockDriverState **file) 166161007b31SStefan Hajnoczi { 166261007b31SStefan Hajnoczi Coroutine *co; 166361007b31SStefan Hajnoczi BdrvCoGetBlockStatusData data = { 166461007b31SStefan Hajnoczi .bs = bs, 1665ba3f0e25SFam Zheng .base = base, 166667a0fd2aSFam Zheng .file = file, 166761007b31SStefan Hajnoczi .sector_num = sector_num, 166861007b31SStefan Hajnoczi .nb_sectors = nb_sectors, 166961007b31SStefan Hajnoczi .pnum = pnum, 167061007b31SStefan Hajnoczi .done = false, 167161007b31SStefan Hajnoczi }; 167261007b31SStefan Hajnoczi 167361007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 167461007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 1675ba3f0e25SFam Zheng bdrv_get_block_status_above_co_entry(&data); 167661007b31SStefan Hajnoczi } else { 167761007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 167861007b31SStefan Hajnoczi 1679ba3f0e25SFam Zheng co = qemu_coroutine_create(bdrv_get_block_status_above_co_entry); 168061007b31SStefan Hajnoczi qemu_coroutine_enter(co, &data); 168161007b31SStefan Hajnoczi while (!data.done) { 168261007b31SStefan Hajnoczi aio_poll(aio_context, true); 168361007b31SStefan Hajnoczi } 168461007b31SStefan Hajnoczi } 168561007b31SStefan Hajnoczi return data.ret; 168661007b31SStefan Hajnoczi } 168761007b31SStefan Hajnoczi 1688ba3f0e25SFam Zheng int64_t bdrv_get_block_status(BlockDriverState *bs, 1689ba3f0e25SFam Zheng int64_t sector_num, 169067a0fd2aSFam Zheng int nb_sectors, int *pnum, 169167a0fd2aSFam Zheng BlockDriverState **file) 1692ba3f0e25SFam Zheng { 1693760e0063SKevin Wolf return bdrv_get_block_status_above(bs, backing_bs(bs), 169467a0fd2aSFam Zheng sector_num, nb_sectors, pnum, file); 1695ba3f0e25SFam Zheng } 1696ba3f0e25SFam Zheng 169761007b31SStefan Hajnoczi int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num, 169861007b31SStefan Hajnoczi int nb_sectors, int *pnum) 169961007b31SStefan Hajnoczi { 170067a0fd2aSFam Zheng BlockDriverState *file; 170167a0fd2aSFam Zheng int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum, 170267a0fd2aSFam Zheng &file); 170361007b31SStefan Hajnoczi if (ret < 0) { 170461007b31SStefan Hajnoczi return ret; 170561007b31SStefan Hajnoczi } 170661007b31SStefan Hajnoczi return !!(ret & BDRV_BLOCK_ALLOCATED); 170761007b31SStefan Hajnoczi } 170861007b31SStefan Hajnoczi 170961007b31SStefan Hajnoczi /* 171061007b31SStefan Hajnoczi * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP] 171161007b31SStefan Hajnoczi * 171261007b31SStefan Hajnoczi * Return true if the given sector is allocated in any image between 171361007b31SStefan Hajnoczi * BASE and TOP (inclusive). BASE can be NULL to check if the given 171461007b31SStefan Hajnoczi * sector is allocated in any image of the chain. Return false otherwise. 171561007b31SStefan Hajnoczi * 171661007b31SStefan Hajnoczi * 'pnum' is set to the number of sectors (including and immediately following 171761007b31SStefan Hajnoczi * the specified sector) that are known to be in the same 171861007b31SStefan Hajnoczi * allocated/unallocated state. 171961007b31SStefan Hajnoczi * 172061007b31SStefan Hajnoczi */ 172161007b31SStefan Hajnoczi int bdrv_is_allocated_above(BlockDriverState *top, 172261007b31SStefan Hajnoczi BlockDriverState *base, 172361007b31SStefan Hajnoczi int64_t sector_num, 172461007b31SStefan Hajnoczi int nb_sectors, int *pnum) 172561007b31SStefan Hajnoczi { 172661007b31SStefan Hajnoczi BlockDriverState *intermediate; 172761007b31SStefan Hajnoczi int ret, n = nb_sectors; 172861007b31SStefan Hajnoczi 172961007b31SStefan Hajnoczi intermediate = top; 173061007b31SStefan Hajnoczi while (intermediate && intermediate != base) { 173161007b31SStefan Hajnoczi int pnum_inter; 173261007b31SStefan Hajnoczi ret = bdrv_is_allocated(intermediate, sector_num, nb_sectors, 173361007b31SStefan Hajnoczi &pnum_inter); 173461007b31SStefan Hajnoczi if (ret < 0) { 173561007b31SStefan Hajnoczi return ret; 173661007b31SStefan Hajnoczi } else if (ret) { 173761007b31SStefan Hajnoczi *pnum = pnum_inter; 173861007b31SStefan Hajnoczi return 1; 173961007b31SStefan Hajnoczi } 174061007b31SStefan Hajnoczi 174161007b31SStefan Hajnoczi /* 174261007b31SStefan Hajnoczi * [sector_num, nb_sectors] is unallocated on top but intermediate 174361007b31SStefan Hajnoczi * might have 174461007b31SStefan Hajnoczi * 174561007b31SStefan Hajnoczi * [sector_num+x, nr_sectors] allocated. 174661007b31SStefan Hajnoczi */ 174761007b31SStefan Hajnoczi if (n > pnum_inter && 174861007b31SStefan Hajnoczi (intermediate == top || 174961007b31SStefan Hajnoczi sector_num + pnum_inter < intermediate->total_sectors)) { 175061007b31SStefan Hajnoczi n = pnum_inter; 175161007b31SStefan Hajnoczi } 175261007b31SStefan Hajnoczi 1753760e0063SKevin Wolf intermediate = backing_bs(intermediate); 175461007b31SStefan Hajnoczi } 175561007b31SStefan Hajnoczi 175661007b31SStefan Hajnoczi *pnum = n; 175761007b31SStefan Hajnoczi return 0; 175861007b31SStefan Hajnoczi } 175961007b31SStefan Hajnoczi 176061007b31SStefan Hajnoczi int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num, 176161007b31SStefan Hajnoczi const uint8_t *buf, int nb_sectors) 176261007b31SStefan Hajnoczi { 176361007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 176461007b31SStefan Hajnoczi int ret; 176561007b31SStefan Hajnoczi 176661007b31SStefan Hajnoczi if (!drv) { 176761007b31SStefan Hajnoczi return -ENOMEDIUM; 176861007b31SStefan Hajnoczi } 176961007b31SStefan Hajnoczi if (!drv->bdrv_write_compressed) { 177061007b31SStefan Hajnoczi return -ENOTSUP; 177161007b31SStefan Hajnoczi } 177261007b31SStefan Hajnoczi ret = bdrv_check_request(bs, sector_num, nb_sectors); 177361007b31SStefan Hajnoczi if (ret < 0) { 177461007b31SStefan Hajnoczi return ret; 177561007b31SStefan Hajnoczi } 177661007b31SStefan Hajnoczi 177761007b31SStefan Hajnoczi assert(QLIST_EMPTY(&bs->dirty_bitmaps)); 177861007b31SStefan Hajnoczi 177961007b31SStefan Hajnoczi return drv->bdrv_write_compressed(bs, sector_num, buf, nb_sectors); 178061007b31SStefan Hajnoczi } 178161007b31SStefan Hajnoczi 178261007b31SStefan Hajnoczi int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf, 178361007b31SStefan Hajnoczi int64_t pos, int size) 178461007b31SStefan Hajnoczi { 178561007b31SStefan Hajnoczi QEMUIOVector qiov; 178661007b31SStefan Hajnoczi struct iovec iov = { 178761007b31SStefan Hajnoczi .iov_base = (void *) buf, 178861007b31SStefan Hajnoczi .iov_len = size, 178961007b31SStefan Hajnoczi }; 179061007b31SStefan Hajnoczi 179161007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 179261007b31SStefan Hajnoczi return bdrv_writev_vmstate(bs, &qiov, pos); 179361007b31SStefan Hajnoczi } 179461007b31SStefan Hajnoczi 179561007b31SStefan Hajnoczi int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 179661007b31SStefan Hajnoczi { 179761007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 179861007b31SStefan Hajnoczi 179961007b31SStefan Hajnoczi if (!drv) { 180061007b31SStefan Hajnoczi return -ENOMEDIUM; 180161007b31SStefan Hajnoczi } else if (drv->bdrv_save_vmstate) { 180261007b31SStefan Hajnoczi return drv->bdrv_save_vmstate(bs, qiov, pos); 180361007b31SStefan Hajnoczi } else if (bs->file) { 18049a4f4c31SKevin Wolf return bdrv_writev_vmstate(bs->file->bs, qiov, pos); 180561007b31SStefan Hajnoczi } 180661007b31SStefan Hajnoczi 180761007b31SStefan Hajnoczi return -ENOTSUP; 180861007b31SStefan Hajnoczi } 180961007b31SStefan Hajnoczi 181061007b31SStefan Hajnoczi int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf, 181161007b31SStefan Hajnoczi int64_t pos, int size) 181261007b31SStefan Hajnoczi { 181361007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 181461007b31SStefan Hajnoczi if (!drv) 181561007b31SStefan Hajnoczi return -ENOMEDIUM; 181661007b31SStefan Hajnoczi if (drv->bdrv_load_vmstate) 181761007b31SStefan Hajnoczi return drv->bdrv_load_vmstate(bs, buf, pos, size); 181861007b31SStefan Hajnoczi if (bs->file) 18199a4f4c31SKevin Wolf return bdrv_load_vmstate(bs->file->bs, buf, pos, size); 182061007b31SStefan Hajnoczi return -ENOTSUP; 182161007b31SStefan Hajnoczi } 182261007b31SStefan Hajnoczi 182361007b31SStefan Hajnoczi /**************************************************************/ 182461007b31SStefan Hajnoczi /* async I/Os */ 182561007b31SStefan Hajnoczi 182661007b31SStefan Hajnoczi BlockAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num, 182761007b31SStefan Hajnoczi QEMUIOVector *qiov, int nb_sectors, 182861007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque) 182961007b31SStefan Hajnoczi { 183061007b31SStefan Hajnoczi trace_bdrv_aio_readv(bs, sector_num, nb_sectors, opaque); 183161007b31SStefan Hajnoczi 183261007b31SStefan Hajnoczi return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0, 183361007b31SStefan Hajnoczi cb, opaque, false); 183461007b31SStefan Hajnoczi } 183561007b31SStefan Hajnoczi 183661007b31SStefan Hajnoczi BlockAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num, 183761007b31SStefan Hajnoczi QEMUIOVector *qiov, int nb_sectors, 183861007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque) 183961007b31SStefan Hajnoczi { 184061007b31SStefan Hajnoczi trace_bdrv_aio_writev(bs, sector_num, nb_sectors, opaque); 184161007b31SStefan Hajnoczi 184261007b31SStefan Hajnoczi return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0, 184361007b31SStefan Hajnoczi cb, opaque, true); 184461007b31SStefan Hajnoczi } 184561007b31SStefan Hajnoczi 184661007b31SStefan Hajnoczi BlockAIOCB *bdrv_aio_write_zeroes(BlockDriverState *bs, 184761007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, BdrvRequestFlags flags, 184861007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque) 184961007b31SStefan Hajnoczi { 185061007b31SStefan Hajnoczi trace_bdrv_aio_write_zeroes(bs, sector_num, nb_sectors, flags, opaque); 185161007b31SStefan Hajnoczi 185261007b31SStefan Hajnoczi return bdrv_co_aio_rw_vector(bs, sector_num, NULL, nb_sectors, 185361007b31SStefan Hajnoczi BDRV_REQ_ZERO_WRITE | flags, 185461007b31SStefan Hajnoczi cb, opaque, true); 185561007b31SStefan Hajnoczi } 185661007b31SStefan Hajnoczi 185761007b31SStefan Hajnoczi 185861007b31SStefan Hajnoczi typedef struct MultiwriteCB { 185961007b31SStefan Hajnoczi int error; 186061007b31SStefan Hajnoczi int num_requests; 186161007b31SStefan Hajnoczi int num_callbacks; 186261007b31SStefan Hajnoczi struct { 186361007b31SStefan Hajnoczi BlockCompletionFunc *cb; 186461007b31SStefan Hajnoczi void *opaque; 186561007b31SStefan Hajnoczi QEMUIOVector *free_qiov; 186661007b31SStefan Hajnoczi } callbacks[]; 186761007b31SStefan Hajnoczi } MultiwriteCB; 186861007b31SStefan Hajnoczi 186961007b31SStefan Hajnoczi static void multiwrite_user_cb(MultiwriteCB *mcb) 187061007b31SStefan Hajnoczi { 187161007b31SStefan Hajnoczi int i; 187261007b31SStefan Hajnoczi 187361007b31SStefan Hajnoczi for (i = 0; i < mcb->num_callbacks; i++) { 187461007b31SStefan Hajnoczi mcb->callbacks[i].cb(mcb->callbacks[i].opaque, mcb->error); 187561007b31SStefan Hajnoczi if (mcb->callbacks[i].free_qiov) { 187661007b31SStefan Hajnoczi qemu_iovec_destroy(mcb->callbacks[i].free_qiov); 187761007b31SStefan Hajnoczi } 187861007b31SStefan Hajnoczi g_free(mcb->callbacks[i].free_qiov); 187961007b31SStefan Hajnoczi } 188061007b31SStefan Hajnoczi } 188161007b31SStefan Hajnoczi 188261007b31SStefan Hajnoczi static void multiwrite_cb(void *opaque, int ret) 188361007b31SStefan Hajnoczi { 188461007b31SStefan Hajnoczi MultiwriteCB *mcb = opaque; 188561007b31SStefan Hajnoczi 188661007b31SStefan Hajnoczi trace_multiwrite_cb(mcb, ret); 188761007b31SStefan Hajnoczi 188861007b31SStefan Hajnoczi if (ret < 0 && !mcb->error) { 188961007b31SStefan Hajnoczi mcb->error = ret; 189061007b31SStefan Hajnoczi } 189161007b31SStefan Hajnoczi 189261007b31SStefan Hajnoczi mcb->num_requests--; 189361007b31SStefan Hajnoczi if (mcb->num_requests == 0) { 189461007b31SStefan Hajnoczi multiwrite_user_cb(mcb); 189561007b31SStefan Hajnoczi g_free(mcb); 189661007b31SStefan Hajnoczi } 189761007b31SStefan Hajnoczi } 189861007b31SStefan Hajnoczi 189961007b31SStefan Hajnoczi static int multiwrite_req_compare(const void *a, const void *b) 190061007b31SStefan Hajnoczi { 190161007b31SStefan Hajnoczi const BlockRequest *req1 = a, *req2 = b; 190261007b31SStefan Hajnoczi 190361007b31SStefan Hajnoczi /* 190461007b31SStefan Hajnoczi * Note that we can't simply subtract req2->sector from req1->sector 190561007b31SStefan Hajnoczi * here as that could overflow the return value. 190661007b31SStefan Hajnoczi */ 190761007b31SStefan Hajnoczi if (req1->sector > req2->sector) { 190861007b31SStefan Hajnoczi return 1; 190961007b31SStefan Hajnoczi } else if (req1->sector < req2->sector) { 191061007b31SStefan Hajnoczi return -1; 191161007b31SStefan Hajnoczi } else { 191261007b31SStefan Hajnoczi return 0; 191361007b31SStefan Hajnoczi } 191461007b31SStefan Hajnoczi } 191561007b31SStefan Hajnoczi 191661007b31SStefan Hajnoczi /* 191761007b31SStefan Hajnoczi * Takes a bunch of requests and tries to merge them. Returns the number of 191861007b31SStefan Hajnoczi * requests that remain after merging. 191961007b31SStefan Hajnoczi */ 192061007b31SStefan Hajnoczi static int multiwrite_merge(BlockDriverState *bs, BlockRequest *reqs, 192161007b31SStefan Hajnoczi int num_reqs, MultiwriteCB *mcb) 192261007b31SStefan Hajnoczi { 192361007b31SStefan Hajnoczi int i, outidx; 192461007b31SStefan Hajnoczi 192561007b31SStefan Hajnoczi // Sort requests by start sector 192661007b31SStefan Hajnoczi qsort(reqs, num_reqs, sizeof(*reqs), &multiwrite_req_compare); 192761007b31SStefan Hajnoczi 192861007b31SStefan Hajnoczi // Check if adjacent requests touch the same clusters. If so, combine them, 192961007b31SStefan Hajnoczi // filling up gaps with zero sectors. 193061007b31SStefan Hajnoczi outidx = 0; 193161007b31SStefan Hajnoczi for (i = 1; i < num_reqs; i++) { 193261007b31SStefan Hajnoczi int merge = 0; 193361007b31SStefan Hajnoczi int64_t oldreq_last = reqs[outidx].sector + reqs[outidx].nb_sectors; 193461007b31SStefan Hajnoczi 193561007b31SStefan Hajnoczi // Handle exactly sequential writes and overlapping writes. 193661007b31SStefan Hajnoczi if (reqs[i].sector <= oldreq_last) { 193761007b31SStefan Hajnoczi merge = 1; 193861007b31SStefan Hajnoczi } 193961007b31SStefan Hajnoczi 1940222565f6SStefan Hajnoczi if (reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1 > 1941222565f6SStefan Hajnoczi bs->bl.max_iov) { 194261007b31SStefan Hajnoczi merge = 0; 194361007b31SStefan Hajnoczi } 194461007b31SStefan Hajnoczi 194561007b31SStefan Hajnoczi if (bs->bl.max_transfer_length && reqs[outidx].nb_sectors + 194661007b31SStefan Hajnoczi reqs[i].nb_sectors > bs->bl.max_transfer_length) { 194761007b31SStefan Hajnoczi merge = 0; 194861007b31SStefan Hajnoczi } 194961007b31SStefan Hajnoczi 195061007b31SStefan Hajnoczi if (merge) { 195161007b31SStefan Hajnoczi size_t size; 195261007b31SStefan Hajnoczi QEMUIOVector *qiov = g_malloc0(sizeof(*qiov)); 195361007b31SStefan Hajnoczi qemu_iovec_init(qiov, 195461007b31SStefan Hajnoczi reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1); 195561007b31SStefan Hajnoczi 195661007b31SStefan Hajnoczi // Add the first request to the merged one. If the requests are 195761007b31SStefan Hajnoczi // overlapping, drop the last sectors of the first request. 195861007b31SStefan Hajnoczi size = (reqs[i].sector - reqs[outidx].sector) << 9; 195961007b31SStefan Hajnoczi qemu_iovec_concat(qiov, reqs[outidx].qiov, 0, size); 196061007b31SStefan Hajnoczi 196161007b31SStefan Hajnoczi // We should need to add any zeros between the two requests 196261007b31SStefan Hajnoczi assert (reqs[i].sector <= oldreq_last); 196361007b31SStefan Hajnoczi 196461007b31SStefan Hajnoczi // Add the second request 196561007b31SStefan Hajnoczi qemu_iovec_concat(qiov, reqs[i].qiov, 0, reqs[i].qiov->size); 196661007b31SStefan Hajnoczi 196761007b31SStefan Hajnoczi // Add tail of first request, if necessary 196861007b31SStefan Hajnoczi if (qiov->size < reqs[outidx].qiov->size) { 196961007b31SStefan Hajnoczi qemu_iovec_concat(qiov, reqs[outidx].qiov, qiov->size, 197061007b31SStefan Hajnoczi reqs[outidx].qiov->size - qiov->size); 197161007b31SStefan Hajnoczi } 197261007b31SStefan Hajnoczi 197361007b31SStefan Hajnoczi reqs[outidx].nb_sectors = qiov->size >> 9; 197461007b31SStefan Hajnoczi reqs[outidx].qiov = qiov; 197561007b31SStefan Hajnoczi 197661007b31SStefan Hajnoczi mcb->callbacks[i].free_qiov = reqs[outidx].qiov; 197761007b31SStefan Hajnoczi } else { 197861007b31SStefan Hajnoczi outidx++; 197961007b31SStefan Hajnoczi reqs[outidx].sector = reqs[i].sector; 198061007b31SStefan Hajnoczi reqs[outidx].nb_sectors = reqs[i].nb_sectors; 198161007b31SStefan Hajnoczi reqs[outidx].qiov = reqs[i].qiov; 198261007b31SStefan Hajnoczi } 198361007b31SStefan Hajnoczi } 198461007b31SStefan Hajnoczi 19857f0e9da6SMax Reitz if (bs->blk) { 19867f0e9da6SMax Reitz block_acct_merge_done(blk_get_stats(bs->blk), BLOCK_ACCT_WRITE, 19877f0e9da6SMax Reitz num_reqs - outidx - 1); 19887f0e9da6SMax Reitz } 198961007b31SStefan Hajnoczi 199061007b31SStefan Hajnoczi return outidx + 1; 199161007b31SStefan Hajnoczi } 199261007b31SStefan Hajnoczi 199361007b31SStefan Hajnoczi /* 199461007b31SStefan Hajnoczi * Submit multiple AIO write requests at once. 199561007b31SStefan Hajnoczi * 199661007b31SStefan Hajnoczi * On success, the function returns 0 and all requests in the reqs array have 199761007b31SStefan Hajnoczi * been submitted. In error case this function returns -1, and any of the 199861007b31SStefan Hajnoczi * requests may or may not be submitted yet. In particular, this means that the 199961007b31SStefan Hajnoczi * callback will be called for some of the requests, for others it won't. The 200061007b31SStefan Hajnoczi * caller must check the error field of the BlockRequest to wait for the right 200161007b31SStefan Hajnoczi * callbacks (if error != 0, no callback will be called). 200261007b31SStefan Hajnoczi * 200361007b31SStefan Hajnoczi * The implementation may modify the contents of the reqs array, e.g. to merge 200461007b31SStefan Hajnoczi * requests. However, the fields opaque and error are left unmodified as they 200561007b31SStefan Hajnoczi * are used to signal failure for a single request to the caller. 200661007b31SStefan Hajnoczi */ 200761007b31SStefan Hajnoczi int bdrv_aio_multiwrite(BlockDriverState *bs, BlockRequest *reqs, int num_reqs) 200861007b31SStefan Hajnoczi { 200961007b31SStefan Hajnoczi MultiwriteCB *mcb; 201061007b31SStefan Hajnoczi int i; 201161007b31SStefan Hajnoczi 201261007b31SStefan Hajnoczi /* don't submit writes if we don't have a medium */ 201361007b31SStefan Hajnoczi if (bs->drv == NULL) { 201461007b31SStefan Hajnoczi for (i = 0; i < num_reqs; i++) { 201561007b31SStefan Hajnoczi reqs[i].error = -ENOMEDIUM; 201661007b31SStefan Hajnoczi } 201761007b31SStefan Hajnoczi return -1; 201861007b31SStefan Hajnoczi } 201961007b31SStefan Hajnoczi 202061007b31SStefan Hajnoczi if (num_reqs == 0) { 202161007b31SStefan Hajnoczi return 0; 202261007b31SStefan Hajnoczi } 202361007b31SStefan Hajnoczi 202461007b31SStefan Hajnoczi // Create MultiwriteCB structure 202561007b31SStefan Hajnoczi mcb = g_malloc0(sizeof(*mcb) + num_reqs * sizeof(*mcb->callbacks)); 202661007b31SStefan Hajnoczi mcb->num_requests = 0; 202761007b31SStefan Hajnoczi mcb->num_callbacks = num_reqs; 202861007b31SStefan Hajnoczi 202961007b31SStefan Hajnoczi for (i = 0; i < num_reqs; i++) { 203061007b31SStefan Hajnoczi mcb->callbacks[i].cb = reqs[i].cb; 203161007b31SStefan Hajnoczi mcb->callbacks[i].opaque = reqs[i].opaque; 203261007b31SStefan Hajnoczi } 203361007b31SStefan Hajnoczi 203461007b31SStefan Hajnoczi // Check for mergable requests 203561007b31SStefan Hajnoczi num_reqs = multiwrite_merge(bs, reqs, num_reqs, mcb); 203661007b31SStefan Hajnoczi 203761007b31SStefan Hajnoczi trace_bdrv_aio_multiwrite(mcb, mcb->num_callbacks, num_reqs); 203861007b31SStefan Hajnoczi 203961007b31SStefan Hajnoczi /* Run the aio requests. */ 204061007b31SStefan Hajnoczi mcb->num_requests = num_reqs; 204161007b31SStefan Hajnoczi for (i = 0; i < num_reqs; i++) { 204261007b31SStefan Hajnoczi bdrv_co_aio_rw_vector(bs, reqs[i].sector, reqs[i].qiov, 204361007b31SStefan Hajnoczi reqs[i].nb_sectors, reqs[i].flags, 204461007b31SStefan Hajnoczi multiwrite_cb, mcb, 204561007b31SStefan Hajnoczi true); 204661007b31SStefan Hajnoczi } 204761007b31SStefan Hajnoczi 204861007b31SStefan Hajnoczi return 0; 204961007b31SStefan Hajnoczi } 205061007b31SStefan Hajnoczi 205161007b31SStefan Hajnoczi void bdrv_aio_cancel(BlockAIOCB *acb) 205261007b31SStefan Hajnoczi { 205361007b31SStefan Hajnoczi qemu_aio_ref(acb); 205461007b31SStefan Hajnoczi bdrv_aio_cancel_async(acb); 205561007b31SStefan Hajnoczi while (acb->refcnt > 1) { 205661007b31SStefan Hajnoczi if (acb->aiocb_info->get_aio_context) { 205761007b31SStefan Hajnoczi aio_poll(acb->aiocb_info->get_aio_context(acb), true); 205861007b31SStefan Hajnoczi } else if (acb->bs) { 205961007b31SStefan Hajnoczi aio_poll(bdrv_get_aio_context(acb->bs), true); 206061007b31SStefan Hajnoczi } else { 206161007b31SStefan Hajnoczi abort(); 206261007b31SStefan Hajnoczi } 206361007b31SStefan Hajnoczi } 206461007b31SStefan Hajnoczi qemu_aio_unref(acb); 206561007b31SStefan Hajnoczi } 206661007b31SStefan Hajnoczi 206761007b31SStefan Hajnoczi /* Async version of aio cancel. The caller is not blocked if the acb implements 206861007b31SStefan Hajnoczi * cancel_async, otherwise we do nothing and let the request normally complete. 206961007b31SStefan Hajnoczi * In either case the completion callback must be called. */ 207061007b31SStefan Hajnoczi void bdrv_aio_cancel_async(BlockAIOCB *acb) 207161007b31SStefan Hajnoczi { 207261007b31SStefan Hajnoczi if (acb->aiocb_info->cancel_async) { 207361007b31SStefan Hajnoczi acb->aiocb_info->cancel_async(acb); 207461007b31SStefan Hajnoczi } 207561007b31SStefan Hajnoczi } 207661007b31SStefan Hajnoczi 207761007b31SStefan Hajnoczi /**************************************************************/ 207861007b31SStefan Hajnoczi /* async block device emulation */ 207961007b31SStefan Hajnoczi 208061007b31SStefan Hajnoczi typedef struct BlockAIOCBSync { 208161007b31SStefan Hajnoczi BlockAIOCB common; 208261007b31SStefan Hajnoczi QEMUBH *bh; 208361007b31SStefan Hajnoczi int ret; 208461007b31SStefan Hajnoczi /* vector translation state */ 208561007b31SStefan Hajnoczi QEMUIOVector *qiov; 208661007b31SStefan Hajnoczi uint8_t *bounce; 208761007b31SStefan Hajnoczi int is_write; 208861007b31SStefan Hajnoczi } BlockAIOCBSync; 208961007b31SStefan Hajnoczi 209061007b31SStefan Hajnoczi static const AIOCBInfo bdrv_em_aiocb_info = { 209161007b31SStefan Hajnoczi .aiocb_size = sizeof(BlockAIOCBSync), 209261007b31SStefan Hajnoczi }; 209361007b31SStefan Hajnoczi 209461007b31SStefan Hajnoczi static void bdrv_aio_bh_cb(void *opaque) 209561007b31SStefan Hajnoczi { 209661007b31SStefan Hajnoczi BlockAIOCBSync *acb = opaque; 209761007b31SStefan Hajnoczi 209861007b31SStefan Hajnoczi if (!acb->is_write && acb->ret >= 0) { 209961007b31SStefan Hajnoczi qemu_iovec_from_buf(acb->qiov, 0, acb->bounce, acb->qiov->size); 210061007b31SStefan Hajnoczi } 210161007b31SStefan Hajnoczi qemu_vfree(acb->bounce); 210261007b31SStefan Hajnoczi acb->common.cb(acb->common.opaque, acb->ret); 210361007b31SStefan Hajnoczi qemu_bh_delete(acb->bh); 210461007b31SStefan Hajnoczi acb->bh = NULL; 210561007b31SStefan Hajnoczi qemu_aio_unref(acb); 210661007b31SStefan Hajnoczi } 210761007b31SStefan Hajnoczi 210861007b31SStefan Hajnoczi static BlockAIOCB *bdrv_aio_rw_vector(BlockDriverState *bs, 210961007b31SStefan Hajnoczi int64_t sector_num, 211061007b31SStefan Hajnoczi QEMUIOVector *qiov, 211161007b31SStefan Hajnoczi int nb_sectors, 211261007b31SStefan Hajnoczi BlockCompletionFunc *cb, 211361007b31SStefan Hajnoczi void *opaque, 211461007b31SStefan Hajnoczi int is_write) 211561007b31SStefan Hajnoczi 211661007b31SStefan Hajnoczi { 211761007b31SStefan Hajnoczi BlockAIOCBSync *acb; 211861007b31SStefan Hajnoczi 211961007b31SStefan Hajnoczi acb = qemu_aio_get(&bdrv_em_aiocb_info, bs, cb, opaque); 212061007b31SStefan Hajnoczi acb->is_write = is_write; 212161007b31SStefan Hajnoczi acb->qiov = qiov; 212261007b31SStefan Hajnoczi acb->bounce = qemu_try_blockalign(bs, qiov->size); 212361007b31SStefan Hajnoczi acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_aio_bh_cb, acb); 212461007b31SStefan Hajnoczi 212561007b31SStefan Hajnoczi if (acb->bounce == NULL) { 212661007b31SStefan Hajnoczi acb->ret = -ENOMEM; 212761007b31SStefan Hajnoczi } else if (is_write) { 212861007b31SStefan Hajnoczi qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size); 212961007b31SStefan Hajnoczi acb->ret = bs->drv->bdrv_write(bs, sector_num, acb->bounce, nb_sectors); 213061007b31SStefan Hajnoczi } else { 213161007b31SStefan Hajnoczi acb->ret = bs->drv->bdrv_read(bs, sector_num, acb->bounce, nb_sectors); 213261007b31SStefan Hajnoczi } 213361007b31SStefan Hajnoczi 213461007b31SStefan Hajnoczi qemu_bh_schedule(acb->bh); 213561007b31SStefan Hajnoczi 213661007b31SStefan Hajnoczi return &acb->common; 213761007b31SStefan Hajnoczi } 213861007b31SStefan Hajnoczi 213961007b31SStefan Hajnoczi static BlockAIOCB *bdrv_aio_readv_em(BlockDriverState *bs, 214061007b31SStefan Hajnoczi int64_t sector_num, QEMUIOVector *qiov, int nb_sectors, 214161007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque) 214261007b31SStefan Hajnoczi { 214361007b31SStefan Hajnoczi return bdrv_aio_rw_vector(bs, sector_num, qiov, nb_sectors, cb, opaque, 0); 214461007b31SStefan Hajnoczi } 214561007b31SStefan Hajnoczi 214661007b31SStefan Hajnoczi static BlockAIOCB *bdrv_aio_writev_em(BlockDriverState *bs, 214761007b31SStefan Hajnoczi int64_t sector_num, QEMUIOVector *qiov, int nb_sectors, 214861007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque) 214961007b31SStefan Hajnoczi { 215061007b31SStefan Hajnoczi return bdrv_aio_rw_vector(bs, sector_num, qiov, nb_sectors, cb, opaque, 1); 215161007b31SStefan Hajnoczi } 215261007b31SStefan Hajnoczi 215361007b31SStefan Hajnoczi 215461007b31SStefan Hajnoczi typedef struct BlockAIOCBCoroutine { 215561007b31SStefan Hajnoczi BlockAIOCB common; 215661007b31SStefan Hajnoczi BlockRequest req; 215761007b31SStefan Hajnoczi bool is_write; 215861007b31SStefan Hajnoczi bool need_bh; 215961007b31SStefan Hajnoczi bool *done; 216061007b31SStefan Hajnoczi QEMUBH* bh; 216161007b31SStefan Hajnoczi } BlockAIOCBCoroutine; 216261007b31SStefan Hajnoczi 216361007b31SStefan Hajnoczi static const AIOCBInfo bdrv_em_co_aiocb_info = { 216461007b31SStefan Hajnoczi .aiocb_size = sizeof(BlockAIOCBCoroutine), 216561007b31SStefan Hajnoczi }; 216661007b31SStefan Hajnoczi 216761007b31SStefan Hajnoczi static void bdrv_co_complete(BlockAIOCBCoroutine *acb) 216861007b31SStefan Hajnoczi { 216961007b31SStefan Hajnoczi if (!acb->need_bh) { 217061007b31SStefan Hajnoczi acb->common.cb(acb->common.opaque, acb->req.error); 217161007b31SStefan Hajnoczi qemu_aio_unref(acb); 217261007b31SStefan Hajnoczi } 217361007b31SStefan Hajnoczi } 217461007b31SStefan Hajnoczi 217561007b31SStefan Hajnoczi static void bdrv_co_em_bh(void *opaque) 217661007b31SStefan Hajnoczi { 217761007b31SStefan Hajnoczi BlockAIOCBCoroutine *acb = opaque; 217861007b31SStefan Hajnoczi 217961007b31SStefan Hajnoczi assert(!acb->need_bh); 218061007b31SStefan Hajnoczi qemu_bh_delete(acb->bh); 218161007b31SStefan Hajnoczi bdrv_co_complete(acb); 218261007b31SStefan Hajnoczi } 218361007b31SStefan Hajnoczi 218461007b31SStefan Hajnoczi static void bdrv_co_maybe_schedule_bh(BlockAIOCBCoroutine *acb) 218561007b31SStefan Hajnoczi { 218661007b31SStefan Hajnoczi acb->need_bh = false; 218761007b31SStefan Hajnoczi if (acb->req.error != -EINPROGRESS) { 218861007b31SStefan Hajnoczi BlockDriverState *bs = acb->common.bs; 218961007b31SStefan Hajnoczi 219061007b31SStefan Hajnoczi acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb); 219161007b31SStefan Hajnoczi qemu_bh_schedule(acb->bh); 219261007b31SStefan Hajnoczi } 219361007b31SStefan Hajnoczi } 219461007b31SStefan Hajnoczi 219561007b31SStefan Hajnoczi /* Invoke bdrv_co_do_readv/bdrv_co_do_writev */ 219661007b31SStefan Hajnoczi static void coroutine_fn bdrv_co_do_rw(void *opaque) 219761007b31SStefan Hajnoczi { 219861007b31SStefan Hajnoczi BlockAIOCBCoroutine *acb = opaque; 219961007b31SStefan Hajnoczi BlockDriverState *bs = acb->common.bs; 220061007b31SStefan Hajnoczi 220161007b31SStefan Hajnoczi if (!acb->is_write) { 220261007b31SStefan Hajnoczi acb->req.error = bdrv_co_do_readv(bs, acb->req.sector, 220361007b31SStefan Hajnoczi acb->req.nb_sectors, acb->req.qiov, acb->req.flags); 220461007b31SStefan Hajnoczi } else { 220561007b31SStefan Hajnoczi acb->req.error = bdrv_co_do_writev(bs, acb->req.sector, 220661007b31SStefan Hajnoczi acb->req.nb_sectors, acb->req.qiov, acb->req.flags); 220761007b31SStefan Hajnoczi } 220861007b31SStefan Hajnoczi 220961007b31SStefan Hajnoczi bdrv_co_complete(acb); 221061007b31SStefan Hajnoczi } 221161007b31SStefan Hajnoczi 221261007b31SStefan Hajnoczi static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs, 221361007b31SStefan Hajnoczi int64_t sector_num, 221461007b31SStefan Hajnoczi QEMUIOVector *qiov, 221561007b31SStefan Hajnoczi int nb_sectors, 221661007b31SStefan Hajnoczi BdrvRequestFlags flags, 221761007b31SStefan Hajnoczi BlockCompletionFunc *cb, 221861007b31SStefan Hajnoczi void *opaque, 221961007b31SStefan Hajnoczi bool is_write) 222061007b31SStefan Hajnoczi { 222161007b31SStefan Hajnoczi Coroutine *co; 222261007b31SStefan Hajnoczi BlockAIOCBCoroutine *acb; 222361007b31SStefan Hajnoczi 222461007b31SStefan Hajnoczi acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque); 222561007b31SStefan Hajnoczi acb->need_bh = true; 222661007b31SStefan Hajnoczi acb->req.error = -EINPROGRESS; 222761007b31SStefan Hajnoczi acb->req.sector = sector_num; 222861007b31SStefan Hajnoczi acb->req.nb_sectors = nb_sectors; 222961007b31SStefan Hajnoczi acb->req.qiov = qiov; 223061007b31SStefan Hajnoczi acb->req.flags = flags; 223161007b31SStefan Hajnoczi acb->is_write = is_write; 223261007b31SStefan Hajnoczi 223361007b31SStefan Hajnoczi co = qemu_coroutine_create(bdrv_co_do_rw); 223461007b31SStefan Hajnoczi qemu_coroutine_enter(co, acb); 223561007b31SStefan Hajnoczi 223661007b31SStefan Hajnoczi bdrv_co_maybe_schedule_bh(acb); 223761007b31SStefan Hajnoczi return &acb->common; 223861007b31SStefan Hajnoczi } 223961007b31SStefan Hajnoczi 224061007b31SStefan Hajnoczi static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque) 224161007b31SStefan Hajnoczi { 224261007b31SStefan Hajnoczi BlockAIOCBCoroutine *acb = opaque; 224361007b31SStefan Hajnoczi BlockDriverState *bs = acb->common.bs; 224461007b31SStefan Hajnoczi 224561007b31SStefan Hajnoczi acb->req.error = bdrv_co_flush(bs); 224661007b31SStefan Hajnoczi bdrv_co_complete(acb); 224761007b31SStefan Hajnoczi } 224861007b31SStefan Hajnoczi 224961007b31SStefan Hajnoczi BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs, 225061007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque) 225161007b31SStefan Hajnoczi { 225261007b31SStefan Hajnoczi trace_bdrv_aio_flush(bs, opaque); 225361007b31SStefan Hajnoczi 225461007b31SStefan Hajnoczi Coroutine *co; 225561007b31SStefan Hajnoczi BlockAIOCBCoroutine *acb; 225661007b31SStefan Hajnoczi 225761007b31SStefan Hajnoczi acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque); 225861007b31SStefan Hajnoczi acb->need_bh = true; 225961007b31SStefan Hajnoczi acb->req.error = -EINPROGRESS; 226061007b31SStefan Hajnoczi 226161007b31SStefan Hajnoczi co = qemu_coroutine_create(bdrv_aio_flush_co_entry); 226261007b31SStefan Hajnoczi qemu_coroutine_enter(co, acb); 226361007b31SStefan Hajnoczi 226461007b31SStefan Hajnoczi bdrv_co_maybe_schedule_bh(acb); 226561007b31SStefan Hajnoczi return &acb->common; 226661007b31SStefan Hajnoczi } 226761007b31SStefan Hajnoczi 226861007b31SStefan Hajnoczi static void coroutine_fn bdrv_aio_discard_co_entry(void *opaque) 226961007b31SStefan Hajnoczi { 227061007b31SStefan Hajnoczi BlockAIOCBCoroutine *acb = opaque; 227161007b31SStefan Hajnoczi BlockDriverState *bs = acb->common.bs; 227261007b31SStefan Hajnoczi 227361007b31SStefan Hajnoczi acb->req.error = bdrv_co_discard(bs, acb->req.sector, acb->req.nb_sectors); 227461007b31SStefan Hajnoczi bdrv_co_complete(acb); 227561007b31SStefan Hajnoczi } 227661007b31SStefan Hajnoczi 227761007b31SStefan Hajnoczi BlockAIOCB *bdrv_aio_discard(BlockDriverState *bs, 227861007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, 227961007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque) 228061007b31SStefan Hajnoczi { 228161007b31SStefan Hajnoczi Coroutine *co; 228261007b31SStefan Hajnoczi BlockAIOCBCoroutine *acb; 228361007b31SStefan Hajnoczi 228461007b31SStefan Hajnoczi trace_bdrv_aio_discard(bs, sector_num, nb_sectors, opaque); 228561007b31SStefan Hajnoczi 228661007b31SStefan Hajnoczi acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque); 228761007b31SStefan Hajnoczi acb->need_bh = true; 228861007b31SStefan Hajnoczi acb->req.error = -EINPROGRESS; 228961007b31SStefan Hajnoczi acb->req.sector = sector_num; 229061007b31SStefan Hajnoczi acb->req.nb_sectors = nb_sectors; 229161007b31SStefan Hajnoczi co = qemu_coroutine_create(bdrv_aio_discard_co_entry); 229261007b31SStefan Hajnoczi qemu_coroutine_enter(co, acb); 229361007b31SStefan Hajnoczi 229461007b31SStefan Hajnoczi bdrv_co_maybe_schedule_bh(acb); 229561007b31SStefan Hajnoczi return &acb->common; 229661007b31SStefan Hajnoczi } 229761007b31SStefan Hajnoczi 229861007b31SStefan Hajnoczi void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs, 229961007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque) 230061007b31SStefan Hajnoczi { 230161007b31SStefan Hajnoczi BlockAIOCB *acb; 230261007b31SStefan Hajnoczi 2303c84b3192SPaolo Bonzini acb = g_malloc(aiocb_info->aiocb_size); 230461007b31SStefan Hajnoczi acb->aiocb_info = aiocb_info; 230561007b31SStefan Hajnoczi acb->bs = bs; 230661007b31SStefan Hajnoczi acb->cb = cb; 230761007b31SStefan Hajnoczi acb->opaque = opaque; 230861007b31SStefan Hajnoczi acb->refcnt = 1; 230961007b31SStefan Hajnoczi return acb; 231061007b31SStefan Hajnoczi } 231161007b31SStefan Hajnoczi 231261007b31SStefan Hajnoczi void qemu_aio_ref(void *p) 231361007b31SStefan Hajnoczi { 231461007b31SStefan Hajnoczi BlockAIOCB *acb = p; 231561007b31SStefan Hajnoczi acb->refcnt++; 231661007b31SStefan Hajnoczi } 231761007b31SStefan Hajnoczi 231861007b31SStefan Hajnoczi void qemu_aio_unref(void *p) 231961007b31SStefan Hajnoczi { 232061007b31SStefan Hajnoczi BlockAIOCB *acb = p; 232161007b31SStefan Hajnoczi assert(acb->refcnt > 0); 232261007b31SStefan Hajnoczi if (--acb->refcnt == 0) { 2323c84b3192SPaolo Bonzini g_free(acb); 232461007b31SStefan Hajnoczi } 232561007b31SStefan Hajnoczi } 232661007b31SStefan Hajnoczi 232761007b31SStefan Hajnoczi /**************************************************************/ 232861007b31SStefan Hajnoczi /* Coroutine block device emulation */ 232961007b31SStefan Hajnoczi 233061007b31SStefan Hajnoczi typedef struct CoroutineIOCompletion { 233161007b31SStefan Hajnoczi Coroutine *coroutine; 233261007b31SStefan Hajnoczi int ret; 233361007b31SStefan Hajnoczi } CoroutineIOCompletion; 233461007b31SStefan Hajnoczi 233561007b31SStefan Hajnoczi static void bdrv_co_io_em_complete(void *opaque, int ret) 233661007b31SStefan Hajnoczi { 233761007b31SStefan Hajnoczi CoroutineIOCompletion *co = opaque; 233861007b31SStefan Hajnoczi 233961007b31SStefan Hajnoczi co->ret = ret; 234061007b31SStefan Hajnoczi qemu_coroutine_enter(co->coroutine, NULL); 234161007b31SStefan Hajnoczi } 234261007b31SStefan Hajnoczi 234361007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_io_em(BlockDriverState *bs, int64_t sector_num, 234461007b31SStefan Hajnoczi int nb_sectors, QEMUIOVector *iov, 234561007b31SStefan Hajnoczi bool is_write) 234661007b31SStefan Hajnoczi { 234761007b31SStefan Hajnoczi CoroutineIOCompletion co = { 234861007b31SStefan Hajnoczi .coroutine = qemu_coroutine_self(), 234961007b31SStefan Hajnoczi }; 235061007b31SStefan Hajnoczi BlockAIOCB *acb; 235161007b31SStefan Hajnoczi 235261007b31SStefan Hajnoczi if (is_write) { 235361007b31SStefan Hajnoczi acb = bs->drv->bdrv_aio_writev(bs, sector_num, iov, nb_sectors, 235461007b31SStefan Hajnoczi bdrv_co_io_em_complete, &co); 235561007b31SStefan Hajnoczi } else { 235661007b31SStefan Hajnoczi acb = bs->drv->bdrv_aio_readv(bs, sector_num, iov, nb_sectors, 235761007b31SStefan Hajnoczi bdrv_co_io_em_complete, &co); 235861007b31SStefan Hajnoczi } 235961007b31SStefan Hajnoczi 236061007b31SStefan Hajnoczi trace_bdrv_co_io_em(bs, sector_num, nb_sectors, is_write, acb); 236161007b31SStefan Hajnoczi if (!acb) { 236261007b31SStefan Hajnoczi return -EIO; 236361007b31SStefan Hajnoczi } 236461007b31SStefan Hajnoczi qemu_coroutine_yield(); 236561007b31SStefan Hajnoczi 236661007b31SStefan Hajnoczi return co.ret; 236761007b31SStefan Hajnoczi } 236861007b31SStefan Hajnoczi 236961007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_readv_em(BlockDriverState *bs, 237061007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, 237161007b31SStefan Hajnoczi QEMUIOVector *iov) 237261007b31SStefan Hajnoczi { 237361007b31SStefan Hajnoczi return bdrv_co_io_em(bs, sector_num, nb_sectors, iov, false); 237461007b31SStefan Hajnoczi } 237561007b31SStefan Hajnoczi 237661007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_writev_em(BlockDriverState *bs, 237761007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, 237861007b31SStefan Hajnoczi QEMUIOVector *iov) 237961007b31SStefan Hajnoczi { 238061007b31SStefan Hajnoczi return bdrv_co_io_em(bs, sector_num, nb_sectors, iov, true); 238161007b31SStefan Hajnoczi } 238261007b31SStefan Hajnoczi 238361007b31SStefan Hajnoczi static void coroutine_fn bdrv_flush_co_entry(void *opaque) 238461007b31SStefan Hajnoczi { 238561007b31SStefan Hajnoczi RwCo *rwco = opaque; 238661007b31SStefan Hajnoczi 238761007b31SStefan Hajnoczi rwco->ret = bdrv_co_flush(rwco->bs); 238861007b31SStefan Hajnoczi } 238961007b31SStefan Hajnoczi 239061007b31SStefan Hajnoczi int coroutine_fn bdrv_co_flush(BlockDriverState *bs) 239161007b31SStefan Hajnoczi { 239261007b31SStefan Hajnoczi int ret; 2393cdb5e315SFam Zheng BdrvTrackedRequest req; 239461007b31SStefan Hajnoczi 23951b6bc94dSDimitris Aragiorgis if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs) || 23961b6bc94dSDimitris Aragiorgis bdrv_is_sg(bs)) { 239761007b31SStefan Hajnoczi return 0; 239861007b31SStefan Hajnoczi } 239961007b31SStefan Hajnoczi 2400cdb5e315SFam Zheng tracked_request_begin(&req, bs, 0, 0, BDRV_TRACKED_FLUSH); 2401c32b82afSPavel Dovgalyuk 2402c32b82afSPavel Dovgalyuk /* Write back all layers by calling one driver function */ 2403c32b82afSPavel Dovgalyuk if (bs->drv->bdrv_co_flush) { 2404c32b82afSPavel Dovgalyuk ret = bs->drv->bdrv_co_flush(bs); 2405c32b82afSPavel Dovgalyuk goto out; 2406c32b82afSPavel Dovgalyuk } 2407c32b82afSPavel Dovgalyuk 240861007b31SStefan Hajnoczi /* Write back cached data to the OS even with cache=unsafe */ 240961007b31SStefan Hajnoczi BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS); 241061007b31SStefan Hajnoczi if (bs->drv->bdrv_co_flush_to_os) { 241161007b31SStefan Hajnoczi ret = bs->drv->bdrv_co_flush_to_os(bs); 241261007b31SStefan Hajnoczi if (ret < 0) { 2413cdb5e315SFam Zheng goto out; 241461007b31SStefan Hajnoczi } 241561007b31SStefan Hajnoczi } 241661007b31SStefan Hajnoczi 241761007b31SStefan Hajnoczi /* But don't actually force it to the disk with cache=unsafe */ 241861007b31SStefan Hajnoczi if (bs->open_flags & BDRV_O_NO_FLUSH) { 241961007b31SStefan Hajnoczi goto flush_parent; 242061007b31SStefan Hajnoczi } 242161007b31SStefan Hajnoczi 242261007b31SStefan Hajnoczi BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK); 242361007b31SStefan Hajnoczi if (bs->drv->bdrv_co_flush_to_disk) { 242461007b31SStefan Hajnoczi ret = bs->drv->bdrv_co_flush_to_disk(bs); 242561007b31SStefan Hajnoczi } else if (bs->drv->bdrv_aio_flush) { 242661007b31SStefan Hajnoczi BlockAIOCB *acb; 242761007b31SStefan Hajnoczi CoroutineIOCompletion co = { 242861007b31SStefan Hajnoczi .coroutine = qemu_coroutine_self(), 242961007b31SStefan Hajnoczi }; 243061007b31SStefan Hajnoczi 243161007b31SStefan Hajnoczi acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co); 243261007b31SStefan Hajnoczi if (acb == NULL) { 243361007b31SStefan Hajnoczi ret = -EIO; 243461007b31SStefan Hajnoczi } else { 243561007b31SStefan Hajnoczi qemu_coroutine_yield(); 243661007b31SStefan Hajnoczi ret = co.ret; 243761007b31SStefan Hajnoczi } 243861007b31SStefan Hajnoczi } else { 243961007b31SStefan Hajnoczi /* 244061007b31SStefan Hajnoczi * Some block drivers always operate in either writethrough or unsafe 244161007b31SStefan Hajnoczi * mode and don't support bdrv_flush therefore. Usually qemu doesn't 244261007b31SStefan Hajnoczi * know how the server works (because the behaviour is hardcoded or 244361007b31SStefan Hajnoczi * depends on server-side configuration), so we can't ensure that 244461007b31SStefan Hajnoczi * everything is safe on disk. Returning an error doesn't work because 244561007b31SStefan Hajnoczi * that would break guests even if the server operates in writethrough 244661007b31SStefan Hajnoczi * mode. 244761007b31SStefan Hajnoczi * 244861007b31SStefan Hajnoczi * Let's hope the user knows what he's doing. 244961007b31SStefan Hajnoczi */ 245061007b31SStefan Hajnoczi ret = 0; 245161007b31SStefan Hajnoczi } 245261007b31SStefan Hajnoczi if (ret < 0) { 2453cdb5e315SFam Zheng goto out; 245461007b31SStefan Hajnoczi } 245561007b31SStefan Hajnoczi 245661007b31SStefan Hajnoczi /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH 245761007b31SStefan Hajnoczi * in the case of cache=unsafe, so there are no useless flushes. 245861007b31SStefan Hajnoczi */ 245961007b31SStefan Hajnoczi flush_parent: 2460cdb5e315SFam Zheng ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0; 2461cdb5e315SFam Zheng out: 2462cdb5e315SFam Zheng tracked_request_end(&req); 2463cdb5e315SFam Zheng return ret; 246461007b31SStefan Hajnoczi } 246561007b31SStefan Hajnoczi 246661007b31SStefan Hajnoczi int bdrv_flush(BlockDriverState *bs) 246761007b31SStefan Hajnoczi { 246861007b31SStefan Hajnoczi Coroutine *co; 246961007b31SStefan Hajnoczi RwCo rwco = { 247061007b31SStefan Hajnoczi .bs = bs, 247161007b31SStefan Hajnoczi .ret = NOT_DONE, 247261007b31SStefan Hajnoczi }; 247361007b31SStefan Hajnoczi 247461007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 247561007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 247661007b31SStefan Hajnoczi bdrv_flush_co_entry(&rwco); 247761007b31SStefan Hajnoczi } else { 247861007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 247961007b31SStefan Hajnoczi 248061007b31SStefan Hajnoczi co = qemu_coroutine_create(bdrv_flush_co_entry); 248161007b31SStefan Hajnoczi qemu_coroutine_enter(co, &rwco); 248261007b31SStefan Hajnoczi while (rwco.ret == NOT_DONE) { 248361007b31SStefan Hajnoczi aio_poll(aio_context, true); 248461007b31SStefan Hajnoczi } 248561007b31SStefan Hajnoczi } 248661007b31SStefan Hajnoczi 248761007b31SStefan Hajnoczi return rwco.ret; 248861007b31SStefan Hajnoczi } 248961007b31SStefan Hajnoczi 249061007b31SStefan Hajnoczi typedef struct DiscardCo { 249161007b31SStefan Hajnoczi BlockDriverState *bs; 249261007b31SStefan Hajnoczi int64_t sector_num; 249361007b31SStefan Hajnoczi int nb_sectors; 249461007b31SStefan Hajnoczi int ret; 249561007b31SStefan Hajnoczi } DiscardCo; 249661007b31SStefan Hajnoczi static void coroutine_fn bdrv_discard_co_entry(void *opaque) 249761007b31SStefan Hajnoczi { 249861007b31SStefan Hajnoczi DiscardCo *rwco = opaque; 249961007b31SStefan Hajnoczi 250061007b31SStefan Hajnoczi rwco->ret = bdrv_co_discard(rwco->bs, rwco->sector_num, rwco->nb_sectors); 250161007b31SStefan Hajnoczi } 250261007b31SStefan Hajnoczi 250361007b31SStefan Hajnoczi int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num, 250461007b31SStefan Hajnoczi int nb_sectors) 250561007b31SStefan Hajnoczi { 2506b1066c87SFam Zheng BdrvTrackedRequest req; 250761007b31SStefan Hajnoczi int max_discard, ret; 250861007b31SStefan Hajnoczi 250961007b31SStefan Hajnoczi if (!bs->drv) { 251061007b31SStefan Hajnoczi return -ENOMEDIUM; 251161007b31SStefan Hajnoczi } 251261007b31SStefan Hajnoczi 251361007b31SStefan Hajnoczi ret = bdrv_check_request(bs, sector_num, nb_sectors); 251461007b31SStefan Hajnoczi if (ret < 0) { 251561007b31SStefan Hajnoczi return ret; 251661007b31SStefan Hajnoczi } else if (bs->read_only) { 2517eaf5fe2dSPaolo Bonzini return -EPERM; 251861007b31SStefan Hajnoczi } 251904c01a5cSKevin Wolf assert(!(bs->open_flags & BDRV_O_INACTIVE)); 252061007b31SStefan Hajnoczi 252161007b31SStefan Hajnoczi /* Do nothing if disabled. */ 252261007b31SStefan Hajnoczi if (!(bs->open_flags & BDRV_O_UNMAP)) { 252361007b31SStefan Hajnoczi return 0; 252461007b31SStefan Hajnoczi } 252561007b31SStefan Hajnoczi 252661007b31SStefan Hajnoczi if (!bs->drv->bdrv_co_discard && !bs->drv->bdrv_aio_discard) { 252761007b31SStefan Hajnoczi return 0; 252861007b31SStefan Hajnoczi } 252961007b31SStefan Hajnoczi 2530b1066c87SFam Zheng tracked_request_begin(&req, bs, sector_num, nb_sectors, 2531b1066c87SFam Zheng BDRV_TRACKED_DISCARD); 253250824995SFam Zheng bdrv_set_dirty(bs, sector_num, nb_sectors); 253350824995SFam Zheng 253461007b31SStefan Hajnoczi max_discard = MIN_NON_ZERO(bs->bl.max_discard, BDRV_REQUEST_MAX_SECTORS); 253561007b31SStefan Hajnoczi while (nb_sectors > 0) { 253661007b31SStefan Hajnoczi int ret; 253761007b31SStefan Hajnoczi int num = nb_sectors; 253861007b31SStefan Hajnoczi 253961007b31SStefan Hajnoczi /* align request */ 254061007b31SStefan Hajnoczi if (bs->bl.discard_alignment && 254161007b31SStefan Hajnoczi num >= bs->bl.discard_alignment && 254261007b31SStefan Hajnoczi sector_num % bs->bl.discard_alignment) { 254361007b31SStefan Hajnoczi if (num > bs->bl.discard_alignment) { 254461007b31SStefan Hajnoczi num = bs->bl.discard_alignment; 254561007b31SStefan Hajnoczi } 254661007b31SStefan Hajnoczi num -= sector_num % bs->bl.discard_alignment; 254761007b31SStefan Hajnoczi } 254861007b31SStefan Hajnoczi 254961007b31SStefan Hajnoczi /* limit request size */ 255061007b31SStefan Hajnoczi if (num > max_discard) { 255161007b31SStefan Hajnoczi num = max_discard; 255261007b31SStefan Hajnoczi } 255361007b31SStefan Hajnoczi 255461007b31SStefan Hajnoczi if (bs->drv->bdrv_co_discard) { 255561007b31SStefan Hajnoczi ret = bs->drv->bdrv_co_discard(bs, sector_num, num); 255661007b31SStefan Hajnoczi } else { 255761007b31SStefan Hajnoczi BlockAIOCB *acb; 255861007b31SStefan Hajnoczi CoroutineIOCompletion co = { 255961007b31SStefan Hajnoczi .coroutine = qemu_coroutine_self(), 256061007b31SStefan Hajnoczi }; 256161007b31SStefan Hajnoczi 256261007b31SStefan Hajnoczi acb = bs->drv->bdrv_aio_discard(bs, sector_num, nb_sectors, 256361007b31SStefan Hajnoczi bdrv_co_io_em_complete, &co); 256461007b31SStefan Hajnoczi if (acb == NULL) { 2565b1066c87SFam Zheng ret = -EIO; 2566b1066c87SFam Zheng goto out; 256761007b31SStefan Hajnoczi } else { 256861007b31SStefan Hajnoczi qemu_coroutine_yield(); 256961007b31SStefan Hajnoczi ret = co.ret; 257061007b31SStefan Hajnoczi } 257161007b31SStefan Hajnoczi } 257261007b31SStefan Hajnoczi if (ret && ret != -ENOTSUP) { 2573b1066c87SFam Zheng goto out; 257461007b31SStefan Hajnoczi } 257561007b31SStefan Hajnoczi 257661007b31SStefan Hajnoczi sector_num += num; 257761007b31SStefan Hajnoczi nb_sectors -= num; 257861007b31SStefan Hajnoczi } 2579b1066c87SFam Zheng ret = 0; 2580b1066c87SFam Zheng out: 2581b1066c87SFam Zheng tracked_request_end(&req); 2582b1066c87SFam Zheng return ret; 258361007b31SStefan Hajnoczi } 258461007b31SStefan Hajnoczi 258561007b31SStefan Hajnoczi int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors) 258661007b31SStefan Hajnoczi { 258761007b31SStefan Hajnoczi Coroutine *co; 258861007b31SStefan Hajnoczi DiscardCo rwco = { 258961007b31SStefan Hajnoczi .bs = bs, 259061007b31SStefan Hajnoczi .sector_num = sector_num, 259161007b31SStefan Hajnoczi .nb_sectors = nb_sectors, 259261007b31SStefan Hajnoczi .ret = NOT_DONE, 259361007b31SStefan Hajnoczi }; 259461007b31SStefan Hajnoczi 259561007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 259661007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 259761007b31SStefan Hajnoczi bdrv_discard_co_entry(&rwco); 259861007b31SStefan Hajnoczi } else { 259961007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 260061007b31SStefan Hajnoczi 260161007b31SStefan Hajnoczi co = qemu_coroutine_create(bdrv_discard_co_entry); 260261007b31SStefan Hajnoczi qemu_coroutine_enter(co, &rwco); 260361007b31SStefan Hajnoczi while (rwco.ret == NOT_DONE) { 260461007b31SStefan Hajnoczi aio_poll(aio_context, true); 260561007b31SStefan Hajnoczi } 260661007b31SStefan Hajnoczi } 260761007b31SStefan Hajnoczi 260861007b31SStefan Hajnoczi return rwco.ret; 260961007b31SStefan Hajnoczi } 261061007b31SStefan Hajnoczi 26115c5ae76aSFam Zheng typedef struct { 26125c5ae76aSFam Zheng CoroutineIOCompletion *co; 26135c5ae76aSFam Zheng QEMUBH *bh; 26145c5ae76aSFam Zheng } BdrvIoctlCompletionData; 261561007b31SStefan Hajnoczi 26165c5ae76aSFam Zheng static void bdrv_ioctl_bh_cb(void *opaque) 26175c5ae76aSFam Zheng { 26185c5ae76aSFam Zheng BdrvIoctlCompletionData *data = opaque; 26195c5ae76aSFam Zheng 26205c5ae76aSFam Zheng bdrv_co_io_em_complete(data->co, -ENOTSUP); 26215c5ae76aSFam Zheng qemu_bh_delete(data->bh); 26225c5ae76aSFam Zheng } 26235c5ae76aSFam Zheng 26245c5ae76aSFam Zheng static int bdrv_co_do_ioctl(BlockDriverState *bs, int req, void *buf) 262561007b31SStefan Hajnoczi { 262661007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 26275c5ae76aSFam Zheng BdrvTrackedRequest tracked_req; 26285c5ae76aSFam Zheng CoroutineIOCompletion co = { 26295c5ae76aSFam Zheng .coroutine = qemu_coroutine_self(), 26305c5ae76aSFam Zheng }; 26315c5ae76aSFam Zheng BlockAIOCB *acb; 263261007b31SStefan Hajnoczi 26335c5ae76aSFam Zheng tracked_request_begin(&tracked_req, bs, 0, 0, BDRV_TRACKED_IOCTL); 26345c5ae76aSFam Zheng if (!drv || !drv->bdrv_aio_ioctl) { 26355c5ae76aSFam Zheng co.ret = -ENOTSUP; 26365c5ae76aSFam Zheng goto out; 26375c5ae76aSFam Zheng } 26385c5ae76aSFam Zheng 26395c5ae76aSFam Zheng acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co); 26405c5ae76aSFam Zheng if (!acb) { 26415c5ae76aSFam Zheng BdrvIoctlCompletionData *data = g_new(BdrvIoctlCompletionData, 1); 26425c5ae76aSFam Zheng data->bh = aio_bh_new(bdrv_get_aio_context(bs), 26435c5ae76aSFam Zheng bdrv_ioctl_bh_cb, data); 26445c5ae76aSFam Zheng data->co = &co; 26455c5ae76aSFam Zheng qemu_bh_schedule(data->bh); 26465c5ae76aSFam Zheng } 26475c5ae76aSFam Zheng qemu_coroutine_yield(); 26485c5ae76aSFam Zheng out: 26495c5ae76aSFam Zheng tracked_request_end(&tracked_req); 26505c5ae76aSFam Zheng return co.ret; 26515c5ae76aSFam Zheng } 26525c5ae76aSFam Zheng 26535c5ae76aSFam Zheng typedef struct { 26545c5ae76aSFam Zheng BlockDriverState *bs; 26555c5ae76aSFam Zheng int req; 26565c5ae76aSFam Zheng void *buf; 26575c5ae76aSFam Zheng int ret; 26585c5ae76aSFam Zheng } BdrvIoctlCoData; 26595c5ae76aSFam Zheng 26605c5ae76aSFam Zheng static void coroutine_fn bdrv_co_ioctl_entry(void *opaque) 26615c5ae76aSFam Zheng { 26625c5ae76aSFam Zheng BdrvIoctlCoData *data = opaque; 26635c5ae76aSFam Zheng data->ret = bdrv_co_do_ioctl(data->bs, data->req, data->buf); 26645c5ae76aSFam Zheng } 26655c5ae76aSFam Zheng 26665c5ae76aSFam Zheng /* needed for generic scsi interface */ 26675c5ae76aSFam Zheng int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf) 26685c5ae76aSFam Zheng { 26695c5ae76aSFam Zheng BdrvIoctlCoData data = { 26705c5ae76aSFam Zheng .bs = bs, 26715c5ae76aSFam Zheng .req = req, 26725c5ae76aSFam Zheng .buf = buf, 26735c5ae76aSFam Zheng .ret = -EINPROGRESS, 26745c5ae76aSFam Zheng }; 26755c5ae76aSFam Zheng 26765c5ae76aSFam Zheng if (qemu_in_coroutine()) { 26775c5ae76aSFam Zheng /* Fast-path if already in coroutine context */ 26785c5ae76aSFam Zheng bdrv_co_ioctl_entry(&data); 26795c5ae76aSFam Zheng } else { 26805c5ae76aSFam Zheng Coroutine *co = qemu_coroutine_create(bdrv_co_ioctl_entry); 2681ba889444SPaolo Bonzini 26825c5ae76aSFam Zheng qemu_coroutine_enter(co, &data); 26835c5ae76aSFam Zheng while (data.ret == -EINPROGRESS) { 26845c5ae76aSFam Zheng aio_poll(bdrv_get_aio_context(bs), true); 26855c5ae76aSFam Zheng } 2686ba889444SPaolo Bonzini } 26875c5ae76aSFam Zheng return data.ret; 26885c5ae76aSFam Zheng } 26895c5ae76aSFam Zheng 26905c5ae76aSFam Zheng static void coroutine_fn bdrv_co_aio_ioctl_entry(void *opaque) 26915c5ae76aSFam Zheng { 26925c5ae76aSFam Zheng BlockAIOCBCoroutine *acb = opaque; 26935c5ae76aSFam Zheng acb->req.error = bdrv_co_do_ioctl(acb->common.bs, 26945c5ae76aSFam Zheng acb->req.req, acb->req.buf); 26955c5ae76aSFam Zheng bdrv_co_complete(acb); 269661007b31SStefan Hajnoczi } 269761007b31SStefan Hajnoczi 269861007b31SStefan Hajnoczi BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs, 269961007b31SStefan Hajnoczi unsigned long int req, void *buf, 270061007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque) 270161007b31SStefan Hajnoczi { 27025c5ae76aSFam Zheng BlockAIOCBCoroutine *acb = qemu_aio_get(&bdrv_em_co_aiocb_info, 27035c5ae76aSFam Zheng bs, cb, opaque); 27045c5ae76aSFam Zheng Coroutine *co; 270561007b31SStefan Hajnoczi 27065c5ae76aSFam Zheng acb->need_bh = true; 27075c5ae76aSFam Zheng acb->req.error = -EINPROGRESS; 27085c5ae76aSFam Zheng acb->req.req = req; 27095c5ae76aSFam Zheng acb->req.buf = buf; 27105c5ae76aSFam Zheng co = qemu_coroutine_create(bdrv_co_aio_ioctl_entry); 27115c5ae76aSFam Zheng qemu_coroutine_enter(co, acb); 27125c5ae76aSFam Zheng 27135c5ae76aSFam Zheng bdrv_co_maybe_schedule_bh(acb); 27145c5ae76aSFam Zheng return &acb->common; 271561007b31SStefan Hajnoczi } 271661007b31SStefan Hajnoczi 271761007b31SStefan Hajnoczi void *qemu_blockalign(BlockDriverState *bs, size_t size) 271861007b31SStefan Hajnoczi { 271961007b31SStefan Hajnoczi return qemu_memalign(bdrv_opt_mem_align(bs), size); 272061007b31SStefan Hajnoczi } 272161007b31SStefan Hajnoczi 272261007b31SStefan Hajnoczi void *qemu_blockalign0(BlockDriverState *bs, size_t size) 272361007b31SStefan Hajnoczi { 272461007b31SStefan Hajnoczi return memset(qemu_blockalign(bs, size), 0, size); 272561007b31SStefan Hajnoczi } 272661007b31SStefan Hajnoczi 272761007b31SStefan Hajnoczi void *qemu_try_blockalign(BlockDriverState *bs, size_t size) 272861007b31SStefan Hajnoczi { 272961007b31SStefan Hajnoczi size_t align = bdrv_opt_mem_align(bs); 273061007b31SStefan Hajnoczi 273161007b31SStefan Hajnoczi /* Ensure that NULL is never returned on success */ 273261007b31SStefan Hajnoczi assert(align > 0); 273361007b31SStefan Hajnoczi if (size == 0) { 273461007b31SStefan Hajnoczi size = align; 273561007b31SStefan Hajnoczi } 273661007b31SStefan Hajnoczi 273761007b31SStefan Hajnoczi return qemu_try_memalign(align, size); 273861007b31SStefan Hajnoczi } 273961007b31SStefan Hajnoczi 274061007b31SStefan Hajnoczi void *qemu_try_blockalign0(BlockDriverState *bs, size_t size) 274161007b31SStefan Hajnoczi { 274261007b31SStefan Hajnoczi void *mem = qemu_try_blockalign(bs, size); 274361007b31SStefan Hajnoczi 274461007b31SStefan Hajnoczi if (mem) { 274561007b31SStefan Hajnoczi memset(mem, 0, size); 274661007b31SStefan Hajnoczi } 274761007b31SStefan Hajnoczi 274861007b31SStefan Hajnoczi return mem; 274961007b31SStefan Hajnoczi } 275061007b31SStefan Hajnoczi 275161007b31SStefan Hajnoczi /* 275261007b31SStefan Hajnoczi * Check if all memory in this vector is sector aligned. 275361007b31SStefan Hajnoczi */ 275461007b31SStefan Hajnoczi bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov) 275561007b31SStefan Hajnoczi { 275661007b31SStefan Hajnoczi int i; 27574196d2f0SDenis V. Lunev size_t alignment = bdrv_min_mem_align(bs); 275861007b31SStefan Hajnoczi 275961007b31SStefan Hajnoczi for (i = 0; i < qiov->niov; i++) { 276061007b31SStefan Hajnoczi if ((uintptr_t) qiov->iov[i].iov_base % alignment) { 276161007b31SStefan Hajnoczi return false; 276261007b31SStefan Hajnoczi } 276361007b31SStefan Hajnoczi if (qiov->iov[i].iov_len % alignment) { 276461007b31SStefan Hajnoczi return false; 276561007b31SStefan Hajnoczi } 276661007b31SStefan Hajnoczi } 276761007b31SStefan Hajnoczi 276861007b31SStefan Hajnoczi return true; 276961007b31SStefan Hajnoczi } 277061007b31SStefan Hajnoczi 277161007b31SStefan Hajnoczi void bdrv_add_before_write_notifier(BlockDriverState *bs, 277261007b31SStefan Hajnoczi NotifierWithReturn *notifier) 277361007b31SStefan Hajnoczi { 277461007b31SStefan Hajnoczi notifier_with_return_list_add(&bs->before_write_notifiers, notifier); 277561007b31SStefan Hajnoczi } 277661007b31SStefan Hajnoczi 277761007b31SStefan Hajnoczi void bdrv_io_plug(BlockDriverState *bs) 277861007b31SStefan Hajnoczi { 27796b98bd64SPaolo Bonzini BdrvChild *child; 27806b98bd64SPaolo Bonzini 27816b98bd64SPaolo Bonzini QLIST_FOREACH(child, &bs->children, next) { 27826b98bd64SPaolo Bonzini bdrv_io_plug(child->bs); 27836b98bd64SPaolo Bonzini } 27846b98bd64SPaolo Bonzini 27856b98bd64SPaolo Bonzini if (bs->io_plugged++ == 0 && bs->io_plug_disabled == 0) { 278661007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 278761007b31SStefan Hajnoczi if (drv && drv->bdrv_io_plug) { 278861007b31SStefan Hajnoczi drv->bdrv_io_plug(bs); 27896b98bd64SPaolo Bonzini } 279061007b31SStefan Hajnoczi } 279161007b31SStefan Hajnoczi } 279261007b31SStefan Hajnoczi 279361007b31SStefan Hajnoczi void bdrv_io_unplug(BlockDriverState *bs) 279461007b31SStefan Hajnoczi { 27956b98bd64SPaolo Bonzini BdrvChild *child; 27966b98bd64SPaolo Bonzini 27976b98bd64SPaolo Bonzini assert(bs->io_plugged); 27986b98bd64SPaolo Bonzini if (--bs->io_plugged == 0 && bs->io_plug_disabled == 0) { 279961007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 280061007b31SStefan Hajnoczi if (drv && drv->bdrv_io_unplug) { 280161007b31SStefan Hajnoczi drv->bdrv_io_unplug(bs); 280261007b31SStefan Hajnoczi } 280361007b31SStefan Hajnoczi } 280461007b31SStefan Hajnoczi 28056b98bd64SPaolo Bonzini QLIST_FOREACH(child, &bs->children, next) { 28066b98bd64SPaolo Bonzini bdrv_io_unplug(child->bs); 28076b98bd64SPaolo Bonzini } 28086b98bd64SPaolo Bonzini } 28096b98bd64SPaolo Bonzini 28106b98bd64SPaolo Bonzini void bdrv_io_unplugged_begin(BlockDriverState *bs) 281161007b31SStefan Hajnoczi { 28126b98bd64SPaolo Bonzini BdrvChild *child; 28136b98bd64SPaolo Bonzini 28146b98bd64SPaolo Bonzini if (bs->io_plug_disabled++ == 0 && bs->io_plugged > 0) { 281561007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 28166b98bd64SPaolo Bonzini if (drv && drv->bdrv_io_unplug) { 28176b98bd64SPaolo Bonzini drv->bdrv_io_unplug(bs); 28186b98bd64SPaolo Bonzini } 28196b98bd64SPaolo Bonzini } 28206b98bd64SPaolo Bonzini 28216b98bd64SPaolo Bonzini QLIST_FOREACH(child, &bs->children, next) { 28226b98bd64SPaolo Bonzini bdrv_io_unplugged_begin(child->bs); 28236b98bd64SPaolo Bonzini } 28246b98bd64SPaolo Bonzini } 28256b98bd64SPaolo Bonzini 28266b98bd64SPaolo Bonzini void bdrv_io_unplugged_end(BlockDriverState *bs) 28276b98bd64SPaolo Bonzini { 28286b98bd64SPaolo Bonzini BdrvChild *child; 28296b98bd64SPaolo Bonzini 28306b98bd64SPaolo Bonzini assert(bs->io_plug_disabled); 28316b98bd64SPaolo Bonzini QLIST_FOREACH(child, &bs->children, next) { 28326b98bd64SPaolo Bonzini bdrv_io_unplugged_end(child->bs); 28336b98bd64SPaolo Bonzini } 28346b98bd64SPaolo Bonzini 28356b98bd64SPaolo Bonzini if (--bs->io_plug_disabled == 0 && bs->io_plugged > 0) { 28366b98bd64SPaolo Bonzini BlockDriver *drv = bs->drv; 28376b98bd64SPaolo Bonzini if (drv && drv->bdrv_io_plug) { 28386b98bd64SPaolo Bonzini drv->bdrv_io_plug(bs); 28396b98bd64SPaolo Bonzini } 284061007b31SStefan Hajnoczi } 284161007b31SStefan Hajnoczi } 284251288d79SFam Zheng 284351288d79SFam Zheng void bdrv_drained_begin(BlockDriverState *bs) 284451288d79SFam Zheng { 284551288d79SFam Zheng if (!bs->quiesce_counter++) { 284651288d79SFam Zheng aio_disable_external(bdrv_get_aio_context(bs)); 284751288d79SFam Zheng } 284851288d79SFam Zheng bdrv_drain(bs); 284951288d79SFam Zheng } 285051288d79SFam Zheng 285151288d79SFam Zheng void bdrv_drained_end(BlockDriverState *bs) 285251288d79SFam Zheng { 285351288d79SFam Zheng assert(bs->quiesce_counter > 0); 285451288d79SFam Zheng if (--bs->quiesce_counter > 0) { 285551288d79SFam Zheng return; 285651288d79SFam Zheng } 285751288d79SFam Zheng aio_enable_external(bdrv_get_aio_context(bs)); 285851288d79SFam Zheng } 2859