161007b31SStefan Hajnoczi /* 261007b31SStefan Hajnoczi * Block layer I/O functions 361007b31SStefan Hajnoczi * 461007b31SStefan Hajnoczi * Copyright (c) 2003 Fabrice Bellard 561007b31SStefan Hajnoczi * 661007b31SStefan Hajnoczi * Permission is hereby granted, free of charge, to any person obtaining a copy 761007b31SStefan Hajnoczi * of this software and associated documentation files (the "Software"), to deal 861007b31SStefan Hajnoczi * in the Software without restriction, including without limitation the rights 961007b31SStefan Hajnoczi * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 1061007b31SStefan Hajnoczi * copies of the Software, and to permit persons to whom the Software is 1161007b31SStefan Hajnoczi * furnished to do so, subject to the following conditions: 1261007b31SStefan Hajnoczi * 1361007b31SStefan Hajnoczi * The above copyright notice and this permission notice shall be included in 1461007b31SStefan Hajnoczi * all copies or substantial portions of the Software. 1561007b31SStefan Hajnoczi * 1661007b31SStefan Hajnoczi * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 1761007b31SStefan Hajnoczi * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 1861007b31SStefan Hajnoczi * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 1961007b31SStefan Hajnoczi * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 2061007b31SStefan Hajnoczi * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 2161007b31SStefan Hajnoczi * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 2261007b31SStefan Hajnoczi * THE SOFTWARE. 2361007b31SStefan Hajnoczi */ 2461007b31SStefan Hajnoczi 25*80c71a24SPeter Maydell #include "qemu/osdep.h" 2661007b31SStefan Hajnoczi #include "trace.h" 277f0e9da6SMax Reitz #include "sysemu/block-backend.h" 2861007b31SStefan Hajnoczi #include "block/blockjob.h" 2961007b31SStefan Hajnoczi #include "block/block_int.h" 3076f4afb4SAlberto Garcia #include "block/throttle-groups.h" 31d49b6836SMarkus Armbruster #include "qemu/error-report.h" 3261007b31SStefan Hajnoczi 3361007b31SStefan Hajnoczi #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */ 3461007b31SStefan Hajnoczi 3561007b31SStefan Hajnoczi static BlockAIOCB *bdrv_aio_readv_em(BlockDriverState *bs, 3661007b31SStefan Hajnoczi int64_t sector_num, QEMUIOVector *qiov, int nb_sectors, 3761007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque); 3861007b31SStefan Hajnoczi static BlockAIOCB *bdrv_aio_writev_em(BlockDriverState *bs, 3961007b31SStefan Hajnoczi int64_t sector_num, QEMUIOVector *qiov, int nb_sectors, 4061007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque); 4161007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_readv_em(BlockDriverState *bs, 4261007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, 4361007b31SStefan Hajnoczi QEMUIOVector *iov); 4461007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_writev_em(BlockDriverState *bs, 4561007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, 4661007b31SStefan Hajnoczi QEMUIOVector *iov); 4761007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_do_preadv(BlockDriverState *bs, 4861007b31SStefan Hajnoczi int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 4961007b31SStefan Hajnoczi BdrvRequestFlags flags); 5061007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_do_pwritev(BlockDriverState *bs, 5161007b31SStefan Hajnoczi int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 5261007b31SStefan Hajnoczi BdrvRequestFlags flags); 5361007b31SStefan Hajnoczi static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs, 5461007b31SStefan Hajnoczi int64_t sector_num, 5561007b31SStefan Hajnoczi QEMUIOVector *qiov, 5661007b31SStefan Hajnoczi int nb_sectors, 5761007b31SStefan Hajnoczi BdrvRequestFlags flags, 5861007b31SStefan Hajnoczi BlockCompletionFunc *cb, 5961007b31SStefan Hajnoczi void *opaque, 6061007b31SStefan Hajnoczi bool is_write); 6161007b31SStefan Hajnoczi static void coroutine_fn bdrv_co_do_rw(void *opaque); 6261007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs, 6361007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, BdrvRequestFlags flags); 6461007b31SStefan Hajnoczi 6561007b31SStefan Hajnoczi /* throttling disk I/O limits */ 6661007b31SStefan Hajnoczi void bdrv_set_io_limits(BlockDriverState *bs, 6761007b31SStefan Hajnoczi ThrottleConfig *cfg) 6861007b31SStefan Hajnoczi { 6961007b31SStefan Hajnoczi int i; 7061007b31SStefan Hajnoczi 7176f4afb4SAlberto Garcia throttle_group_config(bs, cfg); 7261007b31SStefan Hajnoczi 7361007b31SStefan Hajnoczi for (i = 0; i < 2; i++) { 7461007b31SStefan Hajnoczi qemu_co_enter_next(&bs->throttled_reqs[i]); 7561007b31SStefan Hajnoczi } 7661007b31SStefan Hajnoczi } 7761007b31SStefan Hajnoczi 7861007b31SStefan Hajnoczi /* this function drain all the throttled IOs */ 7961007b31SStefan Hajnoczi static bool bdrv_start_throttled_reqs(BlockDriverState *bs) 8061007b31SStefan Hajnoczi { 8161007b31SStefan Hajnoczi bool drained = false; 8261007b31SStefan Hajnoczi bool enabled = bs->io_limits_enabled; 8361007b31SStefan Hajnoczi int i; 8461007b31SStefan Hajnoczi 8561007b31SStefan Hajnoczi bs->io_limits_enabled = false; 8661007b31SStefan Hajnoczi 8761007b31SStefan Hajnoczi for (i = 0; i < 2; i++) { 8861007b31SStefan Hajnoczi while (qemu_co_enter_next(&bs->throttled_reqs[i])) { 8961007b31SStefan Hajnoczi drained = true; 9061007b31SStefan Hajnoczi } 9161007b31SStefan Hajnoczi } 9261007b31SStefan Hajnoczi 9361007b31SStefan Hajnoczi bs->io_limits_enabled = enabled; 9461007b31SStefan Hajnoczi 9561007b31SStefan Hajnoczi return drained; 9661007b31SStefan Hajnoczi } 9761007b31SStefan Hajnoczi 9861007b31SStefan Hajnoczi void bdrv_io_limits_disable(BlockDriverState *bs) 9961007b31SStefan Hajnoczi { 10061007b31SStefan Hajnoczi bs->io_limits_enabled = false; 10161007b31SStefan Hajnoczi bdrv_start_throttled_reqs(bs); 10276f4afb4SAlberto Garcia throttle_group_unregister_bs(bs); 10361007b31SStefan Hajnoczi } 10461007b31SStefan Hajnoczi 10561007b31SStefan Hajnoczi /* should be called before bdrv_set_io_limits if a limit is set */ 10676f4afb4SAlberto Garcia void bdrv_io_limits_enable(BlockDriverState *bs, const char *group) 10761007b31SStefan Hajnoczi { 10861007b31SStefan Hajnoczi assert(!bs->io_limits_enabled); 10976f4afb4SAlberto Garcia throttle_group_register_bs(bs, group); 11061007b31SStefan Hajnoczi bs->io_limits_enabled = true; 11161007b31SStefan Hajnoczi } 11261007b31SStefan Hajnoczi 11376f4afb4SAlberto Garcia void bdrv_io_limits_update_group(BlockDriverState *bs, const char *group) 11461007b31SStefan Hajnoczi { 11576f4afb4SAlberto Garcia /* this bs is not part of any group */ 11676f4afb4SAlberto Garcia if (!bs->throttle_state) { 11761007b31SStefan Hajnoczi return; 11861007b31SStefan Hajnoczi } 11961007b31SStefan Hajnoczi 12076f4afb4SAlberto Garcia /* this bs is a part of the same group than the one we want */ 12176f4afb4SAlberto Garcia if (!g_strcmp0(throttle_group_get_name(bs), group)) { 12276f4afb4SAlberto Garcia return; 12376f4afb4SAlberto Garcia } 12476f4afb4SAlberto Garcia 12576f4afb4SAlberto Garcia /* need to change the group this bs belong to */ 12676f4afb4SAlberto Garcia bdrv_io_limits_disable(bs); 12776f4afb4SAlberto Garcia bdrv_io_limits_enable(bs, group); 12861007b31SStefan Hajnoczi } 12961007b31SStefan Hajnoczi 13061007b31SStefan Hajnoczi void bdrv_setup_io_funcs(BlockDriver *bdrv) 13161007b31SStefan Hajnoczi { 13261007b31SStefan Hajnoczi /* Block drivers without coroutine functions need emulation */ 13361007b31SStefan Hajnoczi if (!bdrv->bdrv_co_readv) { 13461007b31SStefan Hajnoczi bdrv->bdrv_co_readv = bdrv_co_readv_em; 13561007b31SStefan Hajnoczi bdrv->bdrv_co_writev = bdrv_co_writev_em; 13661007b31SStefan Hajnoczi 13761007b31SStefan Hajnoczi /* bdrv_co_readv_em()/brdv_co_writev_em() work in terms of aio, so if 13861007b31SStefan Hajnoczi * the block driver lacks aio we need to emulate that too. 13961007b31SStefan Hajnoczi */ 14061007b31SStefan Hajnoczi if (!bdrv->bdrv_aio_readv) { 14161007b31SStefan Hajnoczi /* add AIO emulation layer */ 14261007b31SStefan Hajnoczi bdrv->bdrv_aio_readv = bdrv_aio_readv_em; 14361007b31SStefan Hajnoczi bdrv->bdrv_aio_writev = bdrv_aio_writev_em; 14461007b31SStefan Hajnoczi } 14561007b31SStefan Hajnoczi } 14661007b31SStefan Hajnoczi } 14761007b31SStefan Hajnoczi 14861007b31SStefan Hajnoczi void bdrv_refresh_limits(BlockDriverState *bs, Error **errp) 14961007b31SStefan Hajnoczi { 15061007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 15161007b31SStefan Hajnoczi Error *local_err = NULL; 15261007b31SStefan Hajnoczi 15361007b31SStefan Hajnoczi memset(&bs->bl, 0, sizeof(bs->bl)); 15461007b31SStefan Hajnoczi 15561007b31SStefan Hajnoczi if (!drv) { 15661007b31SStefan Hajnoczi return; 15761007b31SStefan Hajnoczi } 15861007b31SStefan Hajnoczi 15961007b31SStefan Hajnoczi /* Take some limits from the children as a default */ 16061007b31SStefan Hajnoczi if (bs->file) { 1619a4f4c31SKevin Wolf bdrv_refresh_limits(bs->file->bs, &local_err); 16261007b31SStefan Hajnoczi if (local_err) { 16361007b31SStefan Hajnoczi error_propagate(errp, local_err); 16461007b31SStefan Hajnoczi return; 16561007b31SStefan Hajnoczi } 1669a4f4c31SKevin Wolf bs->bl.opt_transfer_length = bs->file->bs->bl.opt_transfer_length; 1679a4f4c31SKevin Wolf bs->bl.max_transfer_length = bs->file->bs->bl.max_transfer_length; 1689a4f4c31SKevin Wolf bs->bl.min_mem_alignment = bs->file->bs->bl.min_mem_alignment; 1699a4f4c31SKevin Wolf bs->bl.opt_mem_alignment = bs->file->bs->bl.opt_mem_alignment; 170bd44feb7SStefan Hajnoczi bs->bl.max_iov = bs->file->bs->bl.max_iov; 17161007b31SStefan Hajnoczi } else { 1724196d2f0SDenis V. Lunev bs->bl.min_mem_alignment = 512; 173459b4e66SDenis V. Lunev bs->bl.opt_mem_alignment = getpagesize(); 174bd44feb7SStefan Hajnoczi 175bd44feb7SStefan Hajnoczi /* Safe default since most protocols use readv()/writev()/etc */ 176bd44feb7SStefan Hajnoczi bs->bl.max_iov = IOV_MAX; 17761007b31SStefan Hajnoczi } 17861007b31SStefan Hajnoczi 179760e0063SKevin Wolf if (bs->backing) { 180760e0063SKevin Wolf bdrv_refresh_limits(bs->backing->bs, &local_err); 18161007b31SStefan Hajnoczi if (local_err) { 18261007b31SStefan Hajnoczi error_propagate(errp, local_err); 18361007b31SStefan Hajnoczi return; 18461007b31SStefan Hajnoczi } 18561007b31SStefan Hajnoczi bs->bl.opt_transfer_length = 18661007b31SStefan Hajnoczi MAX(bs->bl.opt_transfer_length, 187760e0063SKevin Wolf bs->backing->bs->bl.opt_transfer_length); 18861007b31SStefan Hajnoczi bs->bl.max_transfer_length = 18961007b31SStefan Hajnoczi MIN_NON_ZERO(bs->bl.max_transfer_length, 190760e0063SKevin Wolf bs->backing->bs->bl.max_transfer_length); 19161007b31SStefan Hajnoczi bs->bl.opt_mem_alignment = 19261007b31SStefan Hajnoczi MAX(bs->bl.opt_mem_alignment, 193760e0063SKevin Wolf bs->backing->bs->bl.opt_mem_alignment); 1944196d2f0SDenis V. Lunev bs->bl.min_mem_alignment = 1954196d2f0SDenis V. Lunev MAX(bs->bl.min_mem_alignment, 196760e0063SKevin Wolf bs->backing->bs->bl.min_mem_alignment); 197bd44feb7SStefan Hajnoczi bs->bl.max_iov = 198bd44feb7SStefan Hajnoczi MIN(bs->bl.max_iov, 199bd44feb7SStefan Hajnoczi bs->backing->bs->bl.max_iov); 20061007b31SStefan Hajnoczi } 20161007b31SStefan Hajnoczi 20261007b31SStefan Hajnoczi /* Then let the driver override it */ 20361007b31SStefan Hajnoczi if (drv->bdrv_refresh_limits) { 20461007b31SStefan Hajnoczi drv->bdrv_refresh_limits(bs, errp); 20561007b31SStefan Hajnoczi } 20661007b31SStefan Hajnoczi } 20761007b31SStefan Hajnoczi 20861007b31SStefan Hajnoczi /** 20961007b31SStefan Hajnoczi * The copy-on-read flag is actually a reference count so multiple users may 21061007b31SStefan Hajnoczi * use the feature without worrying about clobbering its previous state. 21161007b31SStefan Hajnoczi * Copy-on-read stays enabled until all users have called to disable it. 21261007b31SStefan Hajnoczi */ 21361007b31SStefan Hajnoczi void bdrv_enable_copy_on_read(BlockDriverState *bs) 21461007b31SStefan Hajnoczi { 21561007b31SStefan Hajnoczi bs->copy_on_read++; 21661007b31SStefan Hajnoczi } 21761007b31SStefan Hajnoczi 21861007b31SStefan Hajnoczi void bdrv_disable_copy_on_read(BlockDriverState *bs) 21961007b31SStefan Hajnoczi { 22061007b31SStefan Hajnoczi assert(bs->copy_on_read > 0); 22161007b31SStefan Hajnoczi bs->copy_on_read--; 22261007b31SStefan Hajnoczi } 22361007b31SStefan Hajnoczi 22461007b31SStefan Hajnoczi /* Check if any requests are in-flight (including throttled requests) */ 225439db28cSKevin Wolf bool bdrv_requests_pending(BlockDriverState *bs) 22661007b31SStefan Hajnoczi { 22737a639a7SKevin Wolf BdrvChild *child; 22837a639a7SKevin Wolf 22961007b31SStefan Hajnoczi if (!QLIST_EMPTY(&bs->tracked_requests)) { 23061007b31SStefan Hajnoczi return true; 23161007b31SStefan Hajnoczi } 23261007b31SStefan Hajnoczi if (!qemu_co_queue_empty(&bs->throttled_reqs[0])) { 23361007b31SStefan Hajnoczi return true; 23461007b31SStefan Hajnoczi } 23561007b31SStefan Hajnoczi if (!qemu_co_queue_empty(&bs->throttled_reqs[1])) { 23661007b31SStefan Hajnoczi return true; 23761007b31SStefan Hajnoczi } 23837a639a7SKevin Wolf 23937a639a7SKevin Wolf QLIST_FOREACH(child, &bs->children, next) { 24037a639a7SKevin Wolf if (bdrv_requests_pending(child->bs)) { 24161007b31SStefan Hajnoczi return true; 24261007b31SStefan Hajnoczi } 24361007b31SStefan Hajnoczi } 24437a639a7SKevin Wolf 24561007b31SStefan Hajnoczi return false; 24661007b31SStefan Hajnoczi } 24761007b31SStefan Hajnoczi 24867da1dc5SFam Zheng static void bdrv_drain_recurse(BlockDriverState *bs) 24967da1dc5SFam Zheng { 25067da1dc5SFam Zheng BdrvChild *child; 25167da1dc5SFam Zheng 25267da1dc5SFam Zheng if (bs->drv && bs->drv->bdrv_drain) { 25367da1dc5SFam Zheng bs->drv->bdrv_drain(bs); 25467da1dc5SFam Zheng } 25567da1dc5SFam Zheng QLIST_FOREACH(child, &bs->children, next) { 25667da1dc5SFam Zheng bdrv_drain_recurse(child->bs); 25767da1dc5SFam Zheng } 25867da1dc5SFam Zheng } 25967da1dc5SFam Zheng 26061007b31SStefan Hajnoczi /* 26167da1dc5SFam Zheng * Wait for pending requests to complete on a single BlockDriverState subtree, 26267da1dc5SFam Zheng * and suspend block driver's internal I/O until next request arrives. 26361007b31SStefan Hajnoczi * 26461007b31SStefan Hajnoczi * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState 26561007b31SStefan Hajnoczi * AioContext. 2667a63f3cdSStefan Hajnoczi * 2677a63f3cdSStefan Hajnoczi * Only this BlockDriverState's AioContext is run, so in-flight requests must 2687a63f3cdSStefan Hajnoczi * not depend on events in other AioContexts. In that case, use 2697a63f3cdSStefan Hajnoczi * bdrv_drain_all() instead. 27061007b31SStefan Hajnoczi */ 27161007b31SStefan Hajnoczi void bdrv_drain(BlockDriverState *bs) 27261007b31SStefan Hajnoczi { 273f406c03cSAlexander Yarygin bool busy = true; 274f406c03cSAlexander Yarygin 27567da1dc5SFam Zheng bdrv_drain_recurse(bs); 276f406c03cSAlexander Yarygin while (busy) { 27761007b31SStefan Hajnoczi /* Keep iterating */ 278f406c03cSAlexander Yarygin bdrv_flush_io_queue(bs); 279f406c03cSAlexander Yarygin busy = bdrv_requests_pending(bs); 280f406c03cSAlexander Yarygin busy |= aio_poll(bdrv_get_aio_context(bs), busy); 28161007b31SStefan Hajnoczi } 28261007b31SStefan Hajnoczi } 28361007b31SStefan Hajnoczi 28461007b31SStefan Hajnoczi /* 28561007b31SStefan Hajnoczi * Wait for pending requests to complete across all BlockDriverStates 28661007b31SStefan Hajnoczi * 28761007b31SStefan Hajnoczi * This function does not flush data to disk, use bdrv_flush_all() for that 28861007b31SStefan Hajnoczi * after calling this function. 28961007b31SStefan Hajnoczi */ 29061007b31SStefan Hajnoczi void bdrv_drain_all(void) 29161007b31SStefan Hajnoczi { 29261007b31SStefan Hajnoczi /* Always run first iteration so any pending completion BHs run */ 29361007b31SStefan Hajnoczi bool busy = true; 29461007b31SStefan Hajnoczi BlockDriverState *bs = NULL; 295f406c03cSAlexander Yarygin GSList *aio_ctxs = NULL, *ctx; 29661007b31SStefan Hajnoczi 29761007b31SStefan Hajnoczi while ((bs = bdrv_next(bs))) { 29861007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 29961007b31SStefan Hajnoczi 30061007b31SStefan Hajnoczi aio_context_acquire(aio_context); 30161007b31SStefan Hajnoczi if (bs->job) { 30261007b31SStefan Hajnoczi block_job_pause(bs->job); 30361007b31SStefan Hajnoczi } 30461007b31SStefan Hajnoczi aio_context_release(aio_context); 305f406c03cSAlexander Yarygin 306764ba3aeSAlberto Garcia if (!g_slist_find(aio_ctxs, aio_context)) { 307f406c03cSAlexander Yarygin aio_ctxs = g_slist_prepend(aio_ctxs, aio_context); 308f406c03cSAlexander Yarygin } 30961007b31SStefan Hajnoczi } 31061007b31SStefan Hajnoczi 3117a63f3cdSStefan Hajnoczi /* Note that completion of an asynchronous I/O operation can trigger any 3127a63f3cdSStefan Hajnoczi * number of other I/O operations on other devices---for example a 3137a63f3cdSStefan Hajnoczi * coroutine can submit an I/O request to another device in response to 3147a63f3cdSStefan Hajnoczi * request completion. Therefore we must keep looping until there was no 3157a63f3cdSStefan Hajnoczi * more activity rather than simply draining each device independently. 3167a63f3cdSStefan Hajnoczi */ 31761007b31SStefan Hajnoczi while (busy) { 31861007b31SStefan Hajnoczi busy = false; 319f406c03cSAlexander Yarygin 320f406c03cSAlexander Yarygin for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) { 321f406c03cSAlexander Yarygin AioContext *aio_context = ctx->data; 32261007b31SStefan Hajnoczi bs = NULL; 32361007b31SStefan Hajnoczi 32461007b31SStefan Hajnoczi aio_context_acquire(aio_context); 325f406c03cSAlexander Yarygin while ((bs = bdrv_next(bs))) { 326f406c03cSAlexander Yarygin if (aio_context == bdrv_get_aio_context(bs)) { 327f406c03cSAlexander Yarygin bdrv_flush_io_queue(bs); 328f406c03cSAlexander Yarygin if (bdrv_requests_pending(bs)) { 329f406c03cSAlexander Yarygin busy = true; 330f406c03cSAlexander Yarygin aio_poll(aio_context, busy); 331f406c03cSAlexander Yarygin } 332f406c03cSAlexander Yarygin } 333f406c03cSAlexander Yarygin } 334f406c03cSAlexander Yarygin busy |= aio_poll(aio_context, false); 33561007b31SStefan Hajnoczi aio_context_release(aio_context); 33661007b31SStefan Hajnoczi } 33761007b31SStefan Hajnoczi } 33861007b31SStefan Hajnoczi 33961007b31SStefan Hajnoczi bs = NULL; 34061007b31SStefan Hajnoczi while ((bs = bdrv_next(bs))) { 34161007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 34261007b31SStefan Hajnoczi 34361007b31SStefan Hajnoczi aio_context_acquire(aio_context); 34461007b31SStefan Hajnoczi if (bs->job) { 34561007b31SStefan Hajnoczi block_job_resume(bs->job); 34661007b31SStefan Hajnoczi } 34761007b31SStefan Hajnoczi aio_context_release(aio_context); 34861007b31SStefan Hajnoczi } 349f406c03cSAlexander Yarygin g_slist_free(aio_ctxs); 35061007b31SStefan Hajnoczi } 35161007b31SStefan Hajnoczi 35261007b31SStefan Hajnoczi /** 35361007b31SStefan Hajnoczi * Remove an active request from the tracked requests list 35461007b31SStefan Hajnoczi * 35561007b31SStefan Hajnoczi * This function should be called when a tracked request is completing. 35661007b31SStefan Hajnoczi */ 35761007b31SStefan Hajnoczi static void tracked_request_end(BdrvTrackedRequest *req) 35861007b31SStefan Hajnoczi { 35961007b31SStefan Hajnoczi if (req->serialising) { 36061007b31SStefan Hajnoczi req->bs->serialising_in_flight--; 36161007b31SStefan Hajnoczi } 36261007b31SStefan Hajnoczi 36361007b31SStefan Hajnoczi QLIST_REMOVE(req, list); 36461007b31SStefan Hajnoczi qemu_co_queue_restart_all(&req->wait_queue); 36561007b31SStefan Hajnoczi } 36661007b31SStefan Hajnoczi 36761007b31SStefan Hajnoczi /** 36861007b31SStefan Hajnoczi * Add an active request to the tracked requests list 36961007b31SStefan Hajnoczi */ 37061007b31SStefan Hajnoczi static void tracked_request_begin(BdrvTrackedRequest *req, 37161007b31SStefan Hajnoczi BlockDriverState *bs, 37261007b31SStefan Hajnoczi int64_t offset, 373ebde595cSFam Zheng unsigned int bytes, 374ebde595cSFam Zheng enum BdrvTrackedRequestType type) 37561007b31SStefan Hajnoczi { 37661007b31SStefan Hajnoczi *req = (BdrvTrackedRequest){ 37761007b31SStefan Hajnoczi .bs = bs, 37861007b31SStefan Hajnoczi .offset = offset, 37961007b31SStefan Hajnoczi .bytes = bytes, 380ebde595cSFam Zheng .type = type, 38161007b31SStefan Hajnoczi .co = qemu_coroutine_self(), 38261007b31SStefan Hajnoczi .serialising = false, 38361007b31SStefan Hajnoczi .overlap_offset = offset, 38461007b31SStefan Hajnoczi .overlap_bytes = bytes, 38561007b31SStefan Hajnoczi }; 38661007b31SStefan Hajnoczi 38761007b31SStefan Hajnoczi qemu_co_queue_init(&req->wait_queue); 38861007b31SStefan Hajnoczi 38961007b31SStefan Hajnoczi QLIST_INSERT_HEAD(&bs->tracked_requests, req, list); 39061007b31SStefan Hajnoczi } 39161007b31SStefan Hajnoczi 39261007b31SStefan Hajnoczi static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align) 39361007b31SStefan Hajnoczi { 39461007b31SStefan Hajnoczi int64_t overlap_offset = req->offset & ~(align - 1); 39561007b31SStefan Hajnoczi unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align) 39661007b31SStefan Hajnoczi - overlap_offset; 39761007b31SStefan Hajnoczi 39861007b31SStefan Hajnoczi if (!req->serialising) { 39961007b31SStefan Hajnoczi req->bs->serialising_in_flight++; 40061007b31SStefan Hajnoczi req->serialising = true; 40161007b31SStefan Hajnoczi } 40261007b31SStefan Hajnoczi 40361007b31SStefan Hajnoczi req->overlap_offset = MIN(req->overlap_offset, overlap_offset); 40461007b31SStefan Hajnoczi req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes); 40561007b31SStefan Hajnoczi } 40661007b31SStefan Hajnoczi 40761007b31SStefan Hajnoczi /** 40861007b31SStefan Hajnoczi * Round a region to cluster boundaries 40961007b31SStefan Hajnoczi */ 41061007b31SStefan Hajnoczi void bdrv_round_to_clusters(BlockDriverState *bs, 41161007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, 41261007b31SStefan Hajnoczi int64_t *cluster_sector_num, 41361007b31SStefan Hajnoczi int *cluster_nb_sectors) 41461007b31SStefan Hajnoczi { 41561007b31SStefan Hajnoczi BlockDriverInfo bdi; 41661007b31SStefan Hajnoczi 41761007b31SStefan Hajnoczi if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 41861007b31SStefan Hajnoczi *cluster_sector_num = sector_num; 41961007b31SStefan Hajnoczi *cluster_nb_sectors = nb_sectors; 42061007b31SStefan Hajnoczi } else { 42161007b31SStefan Hajnoczi int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE; 42261007b31SStefan Hajnoczi *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c); 42361007b31SStefan Hajnoczi *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num + 42461007b31SStefan Hajnoczi nb_sectors, c); 42561007b31SStefan Hajnoczi } 42661007b31SStefan Hajnoczi } 42761007b31SStefan Hajnoczi 42861007b31SStefan Hajnoczi static int bdrv_get_cluster_size(BlockDriverState *bs) 42961007b31SStefan Hajnoczi { 43061007b31SStefan Hajnoczi BlockDriverInfo bdi; 43161007b31SStefan Hajnoczi int ret; 43261007b31SStefan Hajnoczi 43361007b31SStefan Hajnoczi ret = bdrv_get_info(bs, &bdi); 43461007b31SStefan Hajnoczi if (ret < 0 || bdi.cluster_size == 0) { 43561007b31SStefan Hajnoczi return bs->request_alignment; 43661007b31SStefan Hajnoczi } else { 43761007b31SStefan Hajnoczi return bdi.cluster_size; 43861007b31SStefan Hajnoczi } 43961007b31SStefan Hajnoczi } 44061007b31SStefan Hajnoczi 44161007b31SStefan Hajnoczi static bool tracked_request_overlaps(BdrvTrackedRequest *req, 44261007b31SStefan Hajnoczi int64_t offset, unsigned int bytes) 44361007b31SStefan Hajnoczi { 44461007b31SStefan Hajnoczi /* aaaa bbbb */ 44561007b31SStefan Hajnoczi if (offset >= req->overlap_offset + req->overlap_bytes) { 44661007b31SStefan Hajnoczi return false; 44761007b31SStefan Hajnoczi } 44861007b31SStefan Hajnoczi /* bbbb aaaa */ 44961007b31SStefan Hajnoczi if (req->overlap_offset >= offset + bytes) { 45061007b31SStefan Hajnoczi return false; 45161007b31SStefan Hajnoczi } 45261007b31SStefan Hajnoczi return true; 45361007b31SStefan Hajnoczi } 45461007b31SStefan Hajnoczi 45561007b31SStefan Hajnoczi static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self) 45661007b31SStefan Hajnoczi { 45761007b31SStefan Hajnoczi BlockDriverState *bs = self->bs; 45861007b31SStefan Hajnoczi BdrvTrackedRequest *req; 45961007b31SStefan Hajnoczi bool retry; 46061007b31SStefan Hajnoczi bool waited = false; 46161007b31SStefan Hajnoczi 46261007b31SStefan Hajnoczi if (!bs->serialising_in_flight) { 46361007b31SStefan Hajnoczi return false; 46461007b31SStefan Hajnoczi } 46561007b31SStefan Hajnoczi 46661007b31SStefan Hajnoczi do { 46761007b31SStefan Hajnoczi retry = false; 46861007b31SStefan Hajnoczi QLIST_FOREACH(req, &bs->tracked_requests, list) { 46961007b31SStefan Hajnoczi if (req == self || (!req->serialising && !self->serialising)) { 47061007b31SStefan Hajnoczi continue; 47161007b31SStefan Hajnoczi } 47261007b31SStefan Hajnoczi if (tracked_request_overlaps(req, self->overlap_offset, 47361007b31SStefan Hajnoczi self->overlap_bytes)) 47461007b31SStefan Hajnoczi { 47561007b31SStefan Hajnoczi /* Hitting this means there was a reentrant request, for 47661007b31SStefan Hajnoczi * example, a block driver issuing nested requests. This must 47761007b31SStefan Hajnoczi * never happen since it means deadlock. 47861007b31SStefan Hajnoczi */ 47961007b31SStefan Hajnoczi assert(qemu_coroutine_self() != req->co); 48061007b31SStefan Hajnoczi 48161007b31SStefan Hajnoczi /* If the request is already (indirectly) waiting for us, or 48261007b31SStefan Hajnoczi * will wait for us as soon as it wakes up, then just go on 48361007b31SStefan Hajnoczi * (instead of producing a deadlock in the former case). */ 48461007b31SStefan Hajnoczi if (!req->waiting_for) { 48561007b31SStefan Hajnoczi self->waiting_for = req; 48661007b31SStefan Hajnoczi qemu_co_queue_wait(&req->wait_queue); 48761007b31SStefan Hajnoczi self->waiting_for = NULL; 48861007b31SStefan Hajnoczi retry = true; 48961007b31SStefan Hajnoczi waited = true; 49061007b31SStefan Hajnoczi break; 49161007b31SStefan Hajnoczi } 49261007b31SStefan Hajnoczi } 49361007b31SStefan Hajnoczi } 49461007b31SStefan Hajnoczi } while (retry); 49561007b31SStefan Hajnoczi 49661007b31SStefan Hajnoczi return waited; 49761007b31SStefan Hajnoczi } 49861007b31SStefan Hajnoczi 49961007b31SStefan Hajnoczi static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset, 50061007b31SStefan Hajnoczi size_t size) 50161007b31SStefan Hajnoczi { 50261007b31SStefan Hajnoczi if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) { 50361007b31SStefan Hajnoczi return -EIO; 50461007b31SStefan Hajnoczi } 50561007b31SStefan Hajnoczi 50661007b31SStefan Hajnoczi if (!bdrv_is_inserted(bs)) { 50761007b31SStefan Hajnoczi return -ENOMEDIUM; 50861007b31SStefan Hajnoczi } 50961007b31SStefan Hajnoczi 51061007b31SStefan Hajnoczi if (offset < 0) { 51161007b31SStefan Hajnoczi return -EIO; 51261007b31SStefan Hajnoczi } 51361007b31SStefan Hajnoczi 51461007b31SStefan Hajnoczi return 0; 51561007b31SStefan Hajnoczi } 51661007b31SStefan Hajnoczi 51761007b31SStefan Hajnoczi static int bdrv_check_request(BlockDriverState *bs, int64_t sector_num, 51861007b31SStefan Hajnoczi int nb_sectors) 51961007b31SStefan Hajnoczi { 52061007b31SStefan Hajnoczi if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 52161007b31SStefan Hajnoczi return -EIO; 52261007b31SStefan Hajnoczi } 52361007b31SStefan Hajnoczi 52461007b31SStefan Hajnoczi return bdrv_check_byte_request(bs, sector_num * BDRV_SECTOR_SIZE, 52561007b31SStefan Hajnoczi nb_sectors * BDRV_SECTOR_SIZE); 52661007b31SStefan Hajnoczi } 52761007b31SStefan Hajnoczi 52861007b31SStefan Hajnoczi typedef struct RwCo { 52961007b31SStefan Hajnoczi BlockDriverState *bs; 53061007b31SStefan Hajnoczi int64_t offset; 53161007b31SStefan Hajnoczi QEMUIOVector *qiov; 53261007b31SStefan Hajnoczi bool is_write; 53361007b31SStefan Hajnoczi int ret; 53461007b31SStefan Hajnoczi BdrvRequestFlags flags; 53561007b31SStefan Hajnoczi } RwCo; 53661007b31SStefan Hajnoczi 53761007b31SStefan Hajnoczi static void coroutine_fn bdrv_rw_co_entry(void *opaque) 53861007b31SStefan Hajnoczi { 53961007b31SStefan Hajnoczi RwCo *rwco = opaque; 54061007b31SStefan Hajnoczi 54161007b31SStefan Hajnoczi if (!rwco->is_write) { 54261007b31SStefan Hajnoczi rwco->ret = bdrv_co_do_preadv(rwco->bs, rwco->offset, 54361007b31SStefan Hajnoczi rwco->qiov->size, rwco->qiov, 54461007b31SStefan Hajnoczi rwco->flags); 54561007b31SStefan Hajnoczi } else { 54661007b31SStefan Hajnoczi rwco->ret = bdrv_co_do_pwritev(rwco->bs, rwco->offset, 54761007b31SStefan Hajnoczi rwco->qiov->size, rwco->qiov, 54861007b31SStefan Hajnoczi rwco->flags); 54961007b31SStefan Hajnoczi } 55061007b31SStefan Hajnoczi } 55161007b31SStefan Hajnoczi 55261007b31SStefan Hajnoczi /* 55361007b31SStefan Hajnoczi * Process a vectored synchronous request using coroutines 55461007b31SStefan Hajnoczi */ 55561007b31SStefan Hajnoczi static int bdrv_prwv_co(BlockDriverState *bs, int64_t offset, 55661007b31SStefan Hajnoczi QEMUIOVector *qiov, bool is_write, 55761007b31SStefan Hajnoczi BdrvRequestFlags flags) 55861007b31SStefan Hajnoczi { 55961007b31SStefan Hajnoczi Coroutine *co; 56061007b31SStefan Hajnoczi RwCo rwco = { 56161007b31SStefan Hajnoczi .bs = bs, 56261007b31SStefan Hajnoczi .offset = offset, 56361007b31SStefan Hajnoczi .qiov = qiov, 56461007b31SStefan Hajnoczi .is_write = is_write, 56561007b31SStefan Hajnoczi .ret = NOT_DONE, 56661007b31SStefan Hajnoczi .flags = flags, 56761007b31SStefan Hajnoczi }; 56861007b31SStefan Hajnoczi 56961007b31SStefan Hajnoczi /** 57061007b31SStefan Hajnoczi * In sync call context, when the vcpu is blocked, this throttling timer 57161007b31SStefan Hajnoczi * will not fire; so the I/O throttling function has to be disabled here 57261007b31SStefan Hajnoczi * if it has been enabled. 57361007b31SStefan Hajnoczi */ 57461007b31SStefan Hajnoczi if (bs->io_limits_enabled) { 57561007b31SStefan Hajnoczi fprintf(stderr, "Disabling I/O throttling on '%s' due " 57661007b31SStefan Hajnoczi "to synchronous I/O.\n", bdrv_get_device_name(bs)); 57761007b31SStefan Hajnoczi bdrv_io_limits_disable(bs); 57861007b31SStefan Hajnoczi } 57961007b31SStefan Hajnoczi 58061007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 58161007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 58261007b31SStefan Hajnoczi bdrv_rw_co_entry(&rwco); 58361007b31SStefan Hajnoczi } else { 58461007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 58561007b31SStefan Hajnoczi 58661007b31SStefan Hajnoczi co = qemu_coroutine_create(bdrv_rw_co_entry); 58761007b31SStefan Hajnoczi qemu_coroutine_enter(co, &rwco); 58861007b31SStefan Hajnoczi while (rwco.ret == NOT_DONE) { 58961007b31SStefan Hajnoczi aio_poll(aio_context, true); 59061007b31SStefan Hajnoczi } 59161007b31SStefan Hajnoczi } 59261007b31SStefan Hajnoczi return rwco.ret; 59361007b31SStefan Hajnoczi } 59461007b31SStefan Hajnoczi 59561007b31SStefan Hajnoczi /* 59661007b31SStefan Hajnoczi * Process a synchronous request using coroutines 59761007b31SStefan Hajnoczi */ 59861007b31SStefan Hajnoczi static int bdrv_rw_co(BlockDriverState *bs, int64_t sector_num, uint8_t *buf, 59961007b31SStefan Hajnoczi int nb_sectors, bool is_write, BdrvRequestFlags flags) 60061007b31SStefan Hajnoczi { 60161007b31SStefan Hajnoczi QEMUIOVector qiov; 60261007b31SStefan Hajnoczi struct iovec iov = { 60361007b31SStefan Hajnoczi .iov_base = (void *)buf, 60461007b31SStefan Hajnoczi .iov_len = nb_sectors * BDRV_SECTOR_SIZE, 60561007b31SStefan Hajnoczi }; 60661007b31SStefan Hajnoczi 60761007b31SStefan Hajnoczi if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 60861007b31SStefan Hajnoczi return -EINVAL; 60961007b31SStefan Hajnoczi } 61061007b31SStefan Hajnoczi 61161007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 61261007b31SStefan Hajnoczi return bdrv_prwv_co(bs, sector_num << BDRV_SECTOR_BITS, 61361007b31SStefan Hajnoczi &qiov, is_write, flags); 61461007b31SStefan Hajnoczi } 61561007b31SStefan Hajnoczi 61661007b31SStefan Hajnoczi /* return < 0 if error. See bdrv_write() for the return codes */ 61761007b31SStefan Hajnoczi int bdrv_read(BlockDriverState *bs, int64_t sector_num, 61861007b31SStefan Hajnoczi uint8_t *buf, int nb_sectors) 61961007b31SStefan Hajnoczi { 62061007b31SStefan Hajnoczi return bdrv_rw_co(bs, sector_num, buf, nb_sectors, false, 0); 62161007b31SStefan Hajnoczi } 62261007b31SStefan Hajnoczi 62361007b31SStefan Hajnoczi /* Just like bdrv_read(), but with I/O throttling temporarily disabled */ 62461007b31SStefan Hajnoczi int bdrv_read_unthrottled(BlockDriverState *bs, int64_t sector_num, 62561007b31SStefan Hajnoczi uint8_t *buf, int nb_sectors) 62661007b31SStefan Hajnoczi { 62761007b31SStefan Hajnoczi bool enabled; 62861007b31SStefan Hajnoczi int ret; 62961007b31SStefan Hajnoczi 63061007b31SStefan Hajnoczi enabled = bs->io_limits_enabled; 63161007b31SStefan Hajnoczi bs->io_limits_enabled = false; 63261007b31SStefan Hajnoczi ret = bdrv_read(bs, sector_num, buf, nb_sectors); 63361007b31SStefan Hajnoczi bs->io_limits_enabled = enabled; 63461007b31SStefan Hajnoczi return ret; 63561007b31SStefan Hajnoczi } 63661007b31SStefan Hajnoczi 63761007b31SStefan Hajnoczi /* Return < 0 if error. Important errors are: 63861007b31SStefan Hajnoczi -EIO generic I/O error (may happen for all errors) 63961007b31SStefan Hajnoczi -ENOMEDIUM No media inserted. 64061007b31SStefan Hajnoczi -EINVAL Invalid sector number or nb_sectors 64161007b31SStefan Hajnoczi -EACCES Trying to write a read-only device 64261007b31SStefan Hajnoczi */ 64361007b31SStefan Hajnoczi int bdrv_write(BlockDriverState *bs, int64_t sector_num, 64461007b31SStefan Hajnoczi const uint8_t *buf, int nb_sectors) 64561007b31SStefan Hajnoczi { 64661007b31SStefan Hajnoczi return bdrv_rw_co(bs, sector_num, (uint8_t *)buf, nb_sectors, true, 0); 64761007b31SStefan Hajnoczi } 64861007b31SStefan Hajnoczi 64961007b31SStefan Hajnoczi int bdrv_write_zeroes(BlockDriverState *bs, int64_t sector_num, 65061007b31SStefan Hajnoczi int nb_sectors, BdrvRequestFlags flags) 65161007b31SStefan Hajnoczi { 65261007b31SStefan Hajnoczi return bdrv_rw_co(bs, sector_num, NULL, nb_sectors, true, 65361007b31SStefan Hajnoczi BDRV_REQ_ZERO_WRITE | flags); 65461007b31SStefan Hajnoczi } 65561007b31SStefan Hajnoczi 65661007b31SStefan Hajnoczi /* 65761007b31SStefan Hajnoczi * Completely zero out a block device with the help of bdrv_write_zeroes. 65861007b31SStefan Hajnoczi * The operation is sped up by checking the block status and only writing 65961007b31SStefan Hajnoczi * zeroes to the device if they currently do not return zeroes. Optional 66061007b31SStefan Hajnoczi * flags are passed through to bdrv_write_zeroes (e.g. BDRV_REQ_MAY_UNMAP). 66161007b31SStefan Hajnoczi * 66261007b31SStefan Hajnoczi * Returns < 0 on error, 0 on success. For error codes see bdrv_write(). 66361007b31SStefan Hajnoczi */ 66461007b31SStefan Hajnoczi int bdrv_make_zero(BlockDriverState *bs, BdrvRequestFlags flags) 66561007b31SStefan Hajnoczi { 66661007b31SStefan Hajnoczi int64_t target_sectors, ret, nb_sectors, sector_num = 0; 66761007b31SStefan Hajnoczi int n; 66861007b31SStefan Hajnoczi 66961007b31SStefan Hajnoczi target_sectors = bdrv_nb_sectors(bs); 67061007b31SStefan Hajnoczi if (target_sectors < 0) { 67161007b31SStefan Hajnoczi return target_sectors; 67261007b31SStefan Hajnoczi } 67361007b31SStefan Hajnoczi 67461007b31SStefan Hajnoczi for (;;) { 67561007b31SStefan Hajnoczi nb_sectors = MIN(target_sectors - sector_num, BDRV_REQUEST_MAX_SECTORS); 67661007b31SStefan Hajnoczi if (nb_sectors <= 0) { 67761007b31SStefan Hajnoczi return 0; 67861007b31SStefan Hajnoczi } 67961007b31SStefan Hajnoczi ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n); 68061007b31SStefan Hajnoczi if (ret < 0) { 68161007b31SStefan Hajnoczi error_report("error getting block status at sector %" PRId64 ": %s", 68261007b31SStefan Hajnoczi sector_num, strerror(-ret)); 68361007b31SStefan Hajnoczi return ret; 68461007b31SStefan Hajnoczi } 68561007b31SStefan Hajnoczi if (ret & BDRV_BLOCK_ZERO) { 68661007b31SStefan Hajnoczi sector_num += n; 68761007b31SStefan Hajnoczi continue; 68861007b31SStefan Hajnoczi } 68961007b31SStefan Hajnoczi ret = bdrv_write_zeroes(bs, sector_num, n, flags); 69061007b31SStefan Hajnoczi if (ret < 0) { 69161007b31SStefan Hajnoczi error_report("error writing zeroes at sector %" PRId64 ": %s", 69261007b31SStefan Hajnoczi sector_num, strerror(-ret)); 69361007b31SStefan Hajnoczi return ret; 69461007b31SStefan Hajnoczi } 69561007b31SStefan Hajnoczi sector_num += n; 69661007b31SStefan Hajnoczi } 69761007b31SStefan Hajnoczi } 69861007b31SStefan Hajnoczi 69961007b31SStefan Hajnoczi int bdrv_pread(BlockDriverState *bs, int64_t offset, void *buf, int bytes) 70061007b31SStefan Hajnoczi { 70161007b31SStefan Hajnoczi QEMUIOVector qiov; 70261007b31SStefan Hajnoczi struct iovec iov = { 70361007b31SStefan Hajnoczi .iov_base = (void *)buf, 70461007b31SStefan Hajnoczi .iov_len = bytes, 70561007b31SStefan Hajnoczi }; 70661007b31SStefan Hajnoczi int ret; 70761007b31SStefan Hajnoczi 70861007b31SStefan Hajnoczi if (bytes < 0) { 70961007b31SStefan Hajnoczi return -EINVAL; 71061007b31SStefan Hajnoczi } 71161007b31SStefan Hajnoczi 71261007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 71361007b31SStefan Hajnoczi ret = bdrv_prwv_co(bs, offset, &qiov, false, 0); 71461007b31SStefan Hajnoczi if (ret < 0) { 71561007b31SStefan Hajnoczi return ret; 71661007b31SStefan Hajnoczi } 71761007b31SStefan Hajnoczi 71861007b31SStefan Hajnoczi return bytes; 71961007b31SStefan Hajnoczi } 72061007b31SStefan Hajnoczi 72161007b31SStefan Hajnoczi int bdrv_pwritev(BlockDriverState *bs, int64_t offset, QEMUIOVector *qiov) 72261007b31SStefan Hajnoczi { 72361007b31SStefan Hajnoczi int ret; 72461007b31SStefan Hajnoczi 72561007b31SStefan Hajnoczi ret = bdrv_prwv_co(bs, offset, qiov, true, 0); 72661007b31SStefan Hajnoczi if (ret < 0) { 72761007b31SStefan Hajnoczi return ret; 72861007b31SStefan Hajnoczi } 72961007b31SStefan Hajnoczi 73061007b31SStefan Hajnoczi return qiov->size; 73161007b31SStefan Hajnoczi } 73261007b31SStefan Hajnoczi 73361007b31SStefan Hajnoczi int bdrv_pwrite(BlockDriverState *bs, int64_t offset, 73461007b31SStefan Hajnoczi const void *buf, int bytes) 73561007b31SStefan Hajnoczi { 73661007b31SStefan Hajnoczi QEMUIOVector qiov; 73761007b31SStefan Hajnoczi struct iovec iov = { 73861007b31SStefan Hajnoczi .iov_base = (void *) buf, 73961007b31SStefan Hajnoczi .iov_len = bytes, 74061007b31SStefan Hajnoczi }; 74161007b31SStefan Hajnoczi 74261007b31SStefan Hajnoczi if (bytes < 0) { 74361007b31SStefan Hajnoczi return -EINVAL; 74461007b31SStefan Hajnoczi } 74561007b31SStefan Hajnoczi 74661007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 74761007b31SStefan Hajnoczi return bdrv_pwritev(bs, offset, &qiov); 74861007b31SStefan Hajnoczi } 74961007b31SStefan Hajnoczi 75061007b31SStefan Hajnoczi /* 75161007b31SStefan Hajnoczi * Writes to the file and ensures that no writes are reordered across this 75261007b31SStefan Hajnoczi * request (acts as a barrier) 75361007b31SStefan Hajnoczi * 75461007b31SStefan Hajnoczi * Returns 0 on success, -errno in error cases. 75561007b31SStefan Hajnoczi */ 75661007b31SStefan Hajnoczi int bdrv_pwrite_sync(BlockDriverState *bs, int64_t offset, 75761007b31SStefan Hajnoczi const void *buf, int count) 75861007b31SStefan Hajnoczi { 75961007b31SStefan Hajnoczi int ret; 76061007b31SStefan Hajnoczi 76161007b31SStefan Hajnoczi ret = bdrv_pwrite(bs, offset, buf, count); 76261007b31SStefan Hajnoczi if (ret < 0) { 76361007b31SStefan Hajnoczi return ret; 76461007b31SStefan Hajnoczi } 76561007b31SStefan Hajnoczi 76661007b31SStefan Hajnoczi /* No flush needed for cache modes that already do it */ 76761007b31SStefan Hajnoczi if (bs->enable_write_cache) { 76861007b31SStefan Hajnoczi bdrv_flush(bs); 76961007b31SStefan Hajnoczi } 77061007b31SStefan Hajnoczi 77161007b31SStefan Hajnoczi return 0; 77261007b31SStefan Hajnoczi } 77361007b31SStefan Hajnoczi 77461007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs, 77561007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, QEMUIOVector *qiov) 77661007b31SStefan Hajnoczi { 77761007b31SStefan Hajnoczi /* Perform I/O through a temporary buffer so that users who scribble over 77861007b31SStefan Hajnoczi * their read buffer while the operation is in progress do not end up 77961007b31SStefan Hajnoczi * modifying the image file. This is critical for zero-copy guest I/O 78061007b31SStefan Hajnoczi * where anything might happen inside guest memory. 78161007b31SStefan Hajnoczi */ 78261007b31SStefan Hajnoczi void *bounce_buffer; 78361007b31SStefan Hajnoczi 78461007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 78561007b31SStefan Hajnoczi struct iovec iov; 78661007b31SStefan Hajnoczi QEMUIOVector bounce_qiov; 78761007b31SStefan Hajnoczi int64_t cluster_sector_num; 78861007b31SStefan Hajnoczi int cluster_nb_sectors; 78961007b31SStefan Hajnoczi size_t skip_bytes; 79061007b31SStefan Hajnoczi int ret; 79161007b31SStefan Hajnoczi 79261007b31SStefan Hajnoczi /* Cover entire cluster so no additional backing file I/O is required when 79361007b31SStefan Hajnoczi * allocating cluster in the image file. 79461007b31SStefan Hajnoczi */ 79561007b31SStefan Hajnoczi bdrv_round_to_clusters(bs, sector_num, nb_sectors, 79661007b31SStefan Hajnoczi &cluster_sector_num, &cluster_nb_sectors); 79761007b31SStefan Hajnoczi 79861007b31SStefan Hajnoczi trace_bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors, 79961007b31SStefan Hajnoczi cluster_sector_num, cluster_nb_sectors); 80061007b31SStefan Hajnoczi 80161007b31SStefan Hajnoczi iov.iov_len = cluster_nb_sectors * BDRV_SECTOR_SIZE; 80261007b31SStefan Hajnoczi iov.iov_base = bounce_buffer = qemu_try_blockalign(bs, iov.iov_len); 80361007b31SStefan Hajnoczi if (bounce_buffer == NULL) { 80461007b31SStefan Hajnoczi ret = -ENOMEM; 80561007b31SStefan Hajnoczi goto err; 80661007b31SStefan Hajnoczi } 80761007b31SStefan Hajnoczi 80861007b31SStefan Hajnoczi qemu_iovec_init_external(&bounce_qiov, &iov, 1); 80961007b31SStefan Hajnoczi 81061007b31SStefan Hajnoczi ret = drv->bdrv_co_readv(bs, cluster_sector_num, cluster_nb_sectors, 81161007b31SStefan Hajnoczi &bounce_qiov); 81261007b31SStefan Hajnoczi if (ret < 0) { 81361007b31SStefan Hajnoczi goto err; 81461007b31SStefan Hajnoczi } 81561007b31SStefan Hajnoczi 81661007b31SStefan Hajnoczi if (drv->bdrv_co_write_zeroes && 81761007b31SStefan Hajnoczi buffer_is_zero(bounce_buffer, iov.iov_len)) { 81861007b31SStefan Hajnoczi ret = bdrv_co_do_write_zeroes(bs, cluster_sector_num, 81961007b31SStefan Hajnoczi cluster_nb_sectors, 0); 82061007b31SStefan Hajnoczi } else { 82161007b31SStefan Hajnoczi /* This does not change the data on the disk, it is not necessary 82261007b31SStefan Hajnoczi * to flush even in cache=writethrough mode. 82361007b31SStefan Hajnoczi */ 82461007b31SStefan Hajnoczi ret = drv->bdrv_co_writev(bs, cluster_sector_num, cluster_nb_sectors, 82561007b31SStefan Hajnoczi &bounce_qiov); 82661007b31SStefan Hajnoczi } 82761007b31SStefan Hajnoczi 82861007b31SStefan Hajnoczi if (ret < 0) { 82961007b31SStefan Hajnoczi /* It might be okay to ignore write errors for guest requests. If this 83061007b31SStefan Hajnoczi * is a deliberate copy-on-read then we don't want to ignore the error. 83161007b31SStefan Hajnoczi * Simply report it in all cases. 83261007b31SStefan Hajnoczi */ 83361007b31SStefan Hajnoczi goto err; 83461007b31SStefan Hajnoczi } 83561007b31SStefan Hajnoczi 83661007b31SStefan Hajnoczi skip_bytes = (sector_num - cluster_sector_num) * BDRV_SECTOR_SIZE; 83761007b31SStefan Hajnoczi qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes, 83861007b31SStefan Hajnoczi nb_sectors * BDRV_SECTOR_SIZE); 83961007b31SStefan Hajnoczi 84061007b31SStefan Hajnoczi err: 84161007b31SStefan Hajnoczi qemu_vfree(bounce_buffer); 84261007b31SStefan Hajnoczi return ret; 84361007b31SStefan Hajnoczi } 84461007b31SStefan Hajnoczi 84561007b31SStefan Hajnoczi /* 84661007b31SStefan Hajnoczi * Forwards an already correctly aligned request to the BlockDriver. This 84761007b31SStefan Hajnoczi * handles copy on read and zeroing after EOF; any other features must be 84861007b31SStefan Hajnoczi * implemented by the caller. 84961007b31SStefan Hajnoczi */ 85061007b31SStefan Hajnoczi static int coroutine_fn bdrv_aligned_preadv(BlockDriverState *bs, 85161007b31SStefan Hajnoczi BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 85261007b31SStefan Hajnoczi int64_t align, QEMUIOVector *qiov, int flags) 85361007b31SStefan Hajnoczi { 85461007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 85561007b31SStefan Hajnoczi int ret; 85661007b31SStefan Hajnoczi 85761007b31SStefan Hajnoczi int64_t sector_num = offset >> BDRV_SECTOR_BITS; 85861007b31SStefan Hajnoczi unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS; 85961007b31SStefan Hajnoczi 86061007b31SStefan Hajnoczi assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 86161007b31SStefan Hajnoczi assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 86261007b31SStefan Hajnoczi assert(!qiov || bytes == qiov->size); 86361007b31SStefan Hajnoczi 86461007b31SStefan Hajnoczi /* Handle Copy on Read and associated serialisation */ 86561007b31SStefan Hajnoczi if (flags & BDRV_REQ_COPY_ON_READ) { 86661007b31SStefan Hajnoczi /* If we touch the same cluster it counts as an overlap. This 86761007b31SStefan Hajnoczi * guarantees that allocating writes will be serialized and not race 86861007b31SStefan Hajnoczi * with each other for the same cluster. For example, in copy-on-read 86961007b31SStefan Hajnoczi * it ensures that the CoR read and write operations are atomic and 87061007b31SStefan Hajnoczi * guest writes cannot interleave between them. */ 87161007b31SStefan Hajnoczi mark_request_serialising(req, bdrv_get_cluster_size(bs)); 87261007b31SStefan Hajnoczi } 87361007b31SStefan Hajnoczi 87461408b25SFam Zheng if (!(flags & BDRV_REQ_NO_SERIALISING)) { 87561007b31SStefan Hajnoczi wait_serialising_requests(req); 87661408b25SFam Zheng } 87761007b31SStefan Hajnoczi 87861007b31SStefan Hajnoczi if (flags & BDRV_REQ_COPY_ON_READ) { 87961007b31SStefan Hajnoczi int pnum; 88061007b31SStefan Hajnoczi 88161007b31SStefan Hajnoczi ret = bdrv_is_allocated(bs, sector_num, nb_sectors, &pnum); 88261007b31SStefan Hajnoczi if (ret < 0) { 88361007b31SStefan Hajnoczi goto out; 88461007b31SStefan Hajnoczi } 88561007b31SStefan Hajnoczi 88661007b31SStefan Hajnoczi if (!ret || pnum != nb_sectors) { 88761007b31SStefan Hajnoczi ret = bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors, qiov); 88861007b31SStefan Hajnoczi goto out; 88961007b31SStefan Hajnoczi } 89061007b31SStefan Hajnoczi } 89161007b31SStefan Hajnoczi 89261007b31SStefan Hajnoczi /* Forward the request to the BlockDriver */ 89361007b31SStefan Hajnoczi if (!bs->zero_beyond_eof) { 89461007b31SStefan Hajnoczi ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov); 89561007b31SStefan Hajnoczi } else { 89661007b31SStefan Hajnoczi /* Read zeros after EOF */ 89761007b31SStefan Hajnoczi int64_t total_sectors, max_nb_sectors; 89861007b31SStefan Hajnoczi 89961007b31SStefan Hajnoczi total_sectors = bdrv_nb_sectors(bs); 90061007b31SStefan Hajnoczi if (total_sectors < 0) { 90161007b31SStefan Hajnoczi ret = total_sectors; 90261007b31SStefan Hajnoczi goto out; 90361007b31SStefan Hajnoczi } 90461007b31SStefan Hajnoczi 90561007b31SStefan Hajnoczi max_nb_sectors = ROUND_UP(MAX(0, total_sectors - sector_num), 90661007b31SStefan Hajnoczi align >> BDRV_SECTOR_BITS); 90761007b31SStefan Hajnoczi if (nb_sectors < max_nb_sectors) { 90861007b31SStefan Hajnoczi ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov); 90961007b31SStefan Hajnoczi } else if (max_nb_sectors > 0) { 91061007b31SStefan Hajnoczi QEMUIOVector local_qiov; 91161007b31SStefan Hajnoczi 91261007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov); 91361007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, 91461007b31SStefan Hajnoczi max_nb_sectors * BDRV_SECTOR_SIZE); 91561007b31SStefan Hajnoczi 91661007b31SStefan Hajnoczi ret = drv->bdrv_co_readv(bs, sector_num, max_nb_sectors, 91761007b31SStefan Hajnoczi &local_qiov); 91861007b31SStefan Hajnoczi 91961007b31SStefan Hajnoczi qemu_iovec_destroy(&local_qiov); 92061007b31SStefan Hajnoczi } else { 92161007b31SStefan Hajnoczi ret = 0; 92261007b31SStefan Hajnoczi } 92361007b31SStefan Hajnoczi 92461007b31SStefan Hajnoczi /* Reading beyond end of file is supposed to produce zeroes */ 92561007b31SStefan Hajnoczi if (ret == 0 && total_sectors < sector_num + nb_sectors) { 92661007b31SStefan Hajnoczi uint64_t offset = MAX(0, total_sectors - sector_num); 92761007b31SStefan Hajnoczi uint64_t bytes = (sector_num + nb_sectors - offset) * 92861007b31SStefan Hajnoczi BDRV_SECTOR_SIZE; 92961007b31SStefan Hajnoczi qemu_iovec_memset(qiov, offset * BDRV_SECTOR_SIZE, 0, bytes); 93061007b31SStefan Hajnoczi } 93161007b31SStefan Hajnoczi } 93261007b31SStefan Hajnoczi 93361007b31SStefan Hajnoczi out: 93461007b31SStefan Hajnoczi return ret; 93561007b31SStefan Hajnoczi } 93661007b31SStefan Hajnoczi 93761007b31SStefan Hajnoczi /* 93861007b31SStefan Hajnoczi * Handle a read request in coroutine context 93961007b31SStefan Hajnoczi */ 94061007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_do_preadv(BlockDriverState *bs, 94161007b31SStefan Hajnoczi int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 94261007b31SStefan Hajnoczi BdrvRequestFlags flags) 94361007b31SStefan Hajnoczi { 94461007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 94561007b31SStefan Hajnoczi BdrvTrackedRequest req; 94661007b31SStefan Hajnoczi 947d01c07f2SFam Zheng /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */ 948d01c07f2SFam Zheng uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment); 94961007b31SStefan Hajnoczi uint8_t *head_buf = NULL; 95061007b31SStefan Hajnoczi uint8_t *tail_buf = NULL; 95161007b31SStefan Hajnoczi QEMUIOVector local_qiov; 95261007b31SStefan Hajnoczi bool use_local_qiov = false; 95361007b31SStefan Hajnoczi int ret; 95461007b31SStefan Hajnoczi 95561007b31SStefan Hajnoczi if (!drv) { 95661007b31SStefan Hajnoczi return -ENOMEDIUM; 95761007b31SStefan Hajnoczi } 95861007b31SStefan Hajnoczi 95961007b31SStefan Hajnoczi ret = bdrv_check_byte_request(bs, offset, bytes); 96061007b31SStefan Hajnoczi if (ret < 0) { 96161007b31SStefan Hajnoczi return ret; 96261007b31SStefan Hajnoczi } 96361007b31SStefan Hajnoczi 9649568b511SWen Congyang /* Don't do copy-on-read if we read data before write operation */ 96561408b25SFam Zheng if (bs->copy_on_read && !(flags & BDRV_REQ_NO_SERIALISING)) { 96661007b31SStefan Hajnoczi flags |= BDRV_REQ_COPY_ON_READ; 96761007b31SStefan Hajnoczi } 96861007b31SStefan Hajnoczi 96961007b31SStefan Hajnoczi /* throttling disk I/O */ 97061007b31SStefan Hajnoczi if (bs->io_limits_enabled) { 97176f4afb4SAlberto Garcia throttle_group_co_io_limits_intercept(bs, bytes, false); 97261007b31SStefan Hajnoczi } 97361007b31SStefan Hajnoczi 97461007b31SStefan Hajnoczi /* Align read if necessary by padding qiov */ 97561007b31SStefan Hajnoczi if (offset & (align - 1)) { 97661007b31SStefan Hajnoczi head_buf = qemu_blockalign(bs, align); 97761007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 2); 97861007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 97961007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 98061007b31SStefan Hajnoczi use_local_qiov = true; 98161007b31SStefan Hajnoczi 98261007b31SStefan Hajnoczi bytes += offset & (align - 1); 98361007b31SStefan Hajnoczi offset = offset & ~(align - 1); 98461007b31SStefan Hajnoczi } 98561007b31SStefan Hajnoczi 98661007b31SStefan Hajnoczi if ((offset + bytes) & (align - 1)) { 98761007b31SStefan Hajnoczi if (!use_local_qiov) { 98861007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 1); 98961007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 99061007b31SStefan Hajnoczi use_local_qiov = true; 99161007b31SStefan Hajnoczi } 99261007b31SStefan Hajnoczi tail_buf = qemu_blockalign(bs, align); 99361007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, tail_buf, 99461007b31SStefan Hajnoczi align - ((offset + bytes) & (align - 1))); 99561007b31SStefan Hajnoczi 99661007b31SStefan Hajnoczi bytes = ROUND_UP(bytes, align); 99761007b31SStefan Hajnoczi } 99861007b31SStefan Hajnoczi 999ebde595cSFam Zheng tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ); 100061007b31SStefan Hajnoczi ret = bdrv_aligned_preadv(bs, &req, offset, bytes, align, 100161007b31SStefan Hajnoczi use_local_qiov ? &local_qiov : qiov, 100261007b31SStefan Hajnoczi flags); 100361007b31SStefan Hajnoczi tracked_request_end(&req); 100461007b31SStefan Hajnoczi 100561007b31SStefan Hajnoczi if (use_local_qiov) { 100661007b31SStefan Hajnoczi qemu_iovec_destroy(&local_qiov); 100761007b31SStefan Hajnoczi qemu_vfree(head_buf); 100861007b31SStefan Hajnoczi qemu_vfree(tail_buf); 100961007b31SStefan Hajnoczi } 101061007b31SStefan Hajnoczi 101161007b31SStefan Hajnoczi return ret; 101261007b31SStefan Hajnoczi } 101361007b31SStefan Hajnoczi 101461007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_do_readv(BlockDriverState *bs, 101561007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, 101661007b31SStefan Hajnoczi BdrvRequestFlags flags) 101761007b31SStefan Hajnoczi { 101861007b31SStefan Hajnoczi if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 101961007b31SStefan Hajnoczi return -EINVAL; 102061007b31SStefan Hajnoczi } 102161007b31SStefan Hajnoczi 102261007b31SStefan Hajnoczi return bdrv_co_do_preadv(bs, sector_num << BDRV_SECTOR_BITS, 102361007b31SStefan Hajnoczi nb_sectors << BDRV_SECTOR_BITS, qiov, flags); 102461007b31SStefan Hajnoczi } 102561007b31SStefan Hajnoczi 102661007b31SStefan Hajnoczi int coroutine_fn bdrv_co_readv(BlockDriverState *bs, int64_t sector_num, 102761007b31SStefan Hajnoczi int nb_sectors, QEMUIOVector *qiov) 102861007b31SStefan Hajnoczi { 102961007b31SStefan Hajnoczi trace_bdrv_co_readv(bs, sector_num, nb_sectors); 103061007b31SStefan Hajnoczi 103161007b31SStefan Hajnoczi return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 0); 103261007b31SStefan Hajnoczi } 103361007b31SStefan Hajnoczi 103461408b25SFam Zheng int coroutine_fn bdrv_co_readv_no_serialising(BlockDriverState *bs, 10359568b511SWen Congyang int64_t sector_num, int nb_sectors, QEMUIOVector *qiov) 10369568b511SWen Congyang { 103761408b25SFam Zheng trace_bdrv_co_readv_no_serialising(bs, sector_num, nb_sectors); 10389568b511SWen Congyang 10399568b511SWen Congyang return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 104061408b25SFam Zheng BDRV_REQ_NO_SERIALISING); 10419568b511SWen Congyang } 10429568b511SWen Congyang 104361007b31SStefan Hajnoczi int coroutine_fn bdrv_co_copy_on_readv(BlockDriverState *bs, 104461007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, QEMUIOVector *qiov) 104561007b31SStefan Hajnoczi { 104661007b31SStefan Hajnoczi trace_bdrv_co_copy_on_readv(bs, sector_num, nb_sectors); 104761007b31SStefan Hajnoczi 104861007b31SStefan Hajnoczi return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 104961007b31SStefan Hajnoczi BDRV_REQ_COPY_ON_READ); 105061007b31SStefan Hajnoczi } 105161007b31SStefan Hajnoczi 105261007b31SStefan Hajnoczi #define MAX_WRITE_ZEROES_BOUNCE_BUFFER 32768 105361007b31SStefan Hajnoczi 105461007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs, 105561007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, BdrvRequestFlags flags) 105661007b31SStefan Hajnoczi { 105761007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 105861007b31SStefan Hajnoczi QEMUIOVector qiov; 105961007b31SStefan Hajnoczi struct iovec iov = {0}; 106061007b31SStefan Hajnoczi int ret = 0; 106161007b31SStefan Hajnoczi 106261007b31SStefan Hajnoczi int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_write_zeroes, 106361007b31SStefan Hajnoczi BDRV_REQUEST_MAX_SECTORS); 106461007b31SStefan Hajnoczi 106561007b31SStefan Hajnoczi while (nb_sectors > 0 && !ret) { 106661007b31SStefan Hajnoczi int num = nb_sectors; 106761007b31SStefan Hajnoczi 106861007b31SStefan Hajnoczi /* Align request. Block drivers can expect the "bulk" of the request 106961007b31SStefan Hajnoczi * to be aligned. 107061007b31SStefan Hajnoczi */ 107161007b31SStefan Hajnoczi if (bs->bl.write_zeroes_alignment 107261007b31SStefan Hajnoczi && num > bs->bl.write_zeroes_alignment) { 107361007b31SStefan Hajnoczi if (sector_num % bs->bl.write_zeroes_alignment != 0) { 107461007b31SStefan Hajnoczi /* Make a small request up to the first aligned sector. */ 107561007b31SStefan Hajnoczi num = bs->bl.write_zeroes_alignment; 107661007b31SStefan Hajnoczi num -= sector_num % bs->bl.write_zeroes_alignment; 107761007b31SStefan Hajnoczi } else if ((sector_num + num) % bs->bl.write_zeroes_alignment != 0) { 107861007b31SStefan Hajnoczi /* Shorten the request to the last aligned sector. num cannot 107961007b31SStefan Hajnoczi * underflow because num > bs->bl.write_zeroes_alignment. 108061007b31SStefan Hajnoczi */ 108161007b31SStefan Hajnoczi num -= (sector_num + num) % bs->bl.write_zeroes_alignment; 108261007b31SStefan Hajnoczi } 108361007b31SStefan Hajnoczi } 108461007b31SStefan Hajnoczi 108561007b31SStefan Hajnoczi /* limit request size */ 108661007b31SStefan Hajnoczi if (num > max_write_zeroes) { 108761007b31SStefan Hajnoczi num = max_write_zeroes; 108861007b31SStefan Hajnoczi } 108961007b31SStefan Hajnoczi 109061007b31SStefan Hajnoczi ret = -ENOTSUP; 109161007b31SStefan Hajnoczi /* First try the efficient write zeroes operation */ 109261007b31SStefan Hajnoczi if (drv->bdrv_co_write_zeroes) { 109361007b31SStefan Hajnoczi ret = drv->bdrv_co_write_zeroes(bs, sector_num, num, flags); 109461007b31SStefan Hajnoczi } 109561007b31SStefan Hajnoczi 109661007b31SStefan Hajnoczi if (ret == -ENOTSUP) { 109761007b31SStefan Hajnoczi /* Fall back to bounce buffer if write zeroes is unsupported */ 109861007b31SStefan Hajnoczi int max_xfer_len = MIN_NON_ZERO(bs->bl.max_transfer_length, 109961007b31SStefan Hajnoczi MAX_WRITE_ZEROES_BOUNCE_BUFFER); 110061007b31SStefan Hajnoczi num = MIN(num, max_xfer_len); 110161007b31SStefan Hajnoczi iov.iov_len = num * BDRV_SECTOR_SIZE; 110261007b31SStefan Hajnoczi if (iov.iov_base == NULL) { 110361007b31SStefan Hajnoczi iov.iov_base = qemu_try_blockalign(bs, num * BDRV_SECTOR_SIZE); 110461007b31SStefan Hajnoczi if (iov.iov_base == NULL) { 110561007b31SStefan Hajnoczi ret = -ENOMEM; 110661007b31SStefan Hajnoczi goto fail; 110761007b31SStefan Hajnoczi } 110861007b31SStefan Hajnoczi memset(iov.iov_base, 0, num * BDRV_SECTOR_SIZE); 110961007b31SStefan Hajnoczi } 111061007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 111161007b31SStefan Hajnoczi 111261007b31SStefan Hajnoczi ret = drv->bdrv_co_writev(bs, sector_num, num, &qiov); 111361007b31SStefan Hajnoczi 111461007b31SStefan Hajnoczi /* Keep bounce buffer around if it is big enough for all 111561007b31SStefan Hajnoczi * all future requests. 111661007b31SStefan Hajnoczi */ 111761007b31SStefan Hajnoczi if (num < max_xfer_len) { 111861007b31SStefan Hajnoczi qemu_vfree(iov.iov_base); 111961007b31SStefan Hajnoczi iov.iov_base = NULL; 112061007b31SStefan Hajnoczi } 112161007b31SStefan Hajnoczi } 112261007b31SStefan Hajnoczi 112361007b31SStefan Hajnoczi sector_num += num; 112461007b31SStefan Hajnoczi nb_sectors -= num; 112561007b31SStefan Hajnoczi } 112661007b31SStefan Hajnoczi 112761007b31SStefan Hajnoczi fail: 112861007b31SStefan Hajnoczi qemu_vfree(iov.iov_base); 112961007b31SStefan Hajnoczi return ret; 113061007b31SStefan Hajnoczi } 113161007b31SStefan Hajnoczi 113261007b31SStefan Hajnoczi /* 113361007b31SStefan Hajnoczi * Forwards an already correctly aligned write request to the BlockDriver. 113461007b31SStefan Hajnoczi */ 113561007b31SStefan Hajnoczi static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs, 113661007b31SStefan Hajnoczi BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 113761007b31SStefan Hajnoczi QEMUIOVector *qiov, int flags) 113861007b31SStefan Hajnoczi { 113961007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 114061007b31SStefan Hajnoczi bool waited; 114161007b31SStefan Hajnoczi int ret; 114261007b31SStefan Hajnoczi 114361007b31SStefan Hajnoczi int64_t sector_num = offset >> BDRV_SECTOR_BITS; 114461007b31SStefan Hajnoczi unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS; 114561007b31SStefan Hajnoczi 114661007b31SStefan Hajnoczi assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 114761007b31SStefan Hajnoczi assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 114861007b31SStefan Hajnoczi assert(!qiov || bytes == qiov->size); 114961007b31SStefan Hajnoczi 115061007b31SStefan Hajnoczi waited = wait_serialising_requests(req); 115161007b31SStefan Hajnoczi assert(!waited || !req->serialising); 115261007b31SStefan Hajnoczi assert(req->overlap_offset <= offset); 115361007b31SStefan Hajnoczi assert(offset + bytes <= req->overlap_offset + req->overlap_bytes); 115461007b31SStefan Hajnoczi 115561007b31SStefan Hajnoczi ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req); 115661007b31SStefan Hajnoczi 115761007b31SStefan Hajnoczi if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF && 115861007b31SStefan Hajnoczi !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_write_zeroes && 115961007b31SStefan Hajnoczi qemu_iovec_is_zero(qiov)) { 116061007b31SStefan Hajnoczi flags |= BDRV_REQ_ZERO_WRITE; 116161007b31SStefan Hajnoczi if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) { 116261007b31SStefan Hajnoczi flags |= BDRV_REQ_MAY_UNMAP; 116361007b31SStefan Hajnoczi } 116461007b31SStefan Hajnoczi } 116561007b31SStefan Hajnoczi 116661007b31SStefan Hajnoczi if (ret < 0) { 116761007b31SStefan Hajnoczi /* Do nothing, write notifier decided to fail this request */ 116861007b31SStefan Hajnoczi } else if (flags & BDRV_REQ_ZERO_WRITE) { 11699a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO); 117061007b31SStefan Hajnoczi ret = bdrv_co_do_write_zeroes(bs, sector_num, nb_sectors, flags); 117161007b31SStefan Hajnoczi } else { 11729a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV); 117361007b31SStefan Hajnoczi ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov); 117461007b31SStefan Hajnoczi } 11759a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE); 117661007b31SStefan Hajnoczi 117761007b31SStefan Hajnoczi if (ret == 0 && !bs->enable_write_cache) { 117861007b31SStefan Hajnoczi ret = bdrv_co_flush(bs); 117961007b31SStefan Hajnoczi } 118061007b31SStefan Hajnoczi 118161007b31SStefan Hajnoczi bdrv_set_dirty(bs, sector_num, nb_sectors); 118261007b31SStefan Hajnoczi 118353d8f9d8SMax Reitz if (bs->wr_highest_offset < offset + bytes) { 118453d8f9d8SMax Reitz bs->wr_highest_offset = offset + bytes; 118553d8f9d8SMax Reitz } 118661007b31SStefan Hajnoczi 118761007b31SStefan Hajnoczi if (ret >= 0) { 118861007b31SStefan Hajnoczi bs->total_sectors = MAX(bs->total_sectors, sector_num + nb_sectors); 118961007b31SStefan Hajnoczi } 119061007b31SStefan Hajnoczi 119161007b31SStefan Hajnoczi return ret; 119261007b31SStefan Hajnoczi } 119361007b31SStefan Hajnoczi 11949eeb6dd1SFam Zheng static int coroutine_fn bdrv_co_do_zero_pwritev(BlockDriverState *bs, 11959eeb6dd1SFam Zheng int64_t offset, 11969eeb6dd1SFam Zheng unsigned int bytes, 11979eeb6dd1SFam Zheng BdrvRequestFlags flags, 11989eeb6dd1SFam Zheng BdrvTrackedRequest *req) 11999eeb6dd1SFam Zheng { 12009eeb6dd1SFam Zheng uint8_t *buf = NULL; 12019eeb6dd1SFam Zheng QEMUIOVector local_qiov; 12029eeb6dd1SFam Zheng struct iovec iov; 12039eeb6dd1SFam Zheng uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment); 12049eeb6dd1SFam Zheng unsigned int head_padding_bytes, tail_padding_bytes; 12059eeb6dd1SFam Zheng int ret = 0; 12069eeb6dd1SFam Zheng 12079eeb6dd1SFam Zheng head_padding_bytes = offset & (align - 1); 12089eeb6dd1SFam Zheng tail_padding_bytes = align - ((offset + bytes) & (align - 1)); 12099eeb6dd1SFam Zheng 12109eeb6dd1SFam Zheng 12119eeb6dd1SFam Zheng assert(flags & BDRV_REQ_ZERO_WRITE); 12129eeb6dd1SFam Zheng if (head_padding_bytes || tail_padding_bytes) { 12139eeb6dd1SFam Zheng buf = qemu_blockalign(bs, align); 12149eeb6dd1SFam Zheng iov = (struct iovec) { 12159eeb6dd1SFam Zheng .iov_base = buf, 12169eeb6dd1SFam Zheng .iov_len = align, 12179eeb6dd1SFam Zheng }; 12189eeb6dd1SFam Zheng qemu_iovec_init_external(&local_qiov, &iov, 1); 12199eeb6dd1SFam Zheng } 12209eeb6dd1SFam Zheng if (head_padding_bytes) { 12219eeb6dd1SFam Zheng uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes); 12229eeb6dd1SFam Zheng 12239eeb6dd1SFam Zheng /* RMW the unaligned part before head. */ 12249eeb6dd1SFam Zheng mark_request_serialising(req, align); 12259eeb6dd1SFam Zheng wait_serialising_requests(req); 12269a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 12279eeb6dd1SFam Zheng ret = bdrv_aligned_preadv(bs, req, offset & ~(align - 1), align, 12289eeb6dd1SFam Zheng align, &local_qiov, 0); 12299eeb6dd1SFam Zheng if (ret < 0) { 12309eeb6dd1SFam Zheng goto fail; 12319eeb6dd1SFam Zheng } 12329a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 12339eeb6dd1SFam Zheng 12349eeb6dd1SFam Zheng memset(buf + head_padding_bytes, 0, zero_bytes); 12359eeb6dd1SFam Zheng ret = bdrv_aligned_pwritev(bs, req, offset & ~(align - 1), align, 12369eeb6dd1SFam Zheng &local_qiov, 12379eeb6dd1SFam Zheng flags & ~BDRV_REQ_ZERO_WRITE); 12389eeb6dd1SFam Zheng if (ret < 0) { 12399eeb6dd1SFam Zheng goto fail; 12409eeb6dd1SFam Zheng } 12419eeb6dd1SFam Zheng offset += zero_bytes; 12429eeb6dd1SFam Zheng bytes -= zero_bytes; 12439eeb6dd1SFam Zheng } 12449eeb6dd1SFam Zheng 12459eeb6dd1SFam Zheng assert(!bytes || (offset & (align - 1)) == 0); 12469eeb6dd1SFam Zheng if (bytes >= align) { 12479eeb6dd1SFam Zheng /* Write the aligned part in the middle. */ 12489eeb6dd1SFam Zheng uint64_t aligned_bytes = bytes & ~(align - 1); 12499eeb6dd1SFam Zheng ret = bdrv_aligned_pwritev(bs, req, offset, aligned_bytes, 12509eeb6dd1SFam Zheng NULL, flags); 12519eeb6dd1SFam Zheng if (ret < 0) { 12529eeb6dd1SFam Zheng goto fail; 12539eeb6dd1SFam Zheng } 12549eeb6dd1SFam Zheng bytes -= aligned_bytes; 12559eeb6dd1SFam Zheng offset += aligned_bytes; 12569eeb6dd1SFam Zheng } 12579eeb6dd1SFam Zheng 12589eeb6dd1SFam Zheng assert(!bytes || (offset & (align - 1)) == 0); 12599eeb6dd1SFam Zheng if (bytes) { 12609eeb6dd1SFam Zheng assert(align == tail_padding_bytes + bytes); 12619eeb6dd1SFam Zheng /* RMW the unaligned part after tail. */ 12629eeb6dd1SFam Zheng mark_request_serialising(req, align); 12639eeb6dd1SFam Zheng wait_serialising_requests(req); 12649a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 12659eeb6dd1SFam Zheng ret = bdrv_aligned_preadv(bs, req, offset, align, 12669eeb6dd1SFam Zheng align, &local_qiov, 0); 12679eeb6dd1SFam Zheng if (ret < 0) { 12689eeb6dd1SFam Zheng goto fail; 12699eeb6dd1SFam Zheng } 12709a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 12719eeb6dd1SFam Zheng 12729eeb6dd1SFam Zheng memset(buf, 0, bytes); 12739eeb6dd1SFam Zheng ret = bdrv_aligned_pwritev(bs, req, offset, align, 12749eeb6dd1SFam Zheng &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE); 12759eeb6dd1SFam Zheng } 12769eeb6dd1SFam Zheng fail: 12779eeb6dd1SFam Zheng qemu_vfree(buf); 12789eeb6dd1SFam Zheng return ret; 12799eeb6dd1SFam Zheng 12809eeb6dd1SFam Zheng } 12819eeb6dd1SFam Zheng 128261007b31SStefan Hajnoczi /* 128361007b31SStefan Hajnoczi * Handle a write request in coroutine context 128461007b31SStefan Hajnoczi */ 128561007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_do_pwritev(BlockDriverState *bs, 128661007b31SStefan Hajnoczi int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 128761007b31SStefan Hajnoczi BdrvRequestFlags flags) 128861007b31SStefan Hajnoczi { 128961007b31SStefan Hajnoczi BdrvTrackedRequest req; 1290d01c07f2SFam Zheng /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */ 1291d01c07f2SFam Zheng uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment); 129261007b31SStefan Hajnoczi uint8_t *head_buf = NULL; 129361007b31SStefan Hajnoczi uint8_t *tail_buf = NULL; 129461007b31SStefan Hajnoczi QEMUIOVector local_qiov; 129561007b31SStefan Hajnoczi bool use_local_qiov = false; 129661007b31SStefan Hajnoczi int ret; 129761007b31SStefan Hajnoczi 129861007b31SStefan Hajnoczi if (!bs->drv) { 129961007b31SStefan Hajnoczi return -ENOMEDIUM; 130061007b31SStefan Hajnoczi } 130161007b31SStefan Hajnoczi if (bs->read_only) { 1302eaf5fe2dSPaolo Bonzini return -EPERM; 130361007b31SStefan Hajnoczi } 130461007b31SStefan Hajnoczi 130561007b31SStefan Hajnoczi ret = bdrv_check_byte_request(bs, offset, bytes); 130661007b31SStefan Hajnoczi if (ret < 0) { 130761007b31SStefan Hajnoczi return ret; 130861007b31SStefan Hajnoczi } 130961007b31SStefan Hajnoczi 131061007b31SStefan Hajnoczi /* throttling disk I/O */ 131161007b31SStefan Hajnoczi if (bs->io_limits_enabled) { 131276f4afb4SAlberto Garcia throttle_group_co_io_limits_intercept(bs, bytes, true); 131361007b31SStefan Hajnoczi } 131461007b31SStefan Hajnoczi 131561007b31SStefan Hajnoczi /* 131661007b31SStefan Hajnoczi * Align write if necessary by performing a read-modify-write cycle. 131761007b31SStefan Hajnoczi * Pad qiov with the read parts and be sure to have a tracked request not 131861007b31SStefan Hajnoczi * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle. 131961007b31SStefan Hajnoczi */ 1320ebde595cSFam Zheng tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE); 132161007b31SStefan Hajnoczi 13229eeb6dd1SFam Zheng if (!qiov) { 13239eeb6dd1SFam Zheng ret = bdrv_co_do_zero_pwritev(bs, offset, bytes, flags, &req); 13249eeb6dd1SFam Zheng goto out; 13259eeb6dd1SFam Zheng } 13269eeb6dd1SFam Zheng 132761007b31SStefan Hajnoczi if (offset & (align - 1)) { 132861007b31SStefan Hajnoczi QEMUIOVector head_qiov; 132961007b31SStefan Hajnoczi struct iovec head_iov; 133061007b31SStefan Hajnoczi 133161007b31SStefan Hajnoczi mark_request_serialising(&req, align); 133261007b31SStefan Hajnoczi wait_serialising_requests(&req); 133361007b31SStefan Hajnoczi 133461007b31SStefan Hajnoczi head_buf = qemu_blockalign(bs, align); 133561007b31SStefan Hajnoczi head_iov = (struct iovec) { 133661007b31SStefan Hajnoczi .iov_base = head_buf, 133761007b31SStefan Hajnoczi .iov_len = align, 133861007b31SStefan Hajnoczi }; 133961007b31SStefan Hajnoczi qemu_iovec_init_external(&head_qiov, &head_iov, 1); 134061007b31SStefan Hajnoczi 13419a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 134261007b31SStefan Hajnoczi ret = bdrv_aligned_preadv(bs, &req, offset & ~(align - 1), align, 134361007b31SStefan Hajnoczi align, &head_qiov, 0); 134461007b31SStefan Hajnoczi if (ret < 0) { 134561007b31SStefan Hajnoczi goto fail; 134661007b31SStefan Hajnoczi } 13479a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 134861007b31SStefan Hajnoczi 134961007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 2); 135061007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 135161007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 135261007b31SStefan Hajnoczi use_local_qiov = true; 135361007b31SStefan Hajnoczi 135461007b31SStefan Hajnoczi bytes += offset & (align - 1); 135561007b31SStefan Hajnoczi offset = offset & ~(align - 1); 135661007b31SStefan Hajnoczi } 135761007b31SStefan Hajnoczi 135861007b31SStefan Hajnoczi if ((offset + bytes) & (align - 1)) { 135961007b31SStefan Hajnoczi QEMUIOVector tail_qiov; 136061007b31SStefan Hajnoczi struct iovec tail_iov; 136161007b31SStefan Hajnoczi size_t tail_bytes; 136261007b31SStefan Hajnoczi bool waited; 136361007b31SStefan Hajnoczi 136461007b31SStefan Hajnoczi mark_request_serialising(&req, align); 136561007b31SStefan Hajnoczi waited = wait_serialising_requests(&req); 136661007b31SStefan Hajnoczi assert(!waited || !use_local_qiov); 136761007b31SStefan Hajnoczi 136861007b31SStefan Hajnoczi tail_buf = qemu_blockalign(bs, align); 136961007b31SStefan Hajnoczi tail_iov = (struct iovec) { 137061007b31SStefan Hajnoczi .iov_base = tail_buf, 137161007b31SStefan Hajnoczi .iov_len = align, 137261007b31SStefan Hajnoczi }; 137361007b31SStefan Hajnoczi qemu_iovec_init_external(&tail_qiov, &tail_iov, 1); 137461007b31SStefan Hajnoczi 13759a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 137661007b31SStefan Hajnoczi ret = bdrv_aligned_preadv(bs, &req, (offset + bytes) & ~(align - 1), align, 137761007b31SStefan Hajnoczi align, &tail_qiov, 0); 137861007b31SStefan Hajnoczi if (ret < 0) { 137961007b31SStefan Hajnoczi goto fail; 138061007b31SStefan Hajnoczi } 13819a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 138261007b31SStefan Hajnoczi 138361007b31SStefan Hajnoczi if (!use_local_qiov) { 138461007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 1); 138561007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 138661007b31SStefan Hajnoczi use_local_qiov = true; 138761007b31SStefan Hajnoczi } 138861007b31SStefan Hajnoczi 138961007b31SStefan Hajnoczi tail_bytes = (offset + bytes) & (align - 1); 139061007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes); 139161007b31SStefan Hajnoczi 139261007b31SStefan Hajnoczi bytes = ROUND_UP(bytes, align); 139361007b31SStefan Hajnoczi } 139461007b31SStefan Hajnoczi 139561007b31SStefan Hajnoczi ret = bdrv_aligned_pwritev(bs, &req, offset, bytes, 139661007b31SStefan Hajnoczi use_local_qiov ? &local_qiov : qiov, 139761007b31SStefan Hajnoczi flags); 139861007b31SStefan Hajnoczi 139961007b31SStefan Hajnoczi fail: 140061007b31SStefan Hajnoczi 140161007b31SStefan Hajnoczi if (use_local_qiov) { 140261007b31SStefan Hajnoczi qemu_iovec_destroy(&local_qiov); 140361007b31SStefan Hajnoczi } 140461007b31SStefan Hajnoczi qemu_vfree(head_buf); 140561007b31SStefan Hajnoczi qemu_vfree(tail_buf); 14069eeb6dd1SFam Zheng out: 14079eeb6dd1SFam Zheng tracked_request_end(&req); 140861007b31SStefan Hajnoczi return ret; 140961007b31SStefan Hajnoczi } 141061007b31SStefan Hajnoczi 141161007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_do_writev(BlockDriverState *bs, 141261007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, 141361007b31SStefan Hajnoczi BdrvRequestFlags flags) 141461007b31SStefan Hajnoczi { 141561007b31SStefan Hajnoczi if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 141661007b31SStefan Hajnoczi return -EINVAL; 141761007b31SStefan Hajnoczi } 141861007b31SStefan Hajnoczi 141961007b31SStefan Hajnoczi return bdrv_co_do_pwritev(bs, sector_num << BDRV_SECTOR_BITS, 142061007b31SStefan Hajnoczi nb_sectors << BDRV_SECTOR_BITS, qiov, flags); 142161007b31SStefan Hajnoczi } 142261007b31SStefan Hajnoczi 142361007b31SStefan Hajnoczi int coroutine_fn bdrv_co_writev(BlockDriverState *bs, int64_t sector_num, 142461007b31SStefan Hajnoczi int nb_sectors, QEMUIOVector *qiov) 142561007b31SStefan Hajnoczi { 142661007b31SStefan Hajnoczi trace_bdrv_co_writev(bs, sector_num, nb_sectors); 142761007b31SStefan Hajnoczi 142861007b31SStefan Hajnoczi return bdrv_co_do_writev(bs, sector_num, nb_sectors, qiov, 0); 142961007b31SStefan Hajnoczi } 143061007b31SStefan Hajnoczi 143161007b31SStefan Hajnoczi int coroutine_fn bdrv_co_write_zeroes(BlockDriverState *bs, 143261007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, 143361007b31SStefan Hajnoczi BdrvRequestFlags flags) 143461007b31SStefan Hajnoczi { 143561007b31SStefan Hajnoczi trace_bdrv_co_write_zeroes(bs, sector_num, nb_sectors, flags); 143661007b31SStefan Hajnoczi 143761007b31SStefan Hajnoczi if (!(bs->open_flags & BDRV_O_UNMAP)) { 143861007b31SStefan Hajnoczi flags &= ~BDRV_REQ_MAY_UNMAP; 143961007b31SStefan Hajnoczi } 144061007b31SStefan Hajnoczi 1441d01c07f2SFam Zheng return bdrv_co_do_writev(bs, sector_num, nb_sectors, NULL, 144261007b31SStefan Hajnoczi BDRV_REQ_ZERO_WRITE | flags); 144361007b31SStefan Hajnoczi } 144461007b31SStefan Hajnoczi 144561007b31SStefan Hajnoczi int bdrv_flush_all(void) 144661007b31SStefan Hajnoczi { 144761007b31SStefan Hajnoczi BlockDriverState *bs = NULL; 144861007b31SStefan Hajnoczi int result = 0; 144961007b31SStefan Hajnoczi 145061007b31SStefan Hajnoczi while ((bs = bdrv_next(bs))) { 145161007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 145261007b31SStefan Hajnoczi int ret; 145361007b31SStefan Hajnoczi 145461007b31SStefan Hajnoczi aio_context_acquire(aio_context); 145561007b31SStefan Hajnoczi ret = bdrv_flush(bs); 145661007b31SStefan Hajnoczi if (ret < 0 && !result) { 145761007b31SStefan Hajnoczi result = ret; 145861007b31SStefan Hajnoczi } 145961007b31SStefan Hajnoczi aio_context_release(aio_context); 146061007b31SStefan Hajnoczi } 146161007b31SStefan Hajnoczi 146261007b31SStefan Hajnoczi return result; 146361007b31SStefan Hajnoczi } 146461007b31SStefan Hajnoczi 146561007b31SStefan Hajnoczi typedef struct BdrvCoGetBlockStatusData { 146661007b31SStefan Hajnoczi BlockDriverState *bs; 146761007b31SStefan Hajnoczi BlockDriverState *base; 146861007b31SStefan Hajnoczi int64_t sector_num; 146961007b31SStefan Hajnoczi int nb_sectors; 147061007b31SStefan Hajnoczi int *pnum; 147161007b31SStefan Hajnoczi int64_t ret; 147261007b31SStefan Hajnoczi bool done; 147361007b31SStefan Hajnoczi } BdrvCoGetBlockStatusData; 147461007b31SStefan Hajnoczi 147561007b31SStefan Hajnoczi /* 147661007b31SStefan Hajnoczi * Returns the allocation status of the specified sectors. 147761007b31SStefan Hajnoczi * Drivers not implementing the functionality are assumed to not support 147861007b31SStefan Hajnoczi * backing files, hence all their sectors are reported as allocated. 147961007b31SStefan Hajnoczi * 148061007b31SStefan Hajnoczi * If 'sector_num' is beyond the end of the disk image the return value is 0 148161007b31SStefan Hajnoczi * and 'pnum' is set to 0. 148261007b31SStefan Hajnoczi * 148361007b31SStefan Hajnoczi * 'pnum' is set to the number of sectors (including and immediately following 148461007b31SStefan Hajnoczi * the specified sector) that are known to be in the same 148561007b31SStefan Hajnoczi * allocated/unallocated state. 148661007b31SStefan Hajnoczi * 148761007b31SStefan Hajnoczi * 'nb_sectors' is the max value 'pnum' should be set to. If nb_sectors goes 148861007b31SStefan Hajnoczi * beyond the end of the disk image it will be clamped. 148961007b31SStefan Hajnoczi */ 149061007b31SStefan Hajnoczi static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs, 149161007b31SStefan Hajnoczi int64_t sector_num, 149261007b31SStefan Hajnoczi int nb_sectors, int *pnum) 149361007b31SStefan Hajnoczi { 149461007b31SStefan Hajnoczi int64_t total_sectors; 149561007b31SStefan Hajnoczi int64_t n; 149661007b31SStefan Hajnoczi int64_t ret, ret2; 149761007b31SStefan Hajnoczi 149861007b31SStefan Hajnoczi total_sectors = bdrv_nb_sectors(bs); 149961007b31SStefan Hajnoczi if (total_sectors < 0) { 150061007b31SStefan Hajnoczi return total_sectors; 150161007b31SStefan Hajnoczi } 150261007b31SStefan Hajnoczi 150361007b31SStefan Hajnoczi if (sector_num >= total_sectors) { 150461007b31SStefan Hajnoczi *pnum = 0; 150561007b31SStefan Hajnoczi return 0; 150661007b31SStefan Hajnoczi } 150761007b31SStefan Hajnoczi 150861007b31SStefan Hajnoczi n = total_sectors - sector_num; 150961007b31SStefan Hajnoczi if (n < nb_sectors) { 151061007b31SStefan Hajnoczi nb_sectors = n; 151161007b31SStefan Hajnoczi } 151261007b31SStefan Hajnoczi 151361007b31SStefan Hajnoczi if (!bs->drv->bdrv_co_get_block_status) { 151461007b31SStefan Hajnoczi *pnum = nb_sectors; 151561007b31SStefan Hajnoczi ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED; 151661007b31SStefan Hajnoczi if (bs->drv->protocol_name) { 151761007b31SStefan Hajnoczi ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE); 151861007b31SStefan Hajnoczi } 151961007b31SStefan Hajnoczi return ret; 152061007b31SStefan Hajnoczi } 152161007b31SStefan Hajnoczi 152261007b31SStefan Hajnoczi ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum); 152361007b31SStefan Hajnoczi if (ret < 0) { 152461007b31SStefan Hajnoczi *pnum = 0; 152561007b31SStefan Hajnoczi return ret; 152661007b31SStefan Hajnoczi } 152761007b31SStefan Hajnoczi 152861007b31SStefan Hajnoczi if (ret & BDRV_BLOCK_RAW) { 152961007b31SStefan Hajnoczi assert(ret & BDRV_BLOCK_OFFSET_VALID); 15309a4f4c31SKevin Wolf return bdrv_get_block_status(bs->file->bs, ret >> BDRV_SECTOR_BITS, 153161007b31SStefan Hajnoczi *pnum, pnum); 153261007b31SStefan Hajnoczi } 153361007b31SStefan Hajnoczi 153461007b31SStefan Hajnoczi if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) { 153561007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ALLOCATED; 1536a53f1a95SPaolo Bonzini } else { 153761007b31SStefan Hajnoczi if (bdrv_unallocated_blocks_are_zero(bs)) { 153861007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 1539760e0063SKevin Wolf } else if (bs->backing) { 1540760e0063SKevin Wolf BlockDriverState *bs2 = bs->backing->bs; 154161007b31SStefan Hajnoczi int64_t nb_sectors2 = bdrv_nb_sectors(bs2); 154261007b31SStefan Hajnoczi if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) { 154361007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 154461007b31SStefan Hajnoczi } 154561007b31SStefan Hajnoczi } 154661007b31SStefan Hajnoczi } 154761007b31SStefan Hajnoczi 154861007b31SStefan Hajnoczi if (bs->file && 154961007b31SStefan Hajnoczi (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) && 155061007b31SStefan Hajnoczi (ret & BDRV_BLOCK_OFFSET_VALID)) { 155161007b31SStefan Hajnoczi int file_pnum; 155261007b31SStefan Hajnoczi 15539a4f4c31SKevin Wolf ret2 = bdrv_co_get_block_status(bs->file->bs, ret >> BDRV_SECTOR_BITS, 155461007b31SStefan Hajnoczi *pnum, &file_pnum); 155561007b31SStefan Hajnoczi if (ret2 >= 0) { 155661007b31SStefan Hajnoczi /* Ignore errors. This is just providing extra information, it 155761007b31SStefan Hajnoczi * is useful but not necessary. 155861007b31SStefan Hajnoczi */ 155961007b31SStefan Hajnoczi if (!file_pnum) { 156061007b31SStefan Hajnoczi /* !file_pnum indicates an offset at or beyond the EOF; it is 156161007b31SStefan Hajnoczi * perfectly valid for the format block driver to point to such 156261007b31SStefan Hajnoczi * offsets, so catch it and mark everything as zero */ 156361007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 156461007b31SStefan Hajnoczi } else { 156561007b31SStefan Hajnoczi /* Limit request to the range reported by the protocol driver */ 156661007b31SStefan Hajnoczi *pnum = file_pnum; 156761007b31SStefan Hajnoczi ret |= (ret2 & BDRV_BLOCK_ZERO); 156861007b31SStefan Hajnoczi } 156961007b31SStefan Hajnoczi } 157061007b31SStefan Hajnoczi } 157161007b31SStefan Hajnoczi 157261007b31SStefan Hajnoczi return ret; 157361007b31SStefan Hajnoczi } 157461007b31SStefan Hajnoczi 1575ba3f0e25SFam Zheng static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs, 1576ba3f0e25SFam Zheng BlockDriverState *base, 1577ba3f0e25SFam Zheng int64_t sector_num, 1578ba3f0e25SFam Zheng int nb_sectors, 1579ba3f0e25SFam Zheng int *pnum) 1580ba3f0e25SFam Zheng { 1581ba3f0e25SFam Zheng BlockDriverState *p; 1582ba3f0e25SFam Zheng int64_t ret = 0; 1583ba3f0e25SFam Zheng 1584ba3f0e25SFam Zheng assert(bs != base); 1585760e0063SKevin Wolf for (p = bs; p != base; p = backing_bs(p)) { 1586ba3f0e25SFam Zheng ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum); 1587ba3f0e25SFam Zheng if (ret < 0 || ret & BDRV_BLOCK_ALLOCATED) { 1588ba3f0e25SFam Zheng break; 1589ba3f0e25SFam Zheng } 1590ba3f0e25SFam Zheng /* [sector_num, pnum] unallocated on this layer, which could be only 1591ba3f0e25SFam Zheng * the first part of [sector_num, nb_sectors]. */ 1592ba3f0e25SFam Zheng nb_sectors = MIN(nb_sectors, *pnum); 1593ba3f0e25SFam Zheng } 1594ba3f0e25SFam Zheng return ret; 1595ba3f0e25SFam Zheng } 1596ba3f0e25SFam Zheng 1597ba3f0e25SFam Zheng /* Coroutine wrapper for bdrv_get_block_status_above() */ 1598ba3f0e25SFam Zheng static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque) 159961007b31SStefan Hajnoczi { 160061007b31SStefan Hajnoczi BdrvCoGetBlockStatusData *data = opaque; 160161007b31SStefan Hajnoczi 1602ba3f0e25SFam Zheng data->ret = bdrv_co_get_block_status_above(data->bs, data->base, 1603ba3f0e25SFam Zheng data->sector_num, 1604ba3f0e25SFam Zheng data->nb_sectors, 160561007b31SStefan Hajnoczi data->pnum); 160661007b31SStefan Hajnoczi data->done = true; 160761007b31SStefan Hajnoczi } 160861007b31SStefan Hajnoczi 160961007b31SStefan Hajnoczi /* 1610ba3f0e25SFam Zheng * Synchronous wrapper around bdrv_co_get_block_status_above(). 161161007b31SStefan Hajnoczi * 1612ba3f0e25SFam Zheng * See bdrv_co_get_block_status_above() for details. 161361007b31SStefan Hajnoczi */ 1614ba3f0e25SFam Zheng int64_t bdrv_get_block_status_above(BlockDriverState *bs, 1615ba3f0e25SFam Zheng BlockDriverState *base, 1616ba3f0e25SFam Zheng int64_t sector_num, 161761007b31SStefan Hajnoczi int nb_sectors, int *pnum) 161861007b31SStefan Hajnoczi { 161961007b31SStefan Hajnoczi Coroutine *co; 162061007b31SStefan Hajnoczi BdrvCoGetBlockStatusData data = { 162161007b31SStefan Hajnoczi .bs = bs, 1622ba3f0e25SFam Zheng .base = base, 162361007b31SStefan Hajnoczi .sector_num = sector_num, 162461007b31SStefan Hajnoczi .nb_sectors = nb_sectors, 162561007b31SStefan Hajnoczi .pnum = pnum, 162661007b31SStefan Hajnoczi .done = false, 162761007b31SStefan Hajnoczi }; 162861007b31SStefan Hajnoczi 162961007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 163061007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 1631ba3f0e25SFam Zheng bdrv_get_block_status_above_co_entry(&data); 163261007b31SStefan Hajnoczi } else { 163361007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 163461007b31SStefan Hajnoczi 1635ba3f0e25SFam Zheng co = qemu_coroutine_create(bdrv_get_block_status_above_co_entry); 163661007b31SStefan Hajnoczi qemu_coroutine_enter(co, &data); 163761007b31SStefan Hajnoczi while (!data.done) { 163861007b31SStefan Hajnoczi aio_poll(aio_context, true); 163961007b31SStefan Hajnoczi } 164061007b31SStefan Hajnoczi } 164161007b31SStefan Hajnoczi return data.ret; 164261007b31SStefan Hajnoczi } 164361007b31SStefan Hajnoczi 1644ba3f0e25SFam Zheng int64_t bdrv_get_block_status(BlockDriverState *bs, 1645ba3f0e25SFam Zheng int64_t sector_num, 1646ba3f0e25SFam Zheng int nb_sectors, int *pnum) 1647ba3f0e25SFam Zheng { 1648760e0063SKevin Wolf return bdrv_get_block_status_above(bs, backing_bs(bs), 1649ba3f0e25SFam Zheng sector_num, nb_sectors, pnum); 1650ba3f0e25SFam Zheng } 1651ba3f0e25SFam Zheng 165261007b31SStefan Hajnoczi int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num, 165361007b31SStefan Hajnoczi int nb_sectors, int *pnum) 165461007b31SStefan Hajnoczi { 165561007b31SStefan Hajnoczi int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum); 165661007b31SStefan Hajnoczi if (ret < 0) { 165761007b31SStefan Hajnoczi return ret; 165861007b31SStefan Hajnoczi } 165961007b31SStefan Hajnoczi return !!(ret & BDRV_BLOCK_ALLOCATED); 166061007b31SStefan Hajnoczi } 166161007b31SStefan Hajnoczi 166261007b31SStefan Hajnoczi /* 166361007b31SStefan Hajnoczi * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP] 166461007b31SStefan Hajnoczi * 166561007b31SStefan Hajnoczi * Return true if the given sector is allocated in any image between 166661007b31SStefan Hajnoczi * BASE and TOP (inclusive). BASE can be NULL to check if the given 166761007b31SStefan Hajnoczi * sector is allocated in any image of the chain. Return false otherwise. 166861007b31SStefan Hajnoczi * 166961007b31SStefan Hajnoczi * 'pnum' is set to the number of sectors (including and immediately following 167061007b31SStefan Hajnoczi * the specified sector) that are known to be in the same 167161007b31SStefan Hajnoczi * allocated/unallocated state. 167261007b31SStefan Hajnoczi * 167361007b31SStefan Hajnoczi */ 167461007b31SStefan Hajnoczi int bdrv_is_allocated_above(BlockDriverState *top, 167561007b31SStefan Hajnoczi BlockDriverState *base, 167661007b31SStefan Hajnoczi int64_t sector_num, 167761007b31SStefan Hajnoczi int nb_sectors, int *pnum) 167861007b31SStefan Hajnoczi { 167961007b31SStefan Hajnoczi BlockDriverState *intermediate; 168061007b31SStefan Hajnoczi int ret, n = nb_sectors; 168161007b31SStefan Hajnoczi 168261007b31SStefan Hajnoczi intermediate = top; 168361007b31SStefan Hajnoczi while (intermediate && intermediate != base) { 168461007b31SStefan Hajnoczi int pnum_inter; 168561007b31SStefan Hajnoczi ret = bdrv_is_allocated(intermediate, sector_num, nb_sectors, 168661007b31SStefan Hajnoczi &pnum_inter); 168761007b31SStefan Hajnoczi if (ret < 0) { 168861007b31SStefan Hajnoczi return ret; 168961007b31SStefan Hajnoczi } else if (ret) { 169061007b31SStefan Hajnoczi *pnum = pnum_inter; 169161007b31SStefan Hajnoczi return 1; 169261007b31SStefan Hajnoczi } 169361007b31SStefan Hajnoczi 169461007b31SStefan Hajnoczi /* 169561007b31SStefan Hajnoczi * [sector_num, nb_sectors] is unallocated on top but intermediate 169661007b31SStefan Hajnoczi * might have 169761007b31SStefan Hajnoczi * 169861007b31SStefan Hajnoczi * [sector_num+x, nr_sectors] allocated. 169961007b31SStefan Hajnoczi */ 170061007b31SStefan Hajnoczi if (n > pnum_inter && 170161007b31SStefan Hajnoczi (intermediate == top || 170261007b31SStefan Hajnoczi sector_num + pnum_inter < intermediate->total_sectors)) { 170361007b31SStefan Hajnoczi n = pnum_inter; 170461007b31SStefan Hajnoczi } 170561007b31SStefan Hajnoczi 1706760e0063SKevin Wolf intermediate = backing_bs(intermediate); 170761007b31SStefan Hajnoczi } 170861007b31SStefan Hajnoczi 170961007b31SStefan Hajnoczi *pnum = n; 171061007b31SStefan Hajnoczi return 0; 171161007b31SStefan Hajnoczi } 171261007b31SStefan Hajnoczi 171361007b31SStefan Hajnoczi int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num, 171461007b31SStefan Hajnoczi const uint8_t *buf, int nb_sectors) 171561007b31SStefan Hajnoczi { 171661007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 171761007b31SStefan Hajnoczi int ret; 171861007b31SStefan Hajnoczi 171961007b31SStefan Hajnoczi if (!drv) { 172061007b31SStefan Hajnoczi return -ENOMEDIUM; 172161007b31SStefan Hajnoczi } 172261007b31SStefan Hajnoczi if (!drv->bdrv_write_compressed) { 172361007b31SStefan Hajnoczi return -ENOTSUP; 172461007b31SStefan Hajnoczi } 172561007b31SStefan Hajnoczi ret = bdrv_check_request(bs, sector_num, nb_sectors); 172661007b31SStefan Hajnoczi if (ret < 0) { 172761007b31SStefan Hajnoczi return ret; 172861007b31SStefan Hajnoczi } 172961007b31SStefan Hajnoczi 173061007b31SStefan Hajnoczi assert(QLIST_EMPTY(&bs->dirty_bitmaps)); 173161007b31SStefan Hajnoczi 173261007b31SStefan Hajnoczi return drv->bdrv_write_compressed(bs, sector_num, buf, nb_sectors); 173361007b31SStefan Hajnoczi } 173461007b31SStefan Hajnoczi 173561007b31SStefan Hajnoczi int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf, 173661007b31SStefan Hajnoczi int64_t pos, int size) 173761007b31SStefan Hajnoczi { 173861007b31SStefan Hajnoczi QEMUIOVector qiov; 173961007b31SStefan Hajnoczi struct iovec iov = { 174061007b31SStefan Hajnoczi .iov_base = (void *) buf, 174161007b31SStefan Hajnoczi .iov_len = size, 174261007b31SStefan Hajnoczi }; 174361007b31SStefan Hajnoczi 174461007b31SStefan Hajnoczi qemu_iovec_init_external(&qiov, &iov, 1); 174561007b31SStefan Hajnoczi return bdrv_writev_vmstate(bs, &qiov, pos); 174661007b31SStefan Hajnoczi } 174761007b31SStefan Hajnoczi 174861007b31SStefan Hajnoczi int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 174961007b31SStefan Hajnoczi { 175061007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 175161007b31SStefan Hajnoczi 175261007b31SStefan Hajnoczi if (!drv) { 175361007b31SStefan Hajnoczi return -ENOMEDIUM; 175461007b31SStefan Hajnoczi } else if (drv->bdrv_save_vmstate) { 175561007b31SStefan Hajnoczi return drv->bdrv_save_vmstate(bs, qiov, pos); 175661007b31SStefan Hajnoczi } else if (bs->file) { 17579a4f4c31SKevin Wolf return bdrv_writev_vmstate(bs->file->bs, qiov, pos); 175861007b31SStefan Hajnoczi } 175961007b31SStefan Hajnoczi 176061007b31SStefan Hajnoczi return -ENOTSUP; 176161007b31SStefan Hajnoczi } 176261007b31SStefan Hajnoczi 176361007b31SStefan Hajnoczi int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf, 176461007b31SStefan Hajnoczi int64_t pos, int size) 176561007b31SStefan Hajnoczi { 176661007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 176761007b31SStefan Hajnoczi if (!drv) 176861007b31SStefan Hajnoczi return -ENOMEDIUM; 176961007b31SStefan Hajnoczi if (drv->bdrv_load_vmstate) 177061007b31SStefan Hajnoczi return drv->bdrv_load_vmstate(bs, buf, pos, size); 177161007b31SStefan Hajnoczi if (bs->file) 17729a4f4c31SKevin Wolf return bdrv_load_vmstate(bs->file->bs, buf, pos, size); 177361007b31SStefan Hajnoczi return -ENOTSUP; 177461007b31SStefan Hajnoczi } 177561007b31SStefan Hajnoczi 177661007b31SStefan Hajnoczi /**************************************************************/ 177761007b31SStefan Hajnoczi /* async I/Os */ 177861007b31SStefan Hajnoczi 177961007b31SStefan Hajnoczi BlockAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num, 178061007b31SStefan Hajnoczi QEMUIOVector *qiov, int nb_sectors, 178161007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque) 178261007b31SStefan Hajnoczi { 178361007b31SStefan Hajnoczi trace_bdrv_aio_readv(bs, sector_num, nb_sectors, opaque); 178461007b31SStefan Hajnoczi 178561007b31SStefan Hajnoczi return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0, 178661007b31SStefan Hajnoczi cb, opaque, false); 178761007b31SStefan Hajnoczi } 178861007b31SStefan Hajnoczi 178961007b31SStefan Hajnoczi BlockAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num, 179061007b31SStefan Hajnoczi QEMUIOVector *qiov, int nb_sectors, 179161007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque) 179261007b31SStefan Hajnoczi { 179361007b31SStefan Hajnoczi trace_bdrv_aio_writev(bs, sector_num, nb_sectors, opaque); 179461007b31SStefan Hajnoczi 179561007b31SStefan Hajnoczi return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0, 179661007b31SStefan Hajnoczi cb, opaque, true); 179761007b31SStefan Hajnoczi } 179861007b31SStefan Hajnoczi 179961007b31SStefan Hajnoczi BlockAIOCB *bdrv_aio_write_zeroes(BlockDriverState *bs, 180061007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, BdrvRequestFlags flags, 180161007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque) 180261007b31SStefan Hajnoczi { 180361007b31SStefan Hajnoczi trace_bdrv_aio_write_zeroes(bs, sector_num, nb_sectors, flags, opaque); 180461007b31SStefan Hajnoczi 180561007b31SStefan Hajnoczi return bdrv_co_aio_rw_vector(bs, sector_num, NULL, nb_sectors, 180661007b31SStefan Hajnoczi BDRV_REQ_ZERO_WRITE | flags, 180761007b31SStefan Hajnoczi cb, opaque, true); 180861007b31SStefan Hajnoczi } 180961007b31SStefan Hajnoczi 181061007b31SStefan Hajnoczi 181161007b31SStefan Hajnoczi typedef struct MultiwriteCB { 181261007b31SStefan Hajnoczi int error; 181361007b31SStefan Hajnoczi int num_requests; 181461007b31SStefan Hajnoczi int num_callbacks; 181561007b31SStefan Hajnoczi struct { 181661007b31SStefan Hajnoczi BlockCompletionFunc *cb; 181761007b31SStefan Hajnoczi void *opaque; 181861007b31SStefan Hajnoczi QEMUIOVector *free_qiov; 181961007b31SStefan Hajnoczi } callbacks[]; 182061007b31SStefan Hajnoczi } MultiwriteCB; 182161007b31SStefan Hajnoczi 182261007b31SStefan Hajnoczi static void multiwrite_user_cb(MultiwriteCB *mcb) 182361007b31SStefan Hajnoczi { 182461007b31SStefan Hajnoczi int i; 182561007b31SStefan Hajnoczi 182661007b31SStefan Hajnoczi for (i = 0; i < mcb->num_callbacks; i++) { 182761007b31SStefan Hajnoczi mcb->callbacks[i].cb(mcb->callbacks[i].opaque, mcb->error); 182861007b31SStefan Hajnoczi if (mcb->callbacks[i].free_qiov) { 182961007b31SStefan Hajnoczi qemu_iovec_destroy(mcb->callbacks[i].free_qiov); 183061007b31SStefan Hajnoczi } 183161007b31SStefan Hajnoczi g_free(mcb->callbacks[i].free_qiov); 183261007b31SStefan Hajnoczi } 183361007b31SStefan Hajnoczi } 183461007b31SStefan Hajnoczi 183561007b31SStefan Hajnoczi static void multiwrite_cb(void *opaque, int ret) 183661007b31SStefan Hajnoczi { 183761007b31SStefan Hajnoczi MultiwriteCB *mcb = opaque; 183861007b31SStefan Hajnoczi 183961007b31SStefan Hajnoczi trace_multiwrite_cb(mcb, ret); 184061007b31SStefan Hajnoczi 184161007b31SStefan Hajnoczi if (ret < 0 && !mcb->error) { 184261007b31SStefan Hajnoczi mcb->error = ret; 184361007b31SStefan Hajnoczi } 184461007b31SStefan Hajnoczi 184561007b31SStefan Hajnoczi mcb->num_requests--; 184661007b31SStefan Hajnoczi if (mcb->num_requests == 0) { 184761007b31SStefan Hajnoczi multiwrite_user_cb(mcb); 184861007b31SStefan Hajnoczi g_free(mcb); 184961007b31SStefan Hajnoczi } 185061007b31SStefan Hajnoczi } 185161007b31SStefan Hajnoczi 185261007b31SStefan Hajnoczi static int multiwrite_req_compare(const void *a, const void *b) 185361007b31SStefan Hajnoczi { 185461007b31SStefan Hajnoczi const BlockRequest *req1 = a, *req2 = b; 185561007b31SStefan Hajnoczi 185661007b31SStefan Hajnoczi /* 185761007b31SStefan Hajnoczi * Note that we can't simply subtract req2->sector from req1->sector 185861007b31SStefan Hajnoczi * here as that could overflow the return value. 185961007b31SStefan Hajnoczi */ 186061007b31SStefan Hajnoczi if (req1->sector > req2->sector) { 186161007b31SStefan Hajnoczi return 1; 186261007b31SStefan Hajnoczi } else if (req1->sector < req2->sector) { 186361007b31SStefan Hajnoczi return -1; 186461007b31SStefan Hajnoczi } else { 186561007b31SStefan Hajnoczi return 0; 186661007b31SStefan Hajnoczi } 186761007b31SStefan Hajnoczi } 186861007b31SStefan Hajnoczi 186961007b31SStefan Hajnoczi /* 187061007b31SStefan Hajnoczi * Takes a bunch of requests and tries to merge them. Returns the number of 187161007b31SStefan Hajnoczi * requests that remain after merging. 187261007b31SStefan Hajnoczi */ 187361007b31SStefan Hajnoczi static int multiwrite_merge(BlockDriverState *bs, BlockRequest *reqs, 187461007b31SStefan Hajnoczi int num_reqs, MultiwriteCB *mcb) 187561007b31SStefan Hajnoczi { 187661007b31SStefan Hajnoczi int i, outidx; 187761007b31SStefan Hajnoczi 187861007b31SStefan Hajnoczi // Sort requests by start sector 187961007b31SStefan Hajnoczi qsort(reqs, num_reqs, sizeof(*reqs), &multiwrite_req_compare); 188061007b31SStefan Hajnoczi 188161007b31SStefan Hajnoczi // Check if adjacent requests touch the same clusters. If so, combine them, 188261007b31SStefan Hajnoczi // filling up gaps with zero sectors. 188361007b31SStefan Hajnoczi outidx = 0; 188461007b31SStefan Hajnoczi for (i = 1; i < num_reqs; i++) { 188561007b31SStefan Hajnoczi int merge = 0; 188661007b31SStefan Hajnoczi int64_t oldreq_last = reqs[outidx].sector + reqs[outidx].nb_sectors; 188761007b31SStefan Hajnoczi 188861007b31SStefan Hajnoczi // Handle exactly sequential writes and overlapping writes. 188961007b31SStefan Hajnoczi if (reqs[i].sector <= oldreq_last) { 189061007b31SStefan Hajnoczi merge = 1; 189161007b31SStefan Hajnoczi } 189261007b31SStefan Hajnoczi 1893222565f6SStefan Hajnoczi if (reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1 > 1894222565f6SStefan Hajnoczi bs->bl.max_iov) { 189561007b31SStefan Hajnoczi merge = 0; 189661007b31SStefan Hajnoczi } 189761007b31SStefan Hajnoczi 189861007b31SStefan Hajnoczi if (bs->bl.max_transfer_length && reqs[outidx].nb_sectors + 189961007b31SStefan Hajnoczi reqs[i].nb_sectors > bs->bl.max_transfer_length) { 190061007b31SStefan Hajnoczi merge = 0; 190161007b31SStefan Hajnoczi } 190261007b31SStefan Hajnoczi 190361007b31SStefan Hajnoczi if (merge) { 190461007b31SStefan Hajnoczi size_t size; 190561007b31SStefan Hajnoczi QEMUIOVector *qiov = g_malloc0(sizeof(*qiov)); 190661007b31SStefan Hajnoczi qemu_iovec_init(qiov, 190761007b31SStefan Hajnoczi reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1); 190861007b31SStefan Hajnoczi 190961007b31SStefan Hajnoczi // Add the first request to the merged one. If the requests are 191061007b31SStefan Hajnoczi // overlapping, drop the last sectors of the first request. 191161007b31SStefan Hajnoczi size = (reqs[i].sector - reqs[outidx].sector) << 9; 191261007b31SStefan Hajnoczi qemu_iovec_concat(qiov, reqs[outidx].qiov, 0, size); 191361007b31SStefan Hajnoczi 191461007b31SStefan Hajnoczi // We should need to add any zeros between the two requests 191561007b31SStefan Hajnoczi assert (reqs[i].sector <= oldreq_last); 191661007b31SStefan Hajnoczi 191761007b31SStefan Hajnoczi // Add the second request 191861007b31SStefan Hajnoczi qemu_iovec_concat(qiov, reqs[i].qiov, 0, reqs[i].qiov->size); 191961007b31SStefan Hajnoczi 192061007b31SStefan Hajnoczi // Add tail of first request, if necessary 192161007b31SStefan Hajnoczi if (qiov->size < reqs[outidx].qiov->size) { 192261007b31SStefan Hajnoczi qemu_iovec_concat(qiov, reqs[outidx].qiov, qiov->size, 192361007b31SStefan Hajnoczi reqs[outidx].qiov->size - qiov->size); 192461007b31SStefan Hajnoczi } 192561007b31SStefan Hajnoczi 192661007b31SStefan Hajnoczi reqs[outidx].nb_sectors = qiov->size >> 9; 192761007b31SStefan Hajnoczi reqs[outidx].qiov = qiov; 192861007b31SStefan Hajnoczi 192961007b31SStefan Hajnoczi mcb->callbacks[i].free_qiov = reqs[outidx].qiov; 193061007b31SStefan Hajnoczi } else { 193161007b31SStefan Hajnoczi outidx++; 193261007b31SStefan Hajnoczi reqs[outidx].sector = reqs[i].sector; 193361007b31SStefan Hajnoczi reqs[outidx].nb_sectors = reqs[i].nb_sectors; 193461007b31SStefan Hajnoczi reqs[outidx].qiov = reqs[i].qiov; 193561007b31SStefan Hajnoczi } 193661007b31SStefan Hajnoczi } 193761007b31SStefan Hajnoczi 19387f0e9da6SMax Reitz if (bs->blk) { 19397f0e9da6SMax Reitz block_acct_merge_done(blk_get_stats(bs->blk), BLOCK_ACCT_WRITE, 19407f0e9da6SMax Reitz num_reqs - outidx - 1); 19417f0e9da6SMax Reitz } 194261007b31SStefan Hajnoczi 194361007b31SStefan Hajnoczi return outidx + 1; 194461007b31SStefan Hajnoczi } 194561007b31SStefan Hajnoczi 194661007b31SStefan Hajnoczi /* 194761007b31SStefan Hajnoczi * Submit multiple AIO write requests at once. 194861007b31SStefan Hajnoczi * 194961007b31SStefan Hajnoczi * On success, the function returns 0 and all requests in the reqs array have 195061007b31SStefan Hajnoczi * been submitted. In error case this function returns -1, and any of the 195161007b31SStefan Hajnoczi * requests may or may not be submitted yet. In particular, this means that the 195261007b31SStefan Hajnoczi * callback will be called for some of the requests, for others it won't. The 195361007b31SStefan Hajnoczi * caller must check the error field of the BlockRequest to wait for the right 195461007b31SStefan Hajnoczi * callbacks (if error != 0, no callback will be called). 195561007b31SStefan Hajnoczi * 195661007b31SStefan Hajnoczi * The implementation may modify the contents of the reqs array, e.g. to merge 195761007b31SStefan Hajnoczi * requests. However, the fields opaque and error are left unmodified as they 195861007b31SStefan Hajnoczi * are used to signal failure for a single request to the caller. 195961007b31SStefan Hajnoczi */ 196061007b31SStefan Hajnoczi int bdrv_aio_multiwrite(BlockDriverState *bs, BlockRequest *reqs, int num_reqs) 196161007b31SStefan Hajnoczi { 196261007b31SStefan Hajnoczi MultiwriteCB *mcb; 196361007b31SStefan Hajnoczi int i; 196461007b31SStefan Hajnoczi 196561007b31SStefan Hajnoczi /* don't submit writes if we don't have a medium */ 196661007b31SStefan Hajnoczi if (bs->drv == NULL) { 196761007b31SStefan Hajnoczi for (i = 0; i < num_reqs; i++) { 196861007b31SStefan Hajnoczi reqs[i].error = -ENOMEDIUM; 196961007b31SStefan Hajnoczi } 197061007b31SStefan Hajnoczi return -1; 197161007b31SStefan Hajnoczi } 197261007b31SStefan Hajnoczi 197361007b31SStefan Hajnoczi if (num_reqs == 0) { 197461007b31SStefan Hajnoczi return 0; 197561007b31SStefan Hajnoczi } 197661007b31SStefan Hajnoczi 197761007b31SStefan Hajnoczi // Create MultiwriteCB structure 197861007b31SStefan Hajnoczi mcb = g_malloc0(sizeof(*mcb) + num_reqs * sizeof(*mcb->callbacks)); 197961007b31SStefan Hajnoczi mcb->num_requests = 0; 198061007b31SStefan Hajnoczi mcb->num_callbacks = num_reqs; 198161007b31SStefan Hajnoczi 198261007b31SStefan Hajnoczi for (i = 0; i < num_reqs; i++) { 198361007b31SStefan Hajnoczi mcb->callbacks[i].cb = reqs[i].cb; 198461007b31SStefan Hajnoczi mcb->callbacks[i].opaque = reqs[i].opaque; 198561007b31SStefan Hajnoczi } 198661007b31SStefan Hajnoczi 198761007b31SStefan Hajnoczi // Check for mergable requests 198861007b31SStefan Hajnoczi num_reqs = multiwrite_merge(bs, reqs, num_reqs, mcb); 198961007b31SStefan Hajnoczi 199061007b31SStefan Hajnoczi trace_bdrv_aio_multiwrite(mcb, mcb->num_callbacks, num_reqs); 199161007b31SStefan Hajnoczi 199261007b31SStefan Hajnoczi /* Run the aio requests. */ 199361007b31SStefan Hajnoczi mcb->num_requests = num_reqs; 199461007b31SStefan Hajnoczi for (i = 0; i < num_reqs; i++) { 199561007b31SStefan Hajnoczi bdrv_co_aio_rw_vector(bs, reqs[i].sector, reqs[i].qiov, 199661007b31SStefan Hajnoczi reqs[i].nb_sectors, reqs[i].flags, 199761007b31SStefan Hajnoczi multiwrite_cb, mcb, 199861007b31SStefan Hajnoczi true); 199961007b31SStefan Hajnoczi } 200061007b31SStefan Hajnoczi 200161007b31SStefan Hajnoczi return 0; 200261007b31SStefan Hajnoczi } 200361007b31SStefan Hajnoczi 200461007b31SStefan Hajnoczi void bdrv_aio_cancel(BlockAIOCB *acb) 200561007b31SStefan Hajnoczi { 200661007b31SStefan Hajnoczi qemu_aio_ref(acb); 200761007b31SStefan Hajnoczi bdrv_aio_cancel_async(acb); 200861007b31SStefan Hajnoczi while (acb->refcnt > 1) { 200961007b31SStefan Hajnoczi if (acb->aiocb_info->get_aio_context) { 201061007b31SStefan Hajnoczi aio_poll(acb->aiocb_info->get_aio_context(acb), true); 201161007b31SStefan Hajnoczi } else if (acb->bs) { 201261007b31SStefan Hajnoczi aio_poll(bdrv_get_aio_context(acb->bs), true); 201361007b31SStefan Hajnoczi } else { 201461007b31SStefan Hajnoczi abort(); 201561007b31SStefan Hajnoczi } 201661007b31SStefan Hajnoczi } 201761007b31SStefan Hajnoczi qemu_aio_unref(acb); 201861007b31SStefan Hajnoczi } 201961007b31SStefan Hajnoczi 202061007b31SStefan Hajnoczi /* Async version of aio cancel. The caller is not blocked if the acb implements 202161007b31SStefan Hajnoczi * cancel_async, otherwise we do nothing and let the request normally complete. 202261007b31SStefan Hajnoczi * In either case the completion callback must be called. */ 202361007b31SStefan Hajnoczi void bdrv_aio_cancel_async(BlockAIOCB *acb) 202461007b31SStefan Hajnoczi { 202561007b31SStefan Hajnoczi if (acb->aiocb_info->cancel_async) { 202661007b31SStefan Hajnoczi acb->aiocb_info->cancel_async(acb); 202761007b31SStefan Hajnoczi } 202861007b31SStefan Hajnoczi } 202961007b31SStefan Hajnoczi 203061007b31SStefan Hajnoczi /**************************************************************/ 203161007b31SStefan Hajnoczi /* async block device emulation */ 203261007b31SStefan Hajnoczi 203361007b31SStefan Hajnoczi typedef struct BlockAIOCBSync { 203461007b31SStefan Hajnoczi BlockAIOCB common; 203561007b31SStefan Hajnoczi QEMUBH *bh; 203661007b31SStefan Hajnoczi int ret; 203761007b31SStefan Hajnoczi /* vector translation state */ 203861007b31SStefan Hajnoczi QEMUIOVector *qiov; 203961007b31SStefan Hajnoczi uint8_t *bounce; 204061007b31SStefan Hajnoczi int is_write; 204161007b31SStefan Hajnoczi } BlockAIOCBSync; 204261007b31SStefan Hajnoczi 204361007b31SStefan Hajnoczi static const AIOCBInfo bdrv_em_aiocb_info = { 204461007b31SStefan Hajnoczi .aiocb_size = sizeof(BlockAIOCBSync), 204561007b31SStefan Hajnoczi }; 204661007b31SStefan Hajnoczi 204761007b31SStefan Hajnoczi static void bdrv_aio_bh_cb(void *opaque) 204861007b31SStefan Hajnoczi { 204961007b31SStefan Hajnoczi BlockAIOCBSync *acb = opaque; 205061007b31SStefan Hajnoczi 205161007b31SStefan Hajnoczi if (!acb->is_write && acb->ret >= 0) { 205261007b31SStefan Hajnoczi qemu_iovec_from_buf(acb->qiov, 0, acb->bounce, acb->qiov->size); 205361007b31SStefan Hajnoczi } 205461007b31SStefan Hajnoczi qemu_vfree(acb->bounce); 205561007b31SStefan Hajnoczi acb->common.cb(acb->common.opaque, acb->ret); 205661007b31SStefan Hajnoczi qemu_bh_delete(acb->bh); 205761007b31SStefan Hajnoczi acb->bh = NULL; 205861007b31SStefan Hajnoczi qemu_aio_unref(acb); 205961007b31SStefan Hajnoczi } 206061007b31SStefan Hajnoczi 206161007b31SStefan Hajnoczi static BlockAIOCB *bdrv_aio_rw_vector(BlockDriverState *bs, 206261007b31SStefan Hajnoczi int64_t sector_num, 206361007b31SStefan Hajnoczi QEMUIOVector *qiov, 206461007b31SStefan Hajnoczi int nb_sectors, 206561007b31SStefan Hajnoczi BlockCompletionFunc *cb, 206661007b31SStefan Hajnoczi void *opaque, 206761007b31SStefan Hajnoczi int is_write) 206861007b31SStefan Hajnoczi 206961007b31SStefan Hajnoczi { 207061007b31SStefan Hajnoczi BlockAIOCBSync *acb; 207161007b31SStefan Hajnoczi 207261007b31SStefan Hajnoczi acb = qemu_aio_get(&bdrv_em_aiocb_info, bs, cb, opaque); 207361007b31SStefan Hajnoczi acb->is_write = is_write; 207461007b31SStefan Hajnoczi acb->qiov = qiov; 207561007b31SStefan Hajnoczi acb->bounce = qemu_try_blockalign(bs, qiov->size); 207661007b31SStefan Hajnoczi acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_aio_bh_cb, acb); 207761007b31SStefan Hajnoczi 207861007b31SStefan Hajnoczi if (acb->bounce == NULL) { 207961007b31SStefan Hajnoczi acb->ret = -ENOMEM; 208061007b31SStefan Hajnoczi } else if (is_write) { 208161007b31SStefan Hajnoczi qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size); 208261007b31SStefan Hajnoczi acb->ret = bs->drv->bdrv_write(bs, sector_num, acb->bounce, nb_sectors); 208361007b31SStefan Hajnoczi } else { 208461007b31SStefan Hajnoczi acb->ret = bs->drv->bdrv_read(bs, sector_num, acb->bounce, nb_sectors); 208561007b31SStefan Hajnoczi } 208661007b31SStefan Hajnoczi 208761007b31SStefan Hajnoczi qemu_bh_schedule(acb->bh); 208861007b31SStefan Hajnoczi 208961007b31SStefan Hajnoczi return &acb->common; 209061007b31SStefan Hajnoczi } 209161007b31SStefan Hajnoczi 209261007b31SStefan Hajnoczi static BlockAIOCB *bdrv_aio_readv_em(BlockDriverState *bs, 209361007b31SStefan Hajnoczi int64_t sector_num, QEMUIOVector *qiov, int nb_sectors, 209461007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque) 209561007b31SStefan Hajnoczi { 209661007b31SStefan Hajnoczi return bdrv_aio_rw_vector(bs, sector_num, qiov, nb_sectors, cb, opaque, 0); 209761007b31SStefan Hajnoczi } 209861007b31SStefan Hajnoczi 209961007b31SStefan Hajnoczi static BlockAIOCB *bdrv_aio_writev_em(BlockDriverState *bs, 210061007b31SStefan Hajnoczi int64_t sector_num, QEMUIOVector *qiov, int nb_sectors, 210161007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque) 210261007b31SStefan Hajnoczi { 210361007b31SStefan Hajnoczi return bdrv_aio_rw_vector(bs, sector_num, qiov, nb_sectors, cb, opaque, 1); 210461007b31SStefan Hajnoczi } 210561007b31SStefan Hajnoczi 210661007b31SStefan Hajnoczi 210761007b31SStefan Hajnoczi typedef struct BlockAIOCBCoroutine { 210861007b31SStefan Hajnoczi BlockAIOCB common; 210961007b31SStefan Hajnoczi BlockRequest req; 211061007b31SStefan Hajnoczi bool is_write; 211161007b31SStefan Hajnoczi bool need_bh; 211261007b31SStefan Hajnoczi bool *done; 211361007b31SStefan Hajnoczi QEMUBH* bh; 211461007b31SStefan Hajnoczi } BlockAIOCBCoroutine; 211561007b31SStefan Hajnoczi 211661007b31SStefan Hajnoczi static const AIOCBInfo bdrv_em_co_aiocb_info = { 211761007b31SStefan Hajnoczi .aiocb_size = sizeof(BlockAIOCBCoroutine), 211861007b31SStefan Hajnoczi }; 211961007b31SStefan Hajnoczi 212061007b31SStefan Hajnoczi static void bdrv_co_complete(BlockAIOCBCoroutine *acb) 212161007b31SStefan Hajnoczi { 212261007b31SStefan Hajnoczi if (!acb->need_bh) { 212361007b31SStefan Hajnoczi acb->common.cb(acb->common.opaque, acb->req.error); 212461007b31SStefan Hajnoczi qemu_aio_unref(acb); 212561007b31SStefan Hajnoczi } 212661007b31SStefan Hajnoczi } 212761007b31SStefan Hajnoczi 212861007b31SStefan Hajnoczi static void bdrv_co_em_bh(void *opaque) 212961007b31SStefan Hajnoczi { 213061007b31SStefan Hajnoczi BlockAIOCBCoroutine *acb = opaque; 213161007b31SStefan Hajnoczi 213261007b31SStefan Hajnoczi assert(!acb->need_bh); 213361007b31SStefan Hajnoczi qemu_bh_delete(acb->bh); 213461007b31SStefan Hajnoczi bdrv_co_complete(acb); 213561007b31SStefan Hajnoczi } 213661007b31SStefan Hajnoczi 213761007b31SStefan Hajnoczi static void bdrv_co_maybe_schedule_bh(BlockAIOCBCoroutine *acb) 213861007b31SStefan Hajnoczi { 213961007b31SStefan Hajnoczi acb->need_bh = false; 214061007b31SStefan Hajnoczi if (acb->req.error != -EINPROGRESS) { 214161007b31SStefan Hajnoczi BlockDriverState *bs = acb->common.bs; 214261007b31SStefan Hajnoczi 214361007b31SStefan Hajnoczi acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb); 214461007b31SStefan Hajnoczi qemu_bh_schedule(acb->bh); 214561007b31SStefan Hajnoczi } 214661007b31SStefan Hajnoczi } 214761007b31SStefan Hajnoczi 214861007b31SStefan Hajnoczi /* Invoke bdrv_co_do_readv/bdrv_co_do_writev */ 214961007b31SStefan Hajnoczi static void coroutine_fn bdrv_co_do_rw(void *opaque) 215061007b31SStefan Hajnoczi { 215161007b31SStefan Hajnoczi BlockAIOCBCoroutine *acb = opaque; 215261007b31SStefan Hajnoczi BlockDriverState *bs = acb->common.bs; 215361007b31SStefan Hajnoczi 215461007b31SStefan Hajnoczi if (!acb->is_write) { 215561007b31SStefan Hajnoczi acb->req.error = bdrv_co_do_readv(bs, acb->req.sector, 215661007b31SStefan Hajnoczi acb->req.nb_sectors, acb->req.qiov, acb->req.flags); 215761007b31SStefan Hajnoczi } else { 215861007b31SStefan Hajnoczi acb->req.error = bdrv_co_do_writev(bs, acb->req.sector, 215961007b31SStefan Hajnoczi acb->req.nb_sectors, acb->req.qiov, acb->req.flags); 216061007b31SStefan Hajnoczi } 216161007b31SStefan Hajnoczi 216261007b31SStefan Hajnoczi bdrv_co_complete(acb); 216361007b31SStefan Hajnoczi } 216461007b31SStefan Hajnoczi 216561007b31SStefan Hajnoczi static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs, 216661007b31SStefan Hajnoczi int64_t sector_num, 216761007b31SStefan Hajnoczi QEMUIOVector *qiov, 216861007b31SStefan Hajnoczi int nb_sectors, 216961007b31SStefan Hajnoczi BdrvRequestFlags flags, 217061007b31SStefan Hajnoczi BlockCompletionFunc *cb, 217161007b31SStefan Hajnoczi void *opaque, 217261007b31SStefan Hajnoczi bool is_write) 217361007b31SStefan Hajnoczi { 217461007b31SStefan Hajnoczi Coroutine *co; 217561007b31SStefan Hajnoczi BlockAIOCBCoroutine *acb; 217661007b31SStefan Hajnoczi 217761007b31SStefan Hajnoczi acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque); 217861007b31SStefan Hajnoczi acb->need_bh = true; 217961007b31SStefan Hajnoczi acb->req.error = -EINPROGRESS; 218061007b31SStefan Hajnoczi acb->req.sector = sector_num; 218161007b31SStefan Hajnoczi acb->req.nb_sectors = nb_sectors; 218261007b31SStefan Hajnoczi acb->req.qiov = qiov; 218361007b31SStefan Hajnoczi acb->req.flags = flags; 218461007b31SStefan Hajnoczi acb->is_write = is_write; 218561007b31SStefan Hajnoczi 218661007b31SStefan Hajnoczi co = qemu_coroutine_create(bdrv_co_do_rw); 218761007b31SStefan Hajnoczi qemu_coroutine_enter(co, acb); 218861007b31SStefan Hajnoczi 218961007b31SStefan Hajnoczi bdrv_co_maybe_schedule_bh(acb); 219061007b31SStefan Hajnoczi return &acb->common; 219161007b31SStefan Hajnoczi } 219261007b31SStefan Hajnoczi 219361007b31SStefan Hajnoczi static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque) 219461007b31SStefan Hajnoczi { 219561007b31SStefan Hajnoczi BlockAIOCBCoroutine *acb = opaque; 219661007b31SStefan Hajnoczi BlockDriverState *bs = acb->common.bs; 219761007b31SStefan Hajnoczi 219861007b31SStefan Hajnoczi acb->req.error = bdrv_co_flush(bs); 219961007b31SStefan Hajnoczi bdrv_co_complete(acb); 220061007b31SStefan Hajnoczi } 220161007b31SStefan Hajnoczi 220261007b31SStefan Hajnoczi BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs, 220361007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque) 220461007b31SStefan Hajnoczi { 220561007b31SStefan Hajnoczi trace_bdrv_aio_flush(bs, opaque); 220661007b31SStefan Hajnoczi 220761007b31SStefan Hajnoczi Coroutine *co; 220861007b31SStefan Hajnoczi BlockAIOCBCoroutine *acb; 220961007b31SStefan Hajnoczi 221061007b31SStefan Hajnoczi acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque); 221161007b31SStefan Hajnoczi acb->need_bh = true; 221261007b31SStefan Hajnoczi acb->req.error = -EINPROGRESS; 221361007b31SStefan Hajnoczi 221461007b31SStefan Hajnoczi co = qemu_coroutine_create(bdrv_aio_flush_co_entry); 221561007b31SStefan Hajnoczi qemu_coroutine_enter(co, acb); 221661007b31SStefan Hajnoczi 221761007b31SStefan Hajnoczi bdrv_co_maybe_schedule_bh(acb); 221861007b31SStefan Hajnoczi return &acb->common; 221961007b31SStefan Hajnoczi } 222061007b31SStefan Hajnoczi 222161007b31SStefan Hajnoczi static void coroutine_fn bdrv_aio_discard_co_entry(void *opaque) 222261007b31SStefan Hajnoczi { 222361007b31SStefan Hajnoczi BlockAIOCBCoroutine *acb = opaque; 222461007b31SStefan Hajnoczi BlockDriverState *bs = acb->common.bs; 222561007b31SStefan Hajnoczi 222661007b31SStefan Hajnoczi acb->req.error = bdrv_co_discard(bs, acb->req.sector, acb->req.nb_sectors); 222761007b31SStefan Hajnoczi bdrv_co_complete(acb); 222861007b31SStefan Hajnoczi } 222961007b31SStefan Hajnoczi 223061007b31SStefan Hajnoczi BlockAIOCB *bdrv_aio_discard(BlockDriverState *bs, 223161007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, 223261007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque) 223361007b31SStefan Hajnoczi { 223461007b31SStefan Hajnoczi Coroutine *co; 223561007b31SStefan Hajnoczi BlockAIOCBCoroutine *acb; 223661007b31SStefan Hajnoczi 223761007b31SStefan Hajnoczi trace_bdrv_aio_discard(bs, sector_num, nb_sectors, opaque); 223861007b31SStefan Hajnoczi 223961007b31SStefan Hajnoczi acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque); 224061007b31SStefan Hajnoczi acb->need_bh = true; 224161007b31SStefan Hajnoczi acb->req.error = -EINPROGRESS; 224261007b31SStefan Hajnoczi acb->req.sector = sector_num; 224361007b31SStefan Hajnoczi acb->req.nb_sectors = nb_sectors; 224461007b31SStefan Hajnoczi co = qemu_coroutine_create(bdrv_aio_discard_co_entry); 224561007b31SStefan Hajnoczi qemu_coroutine_enter(co, acb); 224661007b31SStefan Hajnoczi 224761007b31SStefan Hajnoczi bdrv_co_maybe_schedule_bh(acb); 224861007b31SStefan Hajnoczi return &acb->common; 224961007b31SStefan Hajnoczi } 225061007b31SStefan Hajnoczi 225161007b31SStefan Hajnoczi void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs, 225261007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque) 225361007b31SStefan Hajnoczi { 225461007b31SStefan Hajnoczi BlockAIOCB *acb; 225561007b31SStefan Hajnoczi 2256c84b3192SPaolo Bonzini acb = g_malloc(aiocb_info->aiocb_size); 225761007b31SStefan Hajnoczi acb->aiocb_info = aiocb_info; 225861007b31SStefan Hajnoczi acb->bs = bs; 225961007b31SStefan Hajnoczi acb->cb = cb; 226061007b31SStefan Hajnoczi acb->opaque = opaque; 226161007b31SStefan Hajnoczi acb->refcnt = 1; 226261007b31SStefan Hajnoczi return acb; 226361007b31SStefan Hajnoczi } 226461007b31SStefan Hajnoczi 226561007b31SStefan Hajnoczi void qemu_aio_ref(void *p) 226661007b31SStefan Hajnoczi { 226761007b31SStefan Hajnoczi BlockAIOCB *acb = p; 226861007b31SStefan Hajnoczi acb->refcnt++; 226961007b31SStefan Hajnoczi } 227061007b31SStefan Hajnoczi 227161007b31SStefan Hajnoczi void qemu_aio_unref(void *p) 227261007b31SStefan Hajnoczi { 227361007b31SStefan Hajnoczi BlockAIOCB *acb = p; 227461007b31SStefan Hajnoczi assert(acb->refcnt > 0); 227561007b31SStefan Hajnoczi if (--acb->refcnt == 0) { 2276c84b3192SPaolo Bonzini g_free(acb); 227761007b31SStefan Hajnoczi } 227861007b31SStefan Hajnoczi } 227961007b31SStefan Hajnoczi 228061007b31SStefan Hajnoczi /**************************************************************/ 228161007b31SStefan Hajnoczi /* Coroutine block device emulation */ 228261007b31SStefan Hajnoczi 228361007b31SStefan Hajnoczi typedef struct CoroutineIOCompletion { 228461007b31SStefan Hajnoczi Coroutine *coroutine; 228561007b31SStefan Hajnoczi int ret; 228661007b31SStefan Hajnoczi } CoroutineIOCompletion; 228761007b31SStefan Hajnoczi 228861007b31SStefan Hajnoczi static void bdrv_co_io_em_complete(void *opaque, int ret) 228961007b31SStefan Hajnoczi { 229061007b31SStefan Hajnoczi CoroutineIOCompletion *co = opaque; 229161007b31SStefan Hajnoczi 229261007b31SStefan Hajnoczi co->ret = ret; 229361007b31SStefan Hajnoczi qemu_coroutine_enter(co->coroutine, NULL); 229461007b31SStefan Hajnoczi } 229561007b31SStefan Hajnoczi 229661007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_io_em(BlockDriverState *bs, int64_t sector_num, 229761007b31SStefan Hajnoczi int nb_sectors, QEMUIOVector *iov, 229861007b31SStefan Hajnoczi bool is_write) 229961007b31SStefan Hajnoczi { 230061007b31SStefan Hajnoczi CoroutineIOCompletion co = { 230161007b31SStefan Hajnoczi .coroutine = qemu_coroutine_self(), 230261007b31SStefan Hajnoczi }; 230361007b31SStefan Hajnoczi BlockAIOCB *acb; 230461007b31SStefan Hajnoczi 230561007b31SStefan Hajnoczi if (is_write) { 230661007b31SStefan Hajnoczi acb = bs->drv->bdrv_aio_writev(bs, sector_num, iov, nb_sectors, 230761007b31SStefan Hajnoczi bdrv_co_io_em_complete, &co); 230861007b31SStefan Hajnoczi } else { 230961007b31SStefan Hajnoczi acb = bs->drv->bdrv_aio_readv(bs, sector_num, iov, nb_sectors, 231061007b31SStefan Hajnoczi bdrv_co_io_em_complete, &co); 231161007b31SStefan Hajnoczi } 231261007b31SStefan Hajnoczi 231361007b31SStefan Hajnoczi trace_bdrv_co_io_em(bs, sector_num, nb_sectors, is_write, acb); 231461007b31SStefan Hajnoczi if (!acb) { 231561007b31SStefan Hajnoczi return -EIO; 231661007b31SStefan Hajnoczi } 231761007b31SStefan Hajnoczi qemu_coroutine_yield(); 231861007b31SStefan Hajnoczi 231961007b31SStefan Hajnoczi return co.ret; 232061007b31SStefan Hajnoczi } 232161007b31SStefan Hajnoczi 232261007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_readv_em(BlockDriverState *bs, 232361007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, 232461007b31SStefan Hajnoczi QEMUIOVector *iov) 232561007b31SStefan Hajnoczi { 232661007b31SStefan Hajnoczi return bdrv_co_io_em(bs, sector_num, nb_sectors, iov, false); 232761007b31SStefan Hajnoczi } 232861007b31SStefan Hajnoczi 232961007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_writev_em(BlockDriverState *bs, 233061007b31SStefan Hajnoczi int64_t sector_num, int nb_sectors, 233161007b31SStefan Hajnoczi QEMUIOVector *iov) 233261007b31SStefan Hajnoczi { 233361007b31SStefan Hajnoczi return bdrv_co_io_em(bs, sector_num, nb_sectors, iov, true); 233461007b31SStefan Hajnoczi } 233561007b31SStefan Hajnoczi 233661007b31SStefan Hajnoczi static void coroutine_fn bdrv_flush_co_entry(void *opaque) 233761007b31SStefan Hajnoczi { 233861007b31SStefan Hajnoczi RwCo *rwco = opaque; 233961007b31SStefan Hajnoczi 234061007b31SStefan Hajnoczi rwco->ret = bdrv_co_flush(rwco->bs); 234161007b31SStefan Hajnoczi } 234261007b31SStefan Hajnoczi 234361007b31SStefan Hajnoczi int coroutine_fn bdrv_co_flush(BlockDriverState *bs) 234461007b31SStefan Hajnoczi { 234561007b31SStefan Hajnoczi int ret; 2346cdb5e315SFam Zheng BdrvTrackedRequest req; 234761007b31SStefan Hajnoczi 23481b6bc94dSDimitris Aragiorgis if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs) || 23491b6bc94dSDimitris Aragiorgis bdrv_is_sg(bs)) { 235061007b31SStefan Hajnoczi return 0; 235161007b31SStefan Hajnoczi } 235261007b31SStefan Hajnoczi 2353cdb5e315SFam Zheng tracked_request_begin(&req, bs, 0, 0, BDRV_TRACKED_FLUSH); 235461007b31SStefan Hajnoczi /* Write back cached data to the OS even with cache=unsafe */ 235561007b31SStefan Hajnoczi BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS); 235661007b31SStefan Hajnoczi if (bs->drv->bdrv_co_flush_to_os) { 235761007b31SStefan Hajnoczi ret = bs->drv->bdrv_co_flush_to_os(bs); 235861007b31SStefan Hajnoczi if (ret < 0) { 2359cdb5e315SFam Zheng goto out; 236061007b31SStefan Hajnoczi } 236161007b31SStefan Hajnoczi } 236261007b31SStefan Hajnoczi 236361007b31SStefan Hajnoczi /* But don't actually force it to the disk with cache=unsafe */ 236461007b31SStefan Hajnoczi if (bs->open_flags & BDRV_O_NO_FLUSH) { 236561007b31SStefan Hajnoczi goto flush_parent; 236661007b31SStefan Hajnoczi } 236761007b31SStefan Hajnoczi 236861007b31SStefan Hajnoczi BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK); 236961007b31SStefan Hajnoczi if (bs->drv->bdrv_co_flush_to_disk) { 237061007b31SStefan Hajnoczi ret = bs->drv->bdrv_co_flush_to_disk(bs); 237161007b31SStefan Hajnoczi } else if (bs->drv->bdrv_aio_flush) { 237261007b31SStefan Hajnoczi BlockAIOCB *acb; 237361007b31SStefan Hajnoczi CoroutineIOCompletion co = { 237461007b31SStefan Hajnoczi .coroutine = qemu_coroutine_self(), 237561007b31SStefan Hajnoczi }; 237661007b31SStefan Hajnoczi 237761007b31SStefan Hajnoczi acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co); 237861007b31SStefan Hajnoczi if (acb == NULL) { 237961007b31SStefan Hajnoczi ret = -EIO; 238061007b31SStefan Hajnoczi } else { 238161007b31SStefan Hajnoczi qemu_coroutine_yield(); 238261007b31SStefan Hajnoczi ret = co.ret; 238361007b31SStefan Hajnoczi } 238461007b31SStefan Hajnoczi } else { 238561007b31SStefan Hajnoczi /* 238661007b31SStefan Hajnoczi * Some block drivers always operate in either writethrough or unsafe 238761007b31SStefan Hajnoczi * mode and don't support bdrv_flush therefore. Usually qemu doesn't 238861007b31SStefan Hajnoczi * know how the server works (because the behaviour is hardcoded or 238961007b31SStefan Hajnoczi * depends on server-side configuration), so we can't ensure that 239061007b31SStefan Hajnoczi * everything is safe on disk. Returning an error doesn't work because 239161007b31SStefan Hajnoczi * that would break guests even if the server operates in writethrough 239261007b31SStefan Hajnoczi * mode. 239361007b31SStefan Hajnoczi * 239461007b31SStefan Hajnoczi * Let's hope the user knows what he's doing. 239561007b31SStefan Hajnoczi */ 239661007b31SStefan Hajnoczi ret = 0; 239761007b31SStefan Hajnoczi } 239861007b31SStefan Hajnoczi if (ret < 0) { 2399cdb5e315SFam Zheng goto out; 240061007b31SStefan Hajnoczi } 240161007b31SStefan Hajnoczi 240261007b31SStefan Hajnoczi /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH 240361007b31SStefan Hajnoczi * in the case of cache=unsafe, so there are no useless flushes. 240461007b31SStefan Hajnoczi */ 240561007b31SStefan Hajnoczi flush_parent: 2406cdb5e315SFam Zheng ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0; 2407cdb5e315SFam Zheng out: 2408cdb5e315SFam Zheng tracked_request_end(&req); 2409cdb5e315SFam Zheng return ret; 241061007b31SStefan Hajnoczi } 241161007b31SStefan Hajnoczi 241261007b31SStefan Hajnoczi int bdrv_flush(BlockDriverState *bs) 241361007b31SStefan Hajnoczi { 241461007b31SStefan Hajnoczi Coroutine *co; 241561007b31SStefan Hajnoczi RwCo rwco = { 241661007b31SStefan Hajnoczi .bs = bs, 241761007b31SStefan Hajnoczi .ret = NOT_DONE, 241861007b31SStefan Hajnoczi }; 241961007b31SStefan Hajnoczi 242061007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 242161007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 242261007b31SStefan Hajnoczi bdrv_flush_co_entry(&rwco); 242361007b31SStefan Hajnoczi } else { 242461007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 242561007b31SStefan Hajnoczi 242661007b31SStefan Hajnoczi co = qemu_coroutine_create(bdrv_flush_co_entry); 242761007b31SStefan Hajnoczi qemu_coroutine_enter(co, &rwco); 242861007b31SStefan Hajnoczi while (rwco.ret == NOT_DONE) { 242961007b31SStefan Hajnoczi aio_poll(aio_context, true); 243061007b31SStefan Hajnoczi } 243161007b31SStefan Hajnoczi } 243261007b31SStefan Hajnoczi 243361007b31SStefan Hajnoczi return rwco.ret; 243461007b31SStefan Hajnoczi } 243561007b31SStefan Hajnoczi 243661007b31SStefan Hajnoczi typedef struct DiscardCo { 243761007b31SStefan Hajnoczi BlockDriverState *bs; 243861007b31SStefan Hajnoczi int64_t sector_num; 243961007b31SStefan Hajnoczi int nb_sectors; 244061007b31SStefan Hajnoczi int ret; 244161007b31SStefan Hajnoczi } DiscardCo; 244261007b31SStefan Hajnoczi static void coroutine_fn bdrv_discard_co_entry(void *opaque) 244361007b31SStefan Hajnoczi { 244461007b31SStefan Hajnoczi DiscardCo *rwco = opaque; 244561007b31SStefan Hajnoczi 244661007b31SStefan Hajnoczi rwco->ret = bdrv_co_discard(rwco->bs, rwco->sector_num, rwco->nb_sectors); 244761007b31SStefan Hajnoczi } 244861007b31SStefan Hajnoczi 244961007b31SStefan Hajnoczi int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num, 245061007b31SStefan Hajnoczi int nb_sectors) 245161007b31SStefan Hajnoczi { 2452b1066c87SFam Zheng BdrvTrackedRequest req; 245361007b31SStefan Hajnoczi int max_discard, ret; 245461007b31SStefan Hajnoczi 245561007b31SStefan Hajnoczi if (!bs->drv) { 245661007b31SStefan Hajnoczi return -ENOMEDIUM; 245761007b31SStefan Hajnoczi } 245861007b31SStefan Hajnoczi 245961007b31SStefan Hajnoczi ret = bdrv_check_request(bs, sector_num, nb_sectors); 246061007b31SStefan Hajnoczi if (ret < 0) { 246161007b31SStefan Hajnoczi return ret; 246261007b31SStefan Hajnoczi } else if (bs->read_only) { 2463eaf5fe2dSPaolo Bonzini return -EPERM; 246461007b31SStefan Hajnoczi } 246561007b31SStefan Hajnoczi 246661007b31SStefan Hajnoczi /* Do nothing if disabled. */ 246761007b31SStefan Hajnoczi if (!(bs->open_flags & BDRV_O_UNMAP)) { 246861007b31SStefan Hajnoczi return 0; 246961007b31SStefan Hajnoczi } 247061007b31SStefan Hajnoczi 247161007b31SStefan Hajnoczi if (!bs->drv->bdrv_co_discard && !bs->drv->bdrv_aio_discard) { 247261007b31SStefan Hajnoczi return 0; 247361007b31SStefan Hajnoczi } 247461007b31SStefan Hajnoczi 2475b1066c87SFam Zheng tracked_request_begin(&req, bs, sector_num, nb_sectors, 2476b1066c87SFam Zheng BDRV_TRACKED_DISCARD); 247750824995SFam Zheng bdrv_set_dirty(bs, sector_num, nb_sectors); 247850824995SFam Zheng 247961007b31SStefan Hajnoczi max_discard = MIN_NON_ZERO(bs->bl.max_discard, BDRV_REQUEST_MAX_SECTORS); 248061007b31SStefan Hajnoczi while (nb_sectors > 0) { 248161007b31SStefan Hajnoczi int ret; 248261007b31SStefan Hajnoczi int num = nb_sectors; 248361007b31SStefan Hajnoczi 248461007b31SStefan Hajnoczi /* align request */ 248561007b31SStefan Hajnoczi if (bs->bl.discard_alignment && 248661007b31SStefan Hajnoczi num >= bs->bl.discard_alignment && 248761007b31SStefan Hajnoczi sector_num % bs->bl.discard_alignment) { 248861007b31SStefan Hajnoczi if (num > bs->bl.discard_alignment) { 248961007b31SStefan Hajnoczi num = bs->bl.discard_alignment; 249061007b31SStefan Hajnoczi } 249161007b31SStefan Hajnoczi num -= sector_num % bs->bl.discard_alignment; 249261007b31SStefan Hajnoczi } 249361007b31SStefan Hajnoczi 249461007b31SStefan Hajnoczi /* limit request size */ 249561007b31SStefan Hajnoczi if (num > max_discard) { 249661007b31SStefan Hajnoczi num = max_discard; 249761007b31SStefan Hajnoczi } 249861007b31SStefan Hajnoczi 249961007b31SStefan Hajnoczi if (bs->drv->bdrv_co_discard) { 250061007b31SStefan Hajnoczi ret = bs->drv->bdrv_co_discard(bs, sector_num, num); 250161007b31SStefan Hajnoczi } else { 250261007b31SStefan Hajnoczi BlockAIOCB *acb; 250361007b31SStefan Hajnoczi CoroutineIOCompletion co = { 250461007b31SStefan Hajnoczi .coroutine = qemu_coroutine_self(), 250561007b31SStefan Hajnoczi }; 250661007b31SStefan Hajnoczi 250761007b31SStefan Hajnoczi acb = bs->drv->bdrv_aio_discard(bs, sector_num, nb_sectors, 250861007b31SStefan Hajnoczi bdrv_co_io_em_complete, &co); 250961007b31SStefan Hajnoczi if (acb == NULL) { 2510b1066c87SFam Zheng ret = -EIO; 2511b1066c87SFam Zheng goto out; 251261007b31SStefan Hajnoczi } else { 251361007b31SStefan Hajnoczi qemu_coroutine_yield(); 251461007b31SStefan Hajnoczi ret = co.ret; 251561007b31SStefan Hajnoczi } 251661007b31SStefan Hajnoczi } 251761007b31SStefan Hajnoczi if (ret && ret != -ENOTSUP) { 2518b1066c87SFam Zheng goto out; 251961007b31SStefan Hajnoczi } 252061007b31SStefan Hajnoczi 252161007b31SStefan Hajnoczi sector_num += num; 252261007b31SStefan Hajnoczi nb_sectors -= num; 252361007b31SStefan Hajnoczi } 2524b1066c87SFam Zheng ret = 0; 2525b1066c87SFam Zheng out: 2526b1066c87SFam Zheng tracked_request_end(&req); 2527b1066c87SFam Zheng return ret; 252861007b31SStefan Hajnoczi } 252961007b31SStefan Hajnoczi 253061007b31SStefan Hajnoczi int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors) 253161007b31SStefan Hajnoczi { 253261007b31SStefan Hajnoczi Coroutine *co; 253361007b31SStefan Hajnoczi DiscardCo rwco = { 253461007b31SStefan Hajnoczi .bs = bs, 253561007b31SStefan Hajnoczi .sector_num = sector_num, 253661007b31SStefan Hajnoczi .nb_sectors = nb_sectors, 253761007b31SStefan Hajnoczi .ret = NOT_DONE, 253861007b31SStefan Hajnoczi }; 253961007b31SStefan Hajnoczi 254061007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 254161007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 254261007b31SStefan Hajnoczi bdrv_discard_co_entry(&rwco); 254361007b31SStefan Hajnoczi } else { 254461007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 254561007b31SStefan Hajnoczi 254661007b31SStefan Hajnoczi co = qemu_coroutine_create(bdrv_discard_co_entry); 254761007b31SStefan Hajnoczi qemu_coroutine_enter(co, &rwco); 254861007b31SStefan Hajnoczi while (rwco.ret == NOT_DONE) { 254961007b31SStefan Hajnoczi aio_poll(aio_context, true); 255061007b31SStefan Hajnoczi } 255161007b31SStefan Hajnoczi } 255261007b31SStefan Hajnoczi 255361007b31SStefan Hajnoczi return rwco.ret; 255461007b31SStefan Hajnoczi } 255561007b31SStefan Hajnoczi 25565c5ae76aSFam Zheng typedef struct { 25575c5ae76aSFam Zheng CoroutineIOCompletion *co; 25585c5ae76aSFam Zheng QEMUBH *bh; 25595c5ae76aSFam Zheng } BdrvIoctlCompletionData; 256061007b31SStefan Hajnoczi 25615c5ae76aSFam Zheng static void bdrv_ioctl_bh_cb(void *opaque) 25625c5ae76aSFam Zheng { 25635c5ae76aSFam Zheng BdrvIoctlCompletionData *data = opaque; 25645c5ae76aSFam Zheng 25655c5ae76aSFam Zheng bdrv_co_io_em_complete(data->co, -ENOTSUP); 25665c5ae76aSFam Zheng qemu_bh_delete(data->bh); 25675c5ae76aSFam Zheng } 25685c5ae76aSFam Zheng 25695c5ae76aSFam Zheng static int bdrv_co_do_ioctl(BlockDriverState *bs, int req, void *buf) 257061007b31SStefan Hajnoczi { 257161007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 25725c5ae76aSFam Zheng BdrvTrackedRequest tracked_req; 25735c5ae76aSFam Zheng CoroutineIOCompletion co = { 25745c5ae76aSFam Zheng .coroutine = qemu_coroutine_self(), 25755c5ae76aSFam Zheng }; 25765c5ae76aSFam Zheng BlockAIOCB *acb; 257761007b31SStefan Hajnoczi 25785c5ae76aSFam Zheng tracked_request_begin(&tracked_req, bs, 0, 0, BDRV_TRACKED_IOCTL); 25795c5ae76aSFam Zheng if (!drv || !drv->bdrv_aio_ioctl) { 25805c5ae76aSFam Zheng co.ret = -ENOTSUP; 25815c5ae76aSFam Zheng goto out; 25825c5ae76aSFam Zheng } 25835c5ae76aSFam Zheng 25845c5ae76aSFam Zheng acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co); 25855c5ae76aSFam Zheng if (!acb) { 25865c5ae76aSFam Zheng BdrvIoctlCompletionData *data = g_new(BdrvIoctlCompletionData, 1); 25875c5ae76aSFam Zheng data->bh = aio_bh_new(bdrv_get_aio_context(bs), 25885c5ae76aSFam Zheng bdrv_ioctl_bh_cb, data); 25895c5ae76aSFam Zheng data->co = &co; 25905c5ae76aSFam Zheng qemu_bh_schedule(data->bh); 25915c5ae76aSFam Zheng } 25925c5ae76aSFam Zheng qemu_coroutine_yield(); 25935c5ae76aSFam Zheng out: 25945c5ae76aSFam Zheng tracked_request_end(&tracked_req); 25955c5ae76aSFam Zheng return co.ret; 25965c5ae76aSFam Zheng } 25975c5ae76aSFam Zheng 25985c5ae76aSFam Zheng typedef struct { 25995c5ae76aSFam Zheng BlockDriverState *bs; 26005c5ae76aSFam Zheng int req; 26015c5ae76aSFam Zheng void *buf; 26025c5ae76aSFam Zheng int ret; 26035c5ae76aSFam Zheng } BdrvIoctlCoData; 26045c5ae76aSFam Zheng 26055c5ae76aSFam Zheng static void coroutine_fn bdrv_co_ioctl_entry(void *opaque) 26065c5ae76aSFam Zheng { 26075c5ae76aSFam Zheng BdrvIoctlCoData *data = opaque; 26085c5ae76aSFam Zheng data->ret = bdrv_co_do_ioctl(data->bs, data->req, data->buf); 26095c5ae76aSFam Zheng } 26105c5ae76aSFam Zheng 26115c5ae76aSFam Zheng /* needed for generic scsi interface */ 26125c5ae76aSFam Zheng int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf) 26135c5ae76aSFam Zheng { 26145c5ae76aSFam Zheng BdrvIoctlCoData data = { 26155c5ae76aSFam Zheng .bs = bs, 26165c5ae76aSFam Zheng .req = req, 26175c5ae76aSFam Zheng .buf = buf, 26185c5ae76aSFam Zheng .ret = -EINPROGRESS, 26195c5ae76aSFam Zheng }; 26205c5ae76aSFam Zheng 26215c5ae76aSFam Zheng if (qemu_in_coroutine()) { 26225c5ae76aSFam Zheng /* Fast-path if already in coroutine context */ 26235c5ae76aSFam Zheng bdrv_co_ioctl_entry(&data); 26245c5ae76aSFam Zheng } else { 26255c5ae76aSFam Zheng Coroutine *co = qemu_coroutine_create(bdrv_co_ioctl_entry); 2626ba889444SPaolo Bonzini 26275c5ae76aSFam Zheng qemu_coroutine_enter(co, &data); 26285c5ae76aSFam Zheng while (data.ret == -EINPROGRESS) { 26295c5ae76aSFam Zheng aio_poll(bdrv_get_aio_context(bs), true); 26305c5ae76aSFam Zheng } 2631ba889444SPaolo Bonzini } 26325c5ae76aSFam Zheng return data.ret; 26335c5ae76aSFam Zheng } 26345c5ae76aSFam Zheng 26355c5ae76aSFam Zheng static void coroutine_fn bdrv_co_aio_ioctl_entry(void *opaque) 26365c5ae76aSFam Zheng { 26375c5ae76aSFam Zheng BlockAIOCBCoroutine *acb = opaque; 26385c5ae76aSFam Zheng acb->req.error = bdrv_co_do_ioctl(acb->common.bs, 26395c5ae76aSFam Zheng acb->req.req, acb->req.buf); 26405c5ae76aSFam Zheng bdrv_co_complete(acb); 264161007b31SStefan Hajnoczi } 264261007b31SStefan Hajnoczi 264361007b31SStefan Hajnoczi BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs, 264461007b31SStefan Hajnoczi unsigned long int req, void *buf, 264561007b31SStefan Hajnoczi BlockCompletionFunc *cb, void *opaque) 264661007b31SStefan Hajnoczi { 26475c5ae76aSFam Zheng BlockAIOCBCoroutine *acb = qemu_aio_get(&bdrv_em_co_aiocb_info, 26485c5ae76aSFam Zheng bs, cb, opaque); 26495c5ae76aSFam Zheng Coroutine *co; 265061007b31SStefan Hajnoczi 26515c5ae76aSFam Zheng acb->need_bh = true; 26525c5ae76aSFam Zheng acb->req.error = -EINPROGRESS; 26535c5ae76aSFam Zheng acb->req.req = req; 26545c5ae76aSFam Zheng acb->req.buf = buf; 26555c5ae76aSFam Zheng co = qemu_coroutine_create(bdrv_co_aio_ioctl_entry); 26565c5ae76aSFam Zheng qemu_coroutine_enter(co, acb); 26575c5ae76aSFam Zheng 26585c5ae76aSFam Zheng bdrv_co_maybe_schedule_bh(acb); 26595c5ae76aSFam Zheng return &acb->common; 266061007b31SStefan Hajnoczi } 266161007b31SStefan Hajnoczi 266261007b31SStefan Hajnoczi void *qemu_blockalign(BlockDriverState *bs, size_t size) 266361007b31SStefan Hajnoczi { 266461007b31SStefan Hajnoczi return qemu_memalign(bdrv_opt_mem_align(bs), size); 266561007b31SStefan Hajnoczi } 266661007b31SStefan Hajnoczi 266761007b31SStefan Hajnoczi void *qemu_blockalign0(BlockDriverState *bs, size_t size) 266861007b31SStefan Hajnoczi { 266961007b31SStefan Hajnoczi return memset(qemu_blockalign(bs, size), 0, size); 267061007b31SStefan Hajnoczi } 267161007b31SStefan Hajnoczi 267261007b31SStefan Hajnoczi void *qemu_try_blockalign(BlockDriverState *bs, size_t size) 267361007b31SStefan Hajnoczi { 267461007b31SStefan Hajnoczi size_t align = bdrv_opt_mem_align(bs); 267561007b31SStefan Hajnoczi 267661007b31SStefan Hajnoczi /* Ensure that NULL is never returned on success */ 267761007b31SStefan Hajnoczi assert(align > 0); 267861007b31SStefan Hajnoczi if (size == 0) { 267961007b31SStefan Hajnoczi size = align; 268061007b31SStefan Hajnoczi } 268161007b31SStefan Hajnoczi 268261007b31SStefan Hajnoczi return qemu_try_memalign(align, size); 268361007b31SStefan Hajnoczi } 268461007b31SStefan Hajnoczi 268561007b31SStefan Hajnoczi void *qemu_try_blockalign0(BlockDriverState *bs, size_t size) 268661007b31SStefan Hajnoczi { 268761007b31SStefan Hajnoczi void *mem = qemu_try_blockalign(bs, size); 268861007b31SStefan Hajnoczi 268961007b31SStefan Hajnoczi if (mem) { 269061007b31SStefan Hajnoczi memset(mem, 0, size); 269161007b31SStefan Hajnoczi } 269261007b31SStefan Hajnoczi 269361007b31SStefan Hajnoczi return mem; 269461007b31SStefan Hajnoczi } 269561007b31SStefan Hajnoczi 269661007b31SStefan Hajnoczi /* 269761007b31SStefan Hajnoczi * Check if all memory in this vector is sector aligned. 269861007b31SStefan Hajnoczi */ 269961007b31SStefan Hajnoczi bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov) 270061007b31SStefan Hajnoczi { 270161007b31SStefan Hajnoczi int i; 27024196d2f0SDenis V. Lunev size_t alignment = bdrv_min_mem_align(bs); 270361007b31SStefan Hajnoczi 270461007b31SStefan Hajnoczi for (i = 0; i < qiov->niov; i++) { 270561007b31SStefan Hajnoczi if ((uintptr_t) qiov->iov[i].iov_base % alignment) { 270661007b31SStefan Hajnoczi return false; 270761007b31SStefan Hajnoczi } 270861007b31SStefan Hajnoczi if (qiov->iov[i].iov_len % alignment) { 270961007b31SStefan Hajnoczi return false; 271061007b31SStefan Hajnoczi } 271161007b31SStefan Hajnoczi } 271261007b31SStefan Hajnoczi 271361007b31SStefan Hajnoczi return true; 271461007b31SStefan Hajnoczi } 271561007b31SStefan Hajnoczi 271661007b31SStefan Hajnoczi void bdrv_add_before_write_notifier(BlockDriverState *bs, 271761007b31SStefan Hajnoczi NotifierWithReturn *notifier) 271861007b31SStefan Hajnoczi { 271961007b31SStefan Hajnoczi notifier_with_return_list_add(&bs->before_write_notifiers, notifier); 272061007b31SStefan Hajnoczi } 272161007b31SStefan Hajnoczi 272261007b31SStefan Hajnoczi void bdrv_io_plug(BlockDriverState *bs) 272361007b31SStefan Hajnoczi { 272461007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 272561007b31SStefan Hajnoczi if (drv && drv->bdrv_io_plug) { 272661007b31SStefan Hajnoczi drv->bdrv_io_plug(bs); 272761007b31SStefan Hajnoczi } else if (bs->file) { 27289a4f4c31SKevin Wolf bdrv_io_plug(bs->file->bs); 272961007b31SStefan Hajnoczi } 273061007b31SStefan Hajnoczi } 273161007b31SStefan Hajnoczi 273261007b31SStefan Hajnoczi void bdrv_io_unplug(BlockDriverState *bs) 273361007b31SStefan Hajnoczi { 273461007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 273561007b31SStefan Hajnoczi if (drv && drv->bdrv_io_unplug) { 273661007b31SStefan Hajnoczi drv->bdrv_io_unplug(bs); 273761007b31SStefan Hajnoczi } else if (bs->file) { 27389a4f4c31SKevin Wolf bdrv_io_unplug(bs->file->bs); 273961007b31SStefan Hajnoczi } 274061007b31SStefan Hajnoczi } 274161007b31SStefan Hajnoczi 274261007b31SStefan Hajnoczi void bdrv_flush_io_queue(BlockDriverState *bs) 274361007b31SStefan Hajnoczi { 274461007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 274561007b31SStefan Hajnoczi if (drv && drv->bdrv_flush_io_queue) { 274661007b31SStefan Hajnoczi drv->bdrv_flush_io_queue(bs); 274761007b31SStefan Hajnoczi } else if (bs->file) { 27489a4f4c31SKevin Wolf bdrv_flush_io_queue(bs->file->bs); 274961007b31SStefan Hajnoczi } 2750f406c03cSAlexander Yarygin bdrv_start_throttled_reqs(bs); 275161007b31SStefan Hajnoczi } 275251288d79SFam Zheng 275351288d79SFam Zheng void bdrv_drained_begin(BlockDriverState *bs) 275451288d79SFam Zheng { 275551288d79SFam Zheng if (!bs->quiesce_counter++) { 275651288d79SFam Zheng aio_disable_external(bdrv_get_aio_context(bs)); 275751288d79SFam Zheng } 275851288d79SFam Zheng bdrv_drain(bs); 275951288d79SFam Zheng } 276051288d79SFam Zheng 276151288d79SFam Zheng void bdrv_drained_end(BlockDriverState *bs) 276251288d79SFam Zheng { 276351288d79SFam Zheng assert(bs->quiesce_counter > 0); 276451288d79SFam Zheng if (--bs->quiesce_counter > 0) { 276551288d79SFam Zheng return; 276651288d79SFam Zheng } 276751288d79SFam Zheng aio_enable_external(bdrv_get_aio_context(bs)); 276851288d79SFam Zheng } 2769