xref: /qemu/block/io.c (revision 166fe9605128e9dfa2b20dd7cc97c64cc98394e4)
161007b31SStefan Hajnoczi /*
261007b31SStefan Hajnoczi  * Block layer I/O functions
361007b31SStefan Hajnoczi  *
461007b31SStefan Hajnoczi  * Copyright (c) 2003 Fabrice Bellard
561007b31SStefan Hajnoczi  *
661007b31SStefan Hajnoczi  * Permission is hereby granted, free of charge, to any person obtaining a copy
761007b31SStefan Hajnoczi  * of this software and associated documentation files (the "Software"), to deal
861007b31SStefan Hajnoczi  * in the Software without restriction, including without limitation the rights
961007b31SStefan Hajnoczi  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
1061007b31SStefan Hajnoczi  * copies of the Software, and to permit persons to whom the Software is
1161007b31SStefan Hajnoczi  * furnished to do so, subject to the following conditions:
1261007b31SStefan Hajnoczi  *
1361007b31SStefan Hajnoczi  * The above copyright notice and this permission notice shall be included in
1461007b31SStefan Hajnoczi  * all copies or substantial portions of the Software.
1561007b31SStefan Hajnoczi  *
1661007b31SStefan Hajnoczi  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
1761007b31SStefan Hajnoczi  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
1861007b31SStefan Hajnoczi  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
1961007b31SStefan Hajnoczi  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
2061007b31SStefan Hajnoczi  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
2161007b31SStefan Hajnoczi  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
2261007b31SStefan Hajnoczi  * THE SOFTWARE.
2361007b31SStefan Hajnoczi  */
2461007b31SStefan Hajnoczi 
2580c71a24SPeter Maydell #include "qemu/osdep.h"
2661007b31SStefan Hajnoczi #include "trace.h"
277f0e9da6SMax Reitz #include "sysemu/block-backend.h"
2861007b31SStefan Hajnoczi #include "block/blockjob.h"
2961007b31SStefan Hajnoczi #include "block/block_int.h"
3076f4afb4SAlberto Garcia #include "block/throttle-groups.h"
31f348b6d1SVeronia Bahaa #include "qemu/cutils.h"
32da34e65cSMarkus Armbruster #include "qapi/error.h"
33d49b6836SMarkus Armbruster #include "qemu/error-report.h"
3461007b31SStefan Hajnoczi 
3561007b31SStefan Hajnoczi #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
3661007b31SStefan Hajnoczi 
3761007b31SStefan Hajnoczi static BlockAIOCB *bdrv_aio_readv_em(BlockDriverState *bs,
3861007b31SStefan Hajnoczi         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
3961007b31SStefan Hajnoczi         BlockCompletionFunc *cb, void *opaque);
4061007b31SStefan Hajnoczi static BlockAIOCB *bdrv_aio_writev_em(BlockDriverState *bs,
4161007b31SStefan Hajnoczi         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
4261007b31SStefan Hajnoczi         BlockCompletionFunc *cb, void *opaque);
4361007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_readv_em(BlockDriverState *bs,
4461007b31SStefan Hajnoczi                                          int64_t sector_num, int nb_sectors,
4561007b31SStefan Hajnoczi                                          QEMUIOVector *iov);
4661007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_writev_em(BlockDriverState *bs,
4761007b31SStefan Hajnoczi                                          int64_t sector_num, int nb_sectors,
4861007b31SStefan Hajnoczi                                          QEMUIOVector *iov);
4961007b31SStefan Hajnoczi static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
5061007b31SStefan Hajnoczi                                          int64_t sector_num,
5161007b31SStefan Hajnoczi                                          QEMUIOVector *qiov,
5261007b31SStefan Hajnoczi                                          int nb_sectors,
5361007b31SStefan Hajnoczi                                          BdrvRequestFlags flags,
5461007b31SStefan Hajnoczi                                          BlockCompletionFunc *cb,
5561007b31SStefan Hajnoczi                                          void *opaque,
5661007b31SStefan Hajnoczi                                          bool is_write);
5761007b31SStefan Hajnoczi static void coroutine_fn bdrv_co_do_rw(void *opaque);
5861007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
5961007b31SStefan Hajnoczi     int64_t sector_num, int nb_sectors, BdrvRequestFlags flags);
6061007b31SStefan Hajnoczi 
6161007b31SStefan Hajnoczi /* throttling disk I/O limits */
6261007b31SStefan Hajnoczi void bdrv_set_io_limits(BlockDriverState *bs,
6361007b31SStefan Hajnoczi                         ThrottleConfig *cfg)
6461007b31SStefan Hajnoczi {
6576f4afb4SAlberto Garcia     throttle_group_config(bs, cfg);
6661007b31SStefan Hajnoczi }
6761007b31SStefan Hajnoczi 
68ce0f1412SPaolo Bonzini void bdrv_no_throttling_begin(BlockDriverState *bs)
6961007b31SStefan Hajnoczi {
70ce0f1412SPaolo Bonzini     if (bs->io_limits_disabled++ == 0) {
71a72f6414SPaolo Bonzini         throttle_group_restart_bs(bs);
72ce0f1412SPaolo Bonzini     }
73ce0f1412SPaolo Bonzini }
74ce0f1412SPaolo Bonzini 
75ce0f1412SPaolo Bonzini void bdrv_no_throttling_end(BlockDriverState *bs)
76ce0f1412SPaolo Bonzini {
77ce0f1412SPaolo Bonzini     assert(bs->io_limits_disabled);
78ce0f1412SPaolo Bonzini     --bs->io_limits_disabled;
7961007b31SStefan Hajnoczi }
8061007b31SStefan Hajnoczi 
8161007b31SStefan Hajnoczi void bdrv_io_limits_disable(BlockDriverState *bs)
8261007b31SStefan Hajnoczi {
83ce0f1412SPaolo Bonzini     assert(bs->throttle_state);
84ce0f1412SPaolo Bonzini     bdrv_no_throttling_begin(bs);
8576f4afb4SAlberto Garcia     throttle_group_unregister_bs(bs);
86ce0f1412SPaolo Bonzini     bdrv_no_throttling_end(bs);
8761007b31SStefan Hajnoczi }
8861007b31SStefan Hajnoczi 
8961007b31SStefan Hajnoczi /* should be called before bdrv_set_io_limits if a limit is set */
9076f4afb4SAlberto Garcia void bdrv_io_limits_enable(BlockDriverState *bs, const char *group)
9161007b31SStefan Hajnoczi {
92ce0f1412SPaolo Bonzini     assert(!bs->throttle_state);
9376f4afb4SAlberto Garcia     throttle_group_register_bs(bs, group);
9461007b31SStefan Hajnoczi }
9561007b31SStefan Hajnoczi 
9676f4afb4SAlberto Garcia void bdrv_io_limits_update_group(BlockDriverState *bs, const char *group)
9761007b31SStefan Hajnoczi {
9876f4afb4SAlberto Garcia     /* this bs is not part of any group */
9976f4afb4SAlberto Garcia     if (!bs->throttle_state) {
10061007b31SStefan Hajnoczi         return;
10161007b31SStefan Hajnoczi     }
10261007b31SStefan Hajnoczi 
10376f4afb4SAlberto Garcia     /* this bs is a part of the same group than the one we want */
10476f4afb4SAlberto Garcia     if (!g_strcmp0(throttle_group_get_name(bs), group)) {
10576f4afb4SAlberto Garcia         return;
10676f4afb4SAlberto Garcia     }
10776f4afb4SAlberto Garcia 
10876f4afb4SAlberto Garcia     /* need to change the group this bs belong to */
10976f4afb4SAlberto Garcia     bdrv_io_limits_disable(bs);
11076f4afb4SAlberto Garcia     bdrv_io_limits_enable(bs, group);
11161007b31SStefan Hajnoczi }
11261007b31SStefan Hajnoczi 
11361007b31SStefan Hajnoczi void bdrv_setup_io_funcs(BlockDriver *bdrv)
11461007b31SStefan Hajnoczi {
11561007b31SStefan Hajnoczi     /* Block drivers without coroutine functions need emulation */
11661007b31SStefan Hajnoczi     if (!bdrv->bdrv_co_readv) {
11761007b31SStefan Hajnoczi         bdrv->bdrv_co_readv = bdrv_co_readv_em;
11861007b31SStefan Hajnoczi         bdrv->bdrv_co_writev = bdrv_co_writev_em;
11961007b31SStefan Hajnoczi 
12061007b31SStefan Hajnoczi         /* bdrv_co_readv_em()/brdv_co_writev_em() work in terms of aio, so if
12161007b31SStefan Hajnoczi          * the block driver lacks aio we need to emulate that too.
12261007b31SStefan Hajnoczi          */
12361007b31SStefan Hajnoczi         if (!bdrv->bdrv_aio_readv) {
12461007b31SStefan Hajnoczi             /* add AIO emulation layer */
12561007b31SStefan Hajnoczi             bdrv->bdrv_aio_readv = bdrv_aio_readv_em;
12661007b31SStefan Hajnoczi             bdrv->bdrv_aio_writev = bdrv_aio_writev_em;
12761007b31SStefan Hajnoczi         }
12861007b31SStefan Hajnoczi     }
12961007b31SStefan Hajnoczi }
13061007b31SStefan Hajnoczi 
13161007b31SStefan Hajnoczi void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
13261007b31SStefan Hajnoczi {
13361007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
13461007b31SStefan Hajnoczi     Error *local_err = NULL;
13561007b31SStefan Hajnoczi 
13661007b31SStefan Hajnoczi     memset(&bs->bl, 0, sizeof(bs->bl));
13761007b31SStefan Hajnoczi 
13861007b31SStefan Hajnoczi     if (!drv) {
13961007b31SStefan Hajnoczi         return;
14061007b31SStefan Hajnoczi     }
14161007b31SStefan Hajnoczi 
14261007b31SStefan Hajnoczi     /* Take some limits from the children as a default */
14361007b31SStefan Hajnoczi     if (bs->file) {
1449a4f4c31SKevin Wolf         bdrv_refresh_limits(bs->file->bs, &local_err);
14561007b31SStefan Hajnoczi         if (local_err) {
14661007b31SStefan Hajnoczi             error_propagate(errp, local_err);
14761007b31SStefan Hajnoczi             return;
14861007b31SStefan Hajnoczi         }
1499a4f4c31SKevin Wolf         bs->bl.opt_transfer_length = bs->file->bs->bl.opt_transfer_length;
1509a4f4c31SKevin Wolf         bs->bl.max_transfer_length = bs->file->bs->bl.max_transfer_length;
1519a4f4c31SKevin Wolf         bs->bl.min_mem_alignment = bs->file->bs->bl.min_mem_alignment;
1529a4f4c31SKevin Wolf         bs->bl.opt_mem_alignment = bs->file->bs->bl.opt_mem_alignment;
153bd44feb7SStefan Hajnoczi         bs->bl.max_iov = bs->file->bs->bl.max_iov;
15461007b31SStefan Hajnoczi     } else {
1554196d2f0SDenis V. Lunev         bs->bl.min_mem_alignment = 512;
156459b4e66SDenis V. Lunev         bs->bl.opt_mem_alignment = getpagesize();
157bd44feb7SStefan Hajnoczi 
158bd44feb7SStefan Hajnoczi         /* Safe default since most protocols use readv()/writev()/etc */
159bd44feb7SStefan Hajnoczi         bs->bl.max_iov = IOV_MAX;
16061007b31SStefan Hajnoczi     }
16161007b31SStefan Hajnoczi 
162760e0063SKevin Wolf     if (bs->backing) {
163760e0063SKevin Wolf         bdrv_refresh_limits(bs->backing->bs, &local_err);
16461007b31SStefan Hajnoczi         if (local_err) {
16561007b31SStefan Hajnoczi             error_propagate(errp, local_err);
16661007b31SStefan Hajnoczi             return;
16761007b31SStefan Hajnoczi         }
16861007b31SStefan Hajnoczi         bs->bl.opt_transfer_length =
16961007b31SStefan Hajnoczi             MAX(bs->bl.opt_transfer_length,
170760e0063SKevin Wolf                 bs->backing->bs->bl.opt_transfer_length);
17161007b31SStefan Hajnoczi         bs->bl.max_transfer_length =
17261007b31SStefan Hajnoczi             MIN_NON_ZERO(bs->bl.max_transfer_length,
173760e0063SKevin Wolf                          bs->backing->bs->bl.max_transfer_length);
17461007b31SStefan Hajnoczi         bs->bl.opt_mem_alignment =
17561007b31SStefan Hajnoczi             MAX(bs->bl.opt_mem_alignment,
176760e0063SKevin Wolf                 bs->backing->bs->bl.opt_mem_alignment);
1774196d2f0SDenis V. Lunev         bs->bl.min_mem_alignment =
1784196d2f0SDenis V. Lunev             MAX(bs->bl.min_mem_alignment,
179760e0063SKevin Wolf                 bs->backing->bs->bl.min_mem_alignment);
180bd44feb7SStefan Hajnoczi         bs->bl.max_iov =
181bd44feb7SStefan Hajnoczi             MIN(bs->bl.max_iov,
182bd44feb7SStefan Hajnoczi                 bs->backing->bs->bl.max_iov);
18361007b31SStefan Hajnoczi     }
18461007b31SStefan Hajnoczi 
18561007b31SStefan Hajnoczi     /* Then let the driver override it */
18661007b31SStefan Hajnoczi     if (drv->bdrv_refresh_limits) {
18761007b31SStefan Hajnoczi         drv->bdrv_refresh_limits(bs, errp);
18861007b31SStefan Hajnoczi     }
18961007b31SStefan Hajnoczi }
19061007b31SStefan Hajnoczi 
19161007b31SStefan Hajnoczi /**
19261007b31SStefan Hajnoczi  * The copy-on-read flag is actually a reference count so multiple users may
19361007b31SStefan Hajnoczi  * use the feature without worrying about clobbering its previous state.
19461007b31SStefan Hajnoczi  * Copy-on-read stays enabled until all users have called to disable it.
19561007b31SStefan Hajnoczi  */
19661007b31SStefan Hajnoczi void bdrv_enable_copy_on_read(BlockDriverState *bs)
19761007b31SStefan Hajnoczi {
19861007b31SStefan Hajnoczi     bs->copy_on_read++;
19961007b31SStefan Hajnoczi }
20061007b31SStefan Hajnoczi 
20161007b31SStefan Hajnoczi void bdrv_disable_copy_on_read(BlockDriverState *bs)
20261007b31SStefan Hajnoczi {
20361007b31SStefan Hajnoczi     assert(bs->copy_on_read > 0);
20461007b31SStefan Hajnoczi     bs->copy_on_read--;
20561007b31SStefan Hajnoczi }
20661007b31SStefan Hajnoczi 
20761007b31SStefan Hajnoczi /* Check if any requests are in-flight (including throttled requests) */
208439db28cSKevin Wolf bool bdrv_requests_pending(BlockDriverState *bs)
20961007b31SStefan Hajnoczi {
21037a639a7SKevin Wolf     BdrvChild *child;
21137a639a7SKevin Wolf 
21261007b31SStefan Hajnoczi     if (!QLIST_EMPTY(&bs->tracked_requests)) {
21361007b31SStefan Hajnoczi         return true;
21461007b31SStefan Hajnoczi     }
21561007b31SStefan Hajnoczi     if (!qemu_co_queue_empty(&bs->throttled_reqs[0])) {
21661007b31SStefan Hajnoczi         return true;
21761007b31SStefan Hajnoczi     }
21861007b31SStefan Hajnoczi     if (!qemu_co_queue_empty(&bs->throttled_reqs[1])) {
21961007b31SStefan Hajnoczi         return true;
22061007b31SStefan Hajnoczi     }
22137a639a7SKevin Wolf 
22237a639a7SKevin Wolf     QLIST_FOREACH(child, &bs->children, next) {
22337a639a7SKevin Wolf         if (bdrv_requests_pending(child->bs)) {
22461007b31SStefan Hajnoczi             return true;
22561007b31SStefan Hajnoczi         }
22661007b31SStefan Hajnoczi     }
22737a639a7SKevin Wolf 
22861007b31SStefan Hajnoczi     return false;
22961007b31SStefan Hajnoczi }
23061007b31SStefan Hajnoczi 
23167da1dc5SFam Zheng static void bdrv_drain_recurse(BlockDriverState *bs)
23267da1dc5SFam Zheng {
23367da1dc5SFam Zheng     BdrvChild *child;
23467da1dc5SFam Zheng 
23567da1dc5SFam Zheng     if (bs->drv && bs->drv->bdrv_drain) {
23667da1dc5SFam Zheng         bs->drv->bdrv_drain(bs);
23767da1dc5SFam Zheng     }
23867da1dc5SFam Zheng     QLIST_FOREACH(child, &bs->children, next) {
23967da1dc5SFam Zheng         bdrv_drain_recurse(child->bs);
24067da1dc5SFam Zheng     }
24167da1dc5SFam Zheng }
24267da1dc5SFam Zheng 
243a77fd4bbSFam Zheng typedef struct {
244a77fd4bbSFam Zheng     Coroutine *co;
245a77fd4bbSFam Zheng     BlockDriverState *bs;
246a77fd4bbSFam Zheng     QEMUBH *bh;
247a77fd4bbSFam Zheng     bool done;
248a77fd4bbSFam Zheng } BdrvCoDrainData;
249a77fd4bbSFam Zheng 
250b6e84c97SPaolo Bonzini static void bdrv_drain_poll(BlockDriverState *bs)
251b6e84c97SPaolo Bonzini {
252b6e84c97SPaolo Bonzini     bool busy = true;
253b6e84c97SPaolo Bonzini 
254b6e84c97SPaolo Bonzini     while (busy) {
255b6e84c97SPaolo Bonzini         /* Keep iterating */
256b6e84c97SPaolo Bonzini         busy = bdrv_requests_pending(bs);
257b6e84c97SPaolo Bonzini         busy |= aio_poll(bdrv_get_aio_context(bs), busy);
258b6e84c97SPaolo Bonzini     }
259b6e84c97SPaolo Bonzini }
260b6e84c97SPaolo Bonzini 
261a77fd4bbSFam Zheng static void bdrv_co_drain_bh_cb(void *opaque)
262a77fd4bbSFam Zheng {
263a77fd4bbSFam Zheng     BdrvCoDrainData *data = opaque;
264a77fd4bbSFam Zheng     Coroutine *co = data->co;
265a77fd4bbSFam Zheng 
266a77fd4bbSFam Zheng     qemu_bh_delete(data->bh);
267b6e84c97SPaolo Bonzini     bdrv_drain_poll(data->bs);
268a77fd4bbSFam Zheng     data->done = true;
269a77fd4bbSFam Zheng     qemu_coroutine_enter(co, NULL);
270a77fd4bbSFam Zheng }
271a77fd4bbSFam Zheng 
272b6e84c97SPaolo Bonzini static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs)
273a77fd4bbSFam Zheng {
274a77fd4bbSFam Zheng     BdrvCoDrainData data;
275a77fd4bbSFam Zheng 
276a77fd4bbSFam Zheng     /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
277a77fd4bbSFam Zheng      * other coroutines run if they were queued from
278a77fd4bbSFam Zheng      * qemu_co_queue_run_restart(). */
279a77fd4bbSFam Zheng 
280a77fd4bbSFam Zheng     assert(qemu_in_coroutine());
281a77fd4bbSFam Zheng     data = (BdrvCoDrainData) {
282a77fd4bbSFam Zheng         .co = qemu_coroutine_self(),
283a77fd4bbSFam Zheng         .bs = bs,
284a77fd4bbSFam Zheng         .done = false,
285a77fd4bbSFam Zheng         .bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_drain_bh_cb, &data),
286a77fd4bbSFam Zheng     };
287a77fd4bbSFam Zheng     qemu_bh_schedule(data.bh);
288a77fd4bbSFam Zheng 
289a77fd4bbSFam Zheng     qemu_coroutine_yield();
290a77fd4bbSFam Zheng     /* If we are resumed from some other event (such as an aio completion or a
291a77fd4bbSFam Zheng      * timer callback), it is a bug in the caller that should be fixed. */
292a77fd4bbSFam Zheng     assert(data.done);
293a77fd4bbSFam Zheng }
294a77fd4bbSFam Zheng 
29561007b31SStefan Hajnoczi /*
29667da1dc5SFam Zheng  * Wait for pending requests to complete on a single BlockDriverState subtree,
29767da1dc5SFam Zheng  * and suspend block driver's internal I/O until next request arrives.
29861007b31SStefan Hajnoczi  *
29961007b31SStefan Hajnoczi  * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState
30061007b31SStefan Hajnoczi  * AioContext.
3017a63f3cdSStefan Hajnoczi  *
3027a63f3cdSStefan Hajnoczi  * Only this BlockDriverState's AioContext is run, so in-flight requests must
3037a63f3cdSStefan Hajnoczi  * not depend on events in other AioContexts.  In that case, use
3047a63f3cdSStefan Hajnoczi  * bdrv_drain_all() instead.
30561007b31SStefan Hajnoczi  */
306b6e84c97SPaolo Bonzini void coroutine_fn bdrv_co_drain(BlockDriverState *bs)
307b6e84c97SPaolo Bonzini {
308ce0f1412SPaolo Bonzini     bdrv_no_throttling_begin(bs);
3096b98bd64SPaolo Bonzini     bdrv_io_unplugged_begin(bs);
310b6e84c97SPaolo Bonzini     bdrv_drain_recurse(bs);
311b6e84c97SPaolo Bonzini     bdrv_co_yield_to_drain(bs);
3126b98bd64SPaolo Bonzini     bdrv_io_unplugged_end(bs);
313ce0f1412SPaolo Bonzini     bdrv_no_throttling_end(bs);
314b6e84c97SPaolo Bonzini }
315b6e84c97SPaolo Bonzini 
31661007b31SStefan Hajnoczi void bdrv_drain(BlockDriverState *bs)
31761007b31SStefan Hajnoczi {
318ce0f1412SPaolo Bonzini     bdrv_no_throttling_begin(bs);
3196b98bd64SPaolo Bonzini     bdrv_io_unplugged_begin(bs);
32067da1dc5SFam Zheng     bdrv_drain_recurse(bs);
321a77fd4bbSFam Zheng     if (qemu_in_coroutine()) {
322b6e84c97SPaolo Bonzini         bdrv_co_yield_to_drain(bs);
323b6e84c97SPaolo Bonzini     } else {
324b6e84c97SPaolo Bonzini         bdrv_drain_poll(bs);
32561007b31SStefan Hajnoczi     }
3266b98bd64SPaolo Bonzini     bdrv_io_unplugged_end(bs);
327ce0f1412SPaolo Bonzini     bdrv_no_throttling_end(bs);
32861007b31SStefan Hajnoczi }
32961007b31SStefan Hajnoczi 
33061007b31SStefan Hajnoczi /*
33161007b31SStefan Hajnoczi  * Wait for pending requests to complete across all BlockDriverStates
33261007b31SStefan Hajnoczi  *
33361007b31SStefan Hajnoczi  * This function does not flush data to disk, use bdrv_flush_all() for that
33461007b31SStefan Hajnoczi  * after calling this function.
33561007b31SStefan Hajnoczi  */
33661007b31SStefan Hajnoczi void bdrv_drain_all(void)
33761007b31SStefan Hajnoczi {
33861007b31SStefan Hajnoczi     /* Always run first iteration so any pending completion BHs run */
33961007b31SStefan Hajnoczi     bool busy = true;
34061007b31SStefan Hajnoczi     BlockDriverState *bs = NULL;
341f406c03cSAlexander Yarygin     GSList *aio_ctxs = NULL, *ctx;
34261007b31SStefan Hajnoczi 
34361007b31SStefan Hajnoczi     while ((bs = bdrv_next(bs))) {
34461007b31SStefan Hajnoczi         AioContext *aio_context = bdrv_get_aio_context(bs);
34561007b31SStefan Hajnoczi 
34661007b31SStefan Hajnoczi         aio_context_acquire(aio_context);
34761007b31SStefan Hajnoczi         if (bs->job) {
34861007b31SStefan Hajnoczi             block_job_pause(bs->job);
34961007b31SStefan Hajnoczi         }
350ce0f1412SPaolo Bonzini         bdrv_no_throttling_begin(bs);
3516b98bd64SPaolo Bonzini         bdrv_io_unplugged_begin(bs);
3529dcf8ecdSPaolo Bonzini         bdrv_drain_recurse(bs);
35361007b31SStefan Hajnoczi         aio_context_release(aio_context);
354f406c03cSAlexander Yarygin 
355764ba3aeSAlberto Garcia         if (!g_slist_find(aio_ctxs, aio_context)) {
356f406c03cSAlexander Yarygin             aio_ctxs = g_slist_prepend(aio_ctxs, aio_context);
357f406c03cSAlexander Yarygin         }
35861007b31SStefan Hajnoczi     }
35961007b31SStefan Hajnoczi 
3607a63f3cdSStefan Hajnoczi     /* Note that completion of an asynchronous I/O operation can trigger any
3617a63f3cdSStefan Hajnoczi      * number of other I/O operations on other devices---for example a
3627a63f3cdSStefan Hajnoczi      * coroutine can submit an I/O request to another device in response to
3637a63f3cdSStefan Hajnoczi      * request completion.  Therefore we must keep looping until there was no
3647a63f3cdSStefan Hajnoczi      * more activity rather than simply draining each device independently.
3657a63f3cdSStefan Hajnoczi      */
36661007b31SStefan Hajnoczi     while (busy) {
36761007b31SStefan Hajnoczi         busy = false;
368f406c03cSAlexander Yarygin 
369f406c03cSAlexander Yarygin         for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) {
370f406c03cSAlexander Yarygin             AioContext *aio_context = ctx->data;
37161007b31SStefan Hajnoczi             bs = NULL;
37261007b31SStefan Hajnoczi 
37361007b31SStefan Hajnoczi             aio_context_acquire(aio_context);
374f406c03cSAlexander Yarygin             while ((bs = bdrv_next(bs))) {
375f406c03cSAlexander Yarygin                 if (aio_context == bdrv_get_aio_context(bs)) {
376f406c03cSAlexander Yarygin                     if (bdrv_requests_pending(bs)) {
377f406c03cSAlexander Yarygin                         busy = true;
378f406c03cSAlexander Yarygin                         aio_poll(aio_context, busy);
379f406c03cSAlexander Yarygin                     }
380f406c03cSAlexander Yarygin                 }
381f406c03cSAlexander Yarygin             }
382f406c03cSAlexander Yarygin             busy |= aio_poll(aio_context, false);
38361007b31SStefan Hajnoczi             aio_context_release(aio_context);
38461007b31SStefan Hajnoczi         }
38561007b31SStefan Hajnoczi     }
38661007b31SStefan Hajnoczi 
38761007b31SStefan Hajnoczi     bs = NULL;
38861007b31SStefan Hajnoczi     while ((bs = bdrv_next(bs))) {
38961007b31SStefan Hajnoczi         AioContext *aio_context = bdrv_get_aio_context(bs);
39061007b31SStefan Hajnoczi 
39161007b31SStefan Hajnoczi         aio_context_acquire(aio_context);
3926b98bd64SPaolo Bonzini         bdrv_io_unplugged_end(bs);
393ce0f1412SPaolo Bonzini         bdrv_no_throttling_end(bs);
39461007b31SStefan Hajnoczi         if (bs->job) {
39561007b31SStefan Hajnoczi             block_job_resume(bs->job);
39661007b31SStefan Hajnoczi         }
39761007b31SStefan Hajnoczi         aio_context_release(aio_context);
39861007b31SStefan Hajnoczi     }
399f406c03cSAlexander Yarygin     g_slist_free(aio_ctxs);
40061007b31SStefan Hajnoczi }
40161007b31SStefan Hajnoczi 
40261007b31SStefan Hajnoczi /**
40361007b31SStefan Hajnoczi  * Remove an active request from the tracked requests list
40461007b31SStefan Hajnoczi  *
40561007b31SStefan Hajnoczi  * This function should be called when a tracked request is completing.
40661007b31SStefan Hajnoczi  */
40761007b31SStefan Hajnoczi static void tracked_request_end(BdrvTrackedRequest *req)
40861007b31SStefan Hajnoczi {
40961007b31SStefan Hajnoczi     if (req->serialising) {
41061007b31SStefan Hajnoczi         req->bs->serialising_in_flight--;
41161007b31SStefan Hajnoczi     }
41261007b31SStefan Hajnoczi 
41361007b31SStefan Hajnoczi     QLIST_REMOVE(req, list);
41461007b31SStefan Hajnoczi     qemu_co_queue_restart_all(&req->wait_queue);
41561007b31SStefan Hajnoczi }
41661007b31SStefan Hajnoczi 
41761007b31SStefan Hajnoczi /**
41861007b31SStefan Hajnoczi  * Add an active request to the tracked requests list
41961007b31SStefan Hajnoczi  */
42061007b31SStefan Hajnoczi static void tracked_request_begin(BdrvTrackedRequest *req,
42161007b31SStefan Hajnoczi                                   BlockDriverState *bs,
42261007b31SStefan Hajnoczi                                   int64_t offset,
423ebde595cSFam Zheng                                   unsigned int bytes,
424ebde595cSFam Zheng                                   enum BdrvTrackedRequestType type)
42561007b31SStefan Hajnoczi {
42661007b31SStefan Hajnoczi     *req = (BdrvTrackedRequest){
42761007b31SStefan Hajnoczi         .bs = bs,
42861007b31SStefan Hajnoczi         .offset         = offset,
42961007b31SStefan Hajnoczi         .bytes          = bytes,
430ebde595cSFam Zheng         .type           = type,
43161007b31SStefan Hajnoczi         .co             = qemu_coroutine_self(),
43261007b31SStefan Hajnoczi         .serialising    = false,
43361007b31SStefan Hajnoczi         .overlap_offset = offset,
43461007b31SStefan Hajnoczi         .overlap_bytes  = bytes,
43561007b31SStefan Hajnoczi     };
43661007b31SStefan Hajnoczi 
43761007b31SStefan Hajnoczi     qemu_co_queue_init(&req->wait_queue);
43861007b31SStefan Hajnoczi 
43961007b31SStefan Hajnoczi     QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
44061007b31SStefan Hajnoczi }
44161007b31SStefan Hajnoczi 
44261007b31SStefan Hajnoczi static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align)
44361007b31SStefan Hajnoczi {
44461007b31SStefan Hajnoczi     int64_t overlap_offset = req->offset & ~(align - 1);
44561007b31SStefan Hajnoczi     unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align)
44661007b31SStefan Hajnoczi                                - overlap_offset;
44761007b31SStefan Hajnoczi 
44861007b31SStefan Hajnoczi     if (!req->serialising) {
44961007b31SStefan Hajnoczi         req->bs->serialising_in_flight++;
45061007b31SStefan Hajnoczi         req->serialising = true;
45161007b31SStefan Hajnoczi     }
45261007b31SStefan Hajnoczi 
45361007b31SStefan Hajnoczi     req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
45461007b31SStefan Hajnoczi     req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
45561007b31SStefan Hajnoczi }
45661007b31SStefan Hajnoczi 
45761007b31SStefan Hajnoczi /**
45861007b31SStefan Hajnoczi  * Round a region to cluster boundaries
45961007b31SStefan Hajnoczi  */
46061007b31SStefan Hajnoczi void bdrv_round_to_clusters(BlockDriverState *bs,
46161007b31SStefan Hajnoczi                             int64_t sector_num, int nb_sectors,
46261007b31SStefan Hajnoczi                             int64_t *cluster_sector_num,
46361007b31SStefan Hajnoczi                             int *cluster_nb_sectors)
46461007b31SStefan Hajnoczi {
46561007b31SStefan Hajnoczi     BlockDriverInfo bdi;
46661007b31SStefan Hajnoczi 
46761007b31SStefan Hajnoczi     if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
46861007b31SStefan Hajnoczi         *cluster_sector_num = sector_num;
46961007b31SStefan Hajnoczi         *cluster_nb_sectors = nb_sectors;
47061007b31SStefan Hajnoczi     } else {
47161007b31SStefan Hajnoczi         int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE;
47261007b31SStefan Hajnoczi         *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c);
47361007b31SStefan Hajnoczi         *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num +
47461007b31SStefan Hajnoczi                                             nb_sectors, c);
47561007b31SStefan Hajnoczi     }
47661007b31SStefan Hajnoczi }
47761007b31SStefan Hajnoczi 
47861007b31SStefan Hajnoczi static int bdrv_get_cluster_size(BlockDriverState *bs)
47961007b31SStefan Hajnoczi {
48061007b31SStefan Hajnoczi     BlockDriverInfo bdi;
48161007b31SStefan Hajnoczi     int ret;
48261007b31SStefan Hajnoczi 
48361007b31SStefan Hajnoczi     ret = bdrv_get_info(bs, &bdi);
48461007b31SStefan Hajnoczi     if (ret < 0 || bdi.cluster_size == 0) {
48561007b31SStefan Hajnoczi         return bs->request_alignment;
48661007b31SStefan Hajnoczi     } else {
48761007b31SStefan Hajnoczi         return bdi.cluster_size;
48861007b31SStefan Hajnoczi     }
48961007b31SStefan Hajnoczi }
49061007b31SStefan Hajnoczi 
49161007b31SStefan Hajnoczi static bool tracked_request_overlaps(BdrvTrackedRequest *req,
49261007b31SStefan Hajnoczi                                      int64_t offset, unsigned int bytes)
49361007b31SStefan Hajnoczi {
49461007b31SStefan Hajnoczi     /*        aaaa   bbbb */
49561007b31SStefan Hajnoczi     if (offset >= req->overlap_offset + req->overlap_bytes) {
49661007b31SStefan Hajnoczi         return false;
49761007b31SStefan Hajnoczi     }
49861007b31SStefan Hajnoczi     /* bbbb   aaaa        */
49961007b31SStefan Hajnoczi     if (req->overlap_offset >= offset + bytes) {
50061007b31SStefan Hajnoczi         return false;
50161007b31SStefan Hajnoczi     }
50261007b31SStefan Hajnoczi     return true;
50361007b31SStefan Hajnoczi }
50461007b31SStefan Hajnoczi 
50561007b31SStefan Hajnoczi static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self)
50661007b31SStefan Hajnoczi {
50761007b31SStefan Hajnoczi     BlockDriverState *bs = self->bs;
50861007b31SStefan Hajnoczi     BdrvTrackedRequest *req;
50961007b31SStefan Hajnoczi     bool retry;
51061007b31SStefan Hajnoczi     bool waited = false;
51161007b31SStefan Hajnoczi 
51261007b31SStefan Hajnoczi     if (!bs->serialising_in_flight) {
51361007b31SStefan Hajnoczi         return false;
51461007b31SStefan Hajnoczi     }
51561007b31SStefan Hajnoczi 
51661007b31SStefan Hajnoczi     do {
51761007b31SStefan Hajnoczi         retry = false;
51861007b31SStefan Hajnoczi         QLIST_FOREACH(req, &bs->tracked_requests, list) {
51961007b31SStefan Hajnoczi             if (req == self || (!req->serialising && !self->serialising)) {
52061007b31SStefan Hajnoczi                 continue;
52161007b31SStefan Hajnoczi             }
52261007b31SStefan Hajnoczi             if (tracked_request_overlaps(req, self->overlap_offset,
52361007b31SStefan Hajnoczi                                          self->overlap_bytes))
52461007b31SStefan Hajnoczi             {
52561007b31SStefan Hajnoczi                 /* Hitting this means there was a reentrant request, for
52661007b31SStefan Hajnoczi                  * example, a block driver issuing nested requests.  This must
52761007b31SStefan Hajnoczi                  * never happen since it means deadlock.
52861007b31SStefan Hajnoczi                  */
52961007b31SStefan Hajnoczi                 assert(qemu_coroutine_self() != req->co);
53061007b31SStefan Hajnoczi 
53161007b31SStefan Hajnoczi                 /* If the request is already (indirectly) waiting for us, or
53261007b31SStefan Hajnoczi                  * will wait for us as soon as it wakes up, then just go on
53361007b31SStefan Hajnoczi                  * (instead of producing a deadlock in the former case). */
53461007b31SStefan Hajnoczi                 if (!req->waiting_for) {
53561007b31SStefan Hajnoczi                     self->waiting_for = req;
53661007b31SStefan Hajnoczi                     qemu_co_queue_wait(&req->wait_queue);
53761007b31SStefan Hajnoczi                     self->waiting_for = NULL;
53861007b31SStefan Hajnoczi                     retry = true;
53961007b31SStefan Hajnoczi                     waited = true;
54061007b31SStefan Hajnoczi                     break;
54161007b31SStefan Hajnoczi                 }
54261007b31SStefan Hajnoczi             }
54361007b31SStefan Hajnoczi         }
54461007b31SStefan Hajnoczi     } while (retry);
54561007b31SStefan Hajnoczi 
54661007b31SStefan Hajnoczi     return waited;
54761007b31SStefan Hajnoczi }
54861007b31SStefan Hajnoczi 
54961007b31SStefan Hajnoczi static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset,
55061007b31SStefan Hajnoczi                                    size_t size)
55161007b31SStefan Hajnoczi {
55261007b31SStefan Hajnoczi     if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) {
55361007b31SStefan Hajnoczi         return -EIO;
55461007b31SStefan Hajnoczi     }
55561007b31SStefan Hajnoczi 
55661007b31SStefan Hajnoczi     if (!bdrv_is_inserted(bs)) {
55761007b31SStefan Hajnoczi         return -ENOMEDIUM;
55861007b31SStefan Hajnoczi     }
55961007b31SStefan Hajnoczi 
56061007b31SStefan Hajnoczi     if (offset < 0) {
56161007b31SStefan Hajnoczi         return -EIO;
56261007b31SStefan Hajnoczi     }
56361007b31SStefan Hajnoczi 
56461007b31SStefan Hajnoczi     return 0;
56561007b31SStefan Hajnoczi }
56661007b31SStefan Hajnoczi 
56761007b31SStefan Hajnoczi static int bdrv_check_request(BlockDriverState *bs, int64_t sector_num,
56861007b31SStefan Hajnoczi                               int nb_sectors)
56961007b31SStefan Hajnoczi {
57061007b31SStefan Hajnoczi     if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
57161007b31SStefan Hajnoczi         return -EIO;
57261007b31SStefan Hajnoczi     }
57361007b31SStefan Hajnoczi 
57461007b31SStefan Hajnoczi     return bdrv_check_byte_request(bs, sector_num * BDRV_SECTOR_SIZE,
57561007b31SStefan Hajnoczi                                    nb_sectors * BDRV_SECTOR_SIZE);
57661007b31SStefan Hajnoczi }
57761007b31SStefan Hajnoczi 
57861007b31SStefan Hajnoczi typedef struct RwCo {
57961007b31SStefan Hajnoczi     BlockDriverState *bs;
58061007b31SStefan Hajnoczi     int64_t offset;
58161007b31SStefan Hajnoczi     QEMUIOVector *qiov;
58261007b31SStefan Hajnoczi     bool is_write;
58361007b31SStefan Hajnoczi     int ret;
58461007b31SStefan Hajnoczi     BdrvRequestFlags flags;
58561007b31SStefan Hajnoczi } RwCo;
58661007b31SStefan Hajnoczi 
58761007b31SStefan Hajnoczi static void coroutine_fn bdrv_rw_co_entry(void *opaque)
58861007b31SStefan Hajnoczi {
58961007b31SStefan Hajnoczi     RwCo *rwco = opaque;
59061007b31SStefan Hajnoczi 
59161007b31SStefan Hajnoczi     if (!rwco->is_write) {
59261007b31SStefan Hajnoczi         rwco->ret = bdrv_co_do_preadv(rwco->bs, rwco->offset,
59361007b31SStefan Hajnoczi                                       rwco->qiov->size, rwco->qiov,
59461007b31SStefan Hajnoczi                                       rwco->flags);
59561007b31SStefan Hajnoczi     } else {
59661007b31SStefan Hajnoczi         rwco->ret = bdrv_co_do_pwritev(rwco->bs, rwco->offset,
59761007b31SStefan Hajnoczi                                        rwco->qiov->size, rwco->qiov,
59861007b31SStefan Hajnoczi                                        rwco->flags);
59961007b31SStefan Hajnoczi     }
60061007b31SStefan Hajnoczi }
60161007b31SStefan Hajnoczi 
60261007b31SStefan Hajnoczi /*
60361007b31SStefan Hajnoczi  * Process a vectored synchronous request using coroutines
60461007b31SStefan Hajnoczi  */
60561007b31SStefan Hajnoczi static int bdrv_prwv_co(BlockDriverState *bs, int64_t offset,
60661007b31SStefan Hajnoczi                         QEMUIOVector *qiov, bool is_write,
60761007b31SStefan Hajnoczi                         BdrvRequestFlags flags)
60861007b31SStefan Hajnoczi {
60961007b31SStefan Hajnoczi     Coroutine *co;
61061007b31SStefan Hajnoczi     RwCo rwco = {
61161007b31SStefan Hajnoczi         .bs = bs,
61261007b31SStefan Hajnoczi         .offset = offset,
61361007b31SStefan Hajnoczi         .qiov = qiov,
61461007b31SStefan Hajnoczi         .is_write = is_write,
61561007b31SStefan Hajnoczi         .ret = NOT_DONE,
61661007b31SStefan Hajnoczi         .flags = flags,
61761007b31SStefan Hajnoczi     };
61861007b31SStefan Hajnoczi 
61961007b31SStefan Hajnoczi     if (qemu_in_coroutine()) {
62061007b31SStefan Hajnoczi         /* Fast-path if already in coroutine context */
62161007b31SStefan Hajnoczi         bdrv_rw_co_entry(&rwco);
62261007b31SStefan Hajnoczi     } else {
62361007b31SStefan Hajnoczi         AioContext *aio_context = bdrv_get_aio_context(bs);
62461007b31SStefan Hajnoczi 
62561007b31SStefan Hajnoczi         co = qemu_coroutine_create(bdrv_rw_co_entry);
62661007b31SStefan Hajnoczi         qemu_coroutine_enter(co, &rwco);
62761007b31SStefan Hajnoczi         while (rwco.ret == NOT_DONE) {
62861007b31SStefan Hajnoczi             aio_poll(aio_context, true);
62961007b31SStefan Hajnoczi         }
63061007b31SStefan Hajnoczi     }
63161007b31SStefan Hajnoczi     return rwco.ret;
63261007b31SStefan Hajnoczi }
63361007b31SStefan Hajnoczi 
63461007b31SStefan Hajnoczi /*
63561007b31SStefan Hajnoczi  * Process a synchronous request using coroutines
63661007b31SStefan Hajnoczi  */
63761007b31SStefan Hajnoczi static int bdrv_rw_co(BlockDriverState *bs, int64_t sector_num, uint8_t *buf,
63861007b31SStefan Hajnoczi                       int nb_sectors, bool is_write, BdrvRequestFlags flags)
63961007b31SStefan Hajnoczi {
64061007b31SStefan Hajnoczi     QEMUIOVector qiov;
64161007b31SStefan Hajnoczi     struct iovec iov = {
64261007b31SStefan Hajnoczi         .iov_base = (void *)buf,
64361007b31SStefan Hajnoczi         .iov_len = nb_sectors * BDRV_SECTOR_SIZE,
64461007b31SStefan Hajnoczi     };
64561007b31SStefan Hajnoczi 
64661007b31SStefan Hajnoczi     if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
64761007b31SStefan Hajnoczi         return -EINVAL;
64861007b31SStefan Hajnoczi     }
64961007b31SStefan Hajnoczi 
65061007b31SStefan Hajnoczi     qemu_iovec_init_external(&qiov, &iov, 1);
65161007b31SStefan Hajnoczi     return bdrv_prwv_co(bs, sector_num << BDRV_SECTOR_BITS,
65261007b31SStefan Hajnoczi                         &qiov, is_write, flags);
65361007b31SStefan Hajnoczi }
65461007b31SStefan Hajnoczi 
65561007b31SStefan Hajnoczi /* return < 0 if error. See bdrv_write() for the return codes */
65661007b31SStefan Hajnoczi int bdrv_read(BlockDriverState *bs, int64_t sector_num,
65761007b31SStefan Hajnoczi               uint8_t *buf, int nb_sectors)
65861007b31SStefan Hajnoczi {
65961007b31SStefan Hajnoczi     return bdrv_rw_co(bs, sector_num, buf, nb_sectors, false, 0);
66061007b31SStefan Hajnoczi }
66161007b31SStefan Hajnoczi 
66261007b31SStefan Hajnoczi /* Return < 0 if error. Important errors are:
66361007b31SStefan Hajnoczi   -EIO         generic I/O error (may happen for all errors)
66461007b31SStefan Hajnoczi   -ENOMEDIUM   No media inserted.
66561007b31SStefan Hajnoczi   -EINVAL      Invalid sector number or nb_sectors
66661007b31SStefan Hajnoczi   -EACCES      Trying to write a read-only device
66761007b31SStefan Hajnoczi */
66861007b31SStefan Hajnoczi int bdrv_write(BlockDriverState *bs, int64_t sector_num,
66961007b31SStefan Hajnoczi                const uint8_t *buf, int nb_sectors)
67061007b31SStefan Hajnoczi {
67161007b31SStefan Hajnoczi     return bdrv_rw_co(bs, sector_num, (uint8_t *)buf, nb_sectors, true, 0);
67261007b31SStefan Hajnoczi }
67361007b31SStefan Hajnoczi 
67461007b31SStefan Hajnoczi int bdrv_write_zeroes(BlockDriverState *bs, int64_t sector_num,
67561007b31SStefan Hajnoczi                       int nb_sectors, BdrvRequestFlags flags)
67661007b31SStefan Hajnoczi {
67761007b31SStefan Hajnoczi     return bdrv_rw_co(bs, sector_num, NULL, nb_sectors, true,
67861007b31SStefan Hajnoczi                       BDRV_REQ_ZERO_WRITE | flags);
67961007b31SStefan Hajnoczi }
68061007b31SStefan Hajnoczi 
68161007b31SStefan Hajnoczi /*
68261007b31SStefan Hajnoczi  * Completely zero out a block device with the help of bdrv_write_zeroes.
68361007b31SStefan Hajnoczi  * The operation is sped up by checking the block status and only writing
68461007b31SStefan Hajnoczi  * zeroes to the device if they currently do not return zeroes. Optional
68561007b31SStefan Hajnoczi  * flags are passed through to bdrv_write_zeroes (e.g. BDRV_REQ_MAY_UNMAP).
68661007b31SStefan Hajnoczi  *
68761007b31SStefan Hajnoczi  * Returns < 0 on error, 0 on success. For error codes see bdrv_write().
68861007b31SStefan Hajnoczi  */
68961007b31SStefan Hajnoczi int bdrv_make_zero(BlockDriverState *bs, BdrvRequestFlags flags)
69061007b31SStefan Hajnoczi {
69161007b31SStefan Hajnoczi     int64_t target_sectors, ret, nb_sectors, sector_num = 0;
69267a0fd2aSFam Zheng     BlockDriverState *file;
69361007b31SStefan Hajnoczi     int n;
69461007b31SStefan Hajnoczi 
69561007b31SStefan Hajnoczi     target_sectors = bdrv_nb_sectors(bs);
69661007b31SStefan Hajnoczi     if (target_sectors < 0) {
69761007b31SStefan Hajnoczi         return target_sectors;
69861007b31SStefan Hajnoczi     }
69961007b31SStefan Hajnoczi 
70061007b31SStefan Hajnoczi     for (;;) {
70161007b31SStefan Hajnoczi         nb_sectors = MIN(target_sectors - sector_num, BDRV_REQUEST_MAX_SECTORS);
70261007b31SStefan Hajnoczi         if (nb_sectors <= 0) {
70361007b31SStefan Hajnoczi             return 0;
70461007b31SStefan Hajnoczi         }
70567a0fd2aSFam Zheng         ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n, &file);
70661007b31SStefan Hajnoczi         if (ret < 0) {
70761007b31SStefan Hajnoczi             error_report("error getting block status at sector %" PRId64 ": %s",
70861007b31SStefan Hajnoczi                          sector_num, strerror(-ret));
70961007b31SStefan Hajnoczi             return ret;
71061007b31SStefan Hajnoczi         }
71161007b31SStefan Hajnoczi         if (ret & BDRV_BLOCK_ZERO) {
71261007b31SStefan Hajnoczi             sector_num += n;
71361007b31SStefan Hajnoczi             continue;
71461007b31SStefan Hajnoczi         }
71561007b31SStefan Hajnoczi         ret = bdrv_write_zeroes(bs, sector_num, n, flags);
71661007b31SStefan Hajnoczi         if (ret < 0) {
71761007b31SStefan Hajnoczi             error_report("error writing zeroes at sector %" PRId64 ": %s",
71861007b31SStefan Hajnoczi                          sector_num, strerror(-ret));
71961007b31SStefan Hajnoczi             return ret;
72061007b31SStefan Hajnoczi         }
72161007b31SStefan Hajnoczi         sector_num += n;
72261007b31SStefan Hajnoczi     }
72361007b31SStefan Hajnoczi }
72461007b31SStefan Hajnoczi 
72561007b31SStefan Hajnoczi int bdrv_pread(BlockDriverState *bs, int64_t offset, void *buf, int bytes)
72661007b31SStefan Hajnoczi {
72761007b31SStefan Hajnoczi     QEMUIOVector qiov;
72861007b31SStefan Hajnoczi     struct iovec iov = {
72961007b31SStefan Hajnoczi         .iov_base = (void *)buf,
73061007b31SStefan Hajnoczi         .iov_len = bytes,
73161007b31SStefan Hajnoczi     };
73261007b31SStefan Hajnoczi     int ret;
73361007b31SStefan Hajnoczi 
73461007b31SStefan Hajnoczi     if (bytes < 0) {
73561007b31SStefan Hajnoczi         return -EINVAL;
73661007b31SStefan Hajnoczi     }
73761007b31SStefan Hajnoczi 
73861007b31SStefan Hajnoczi     qemu_iovec_init_external(&qiov, &iov, 1);
73961007b31SStefan Hajnoczi     ret = bdrv_prwv_co(bs, offset, &qiov, false, 0);
74061007b31SStefan Hajnoczi     if (ret < 0) {
74161007b31SStefan Hajnoczi         return ret;
74261007b31SStefan Hajnoczi     }
74361007b31SStefan Hajnoczi 
74461007b31SStefan Hajnoczi     return bytes;
74561007b31SStefan Hajnoczi }
74661007b31SStefan Hajnoczi 
74761007b31SStefan Hajnoczi int bdrv_pwritev(BlockDriverState *bs, int64_t offset, QEMUIOVector *qiov)
74861007b31SStefan Hajnoczi {
74961007b31SStefan Hajnoczi     int ret;
75061007b31SStefan Hajnoczi 
75161007b31SStefan Hajnoczi     ret = bdrv_prwv_co(bs, offset, qiov, true, 0);
75261007b31SStefan Hajnoczi     if (ret < 0) {
75361007b31SStefan Hajnoczi         return ret;
75461007b31SStefan Hajnoczi     }
75561007b31SStefan Hajnoczi 
75661007b31SStefan Hajnoczi     return qiov->size;
75761007b31SStefan Hajnoczi }
75861007b31SStefan Hajnoczi 
75961007b31SStefan Hajnoczi int bdrv_pwrite(BlockDriverState *bs, int64_t offset,
76061007b31SStefan Hajnoczi                 const void *buf, int bytes)
76161007b31SStefan Hajnoczi {
76261007b31SStefan Hajnoczi     QEMUIOVector qiov;
76361007b31SStefan Hajnoczi     struct iovec iov = {
76461007b31SStefan Hajnoczi         .iov_base   = (void *) buf,
76561007b31SStefan Hajnoczi         .iov_len    = bytes,
76661007b31SStefan Hajnoczi     };
76761007b31SStefan Hajnoczi 
76861007b31SStefan Hajnoczi     if (bytes < 0) {
76961007b31SStefan Hajnoczi         return -EINVAL;
77061007b31SStefan Hajnoczi     }
77161007b31SStefan Hajnoczi 
77261007b31SStefan Hajnoczi     qemu_iovec_init_external(&qiov, &iov, 1);
77361007b31SStefan Hajnoczi     return bdrv_pwritev(bs, offset, &qiov);
77461007b31SStefan Hajnoczi }
77561007b31SStefan Hajnoczi 
77661007b31SStefan Hajnoczi /*
77761007b31SStefan Hajnoczi  * Writes to the file and ensures that no writes are reordered across this
77861007b31SStefan Hajnoczi  * request (acts as a barrier)
77961007b31SStefan Hajnoczi  *
78061007b31SStefan Hajnoczi  * Returns 0 on success, -errno in error cases.
78161007b31SStefan Hajnoczi  */
78261007b31SStefan Hajnoczi int bdrv_pwrite_sync(BlockDriverState *bs, int64_t offset,
78361007b31SStefan Hajnoczi     const void *buf, int count)
78461007b31SStefan Hajnoczi {
78561007b31SStefan Hajnoczi     int ret;
78661007b31SStefan Hajnoczi 
78761007b31SStefan Hajnoczi     ret = bdrv_pwrite(bs, offset, buf, count);
78861007b31SStefan Hajnoczi     if (ret < 0) {
78961007b31SStefan Hajnoczi         return ret;
79061007b31SStefan Hajnoczi     }
79161007b31SStefan Hajnoczi 
792855a6a93SKevin Wolf     ret = bdrv_flush(bs);
793855a6a93SKevin Wolf     if (ret < 0) {
794855a6a93SKevin Wolf         return ret;
79561007b31SStefan Hajnoczi     }
79661007b31SStefan Hajnoczi 
79761007b31SStefan Hajnoczi     return 0;
79861007b31SStefan Hajnoczi }
79961007b31SStefan Hajnoczi 
800*166fe960SKevin Wolf static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
801*166fe960SKevin Wolf                                            uint64_t offset, uint64_t bytes,
802*166fe960SKevin Wolf                                            QEMUIOVector *qiov, int flags)
803*166fe960SKevin Wolf {
804*166fe960SKevin Wolf     BlockDriver *drv = bs->drv;
805*166fe960SKevin Wolf     int64_t sector_num = offset >> BDRV_SECTOR_BITS;
806*166fe960SKevin Wolf     unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS;
807*166fe960SKevin Wolf 
808*166fe960SKevin Wolf     assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
809*166fe960SKevin Wolf     assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
810*166fe960SKevin Wolf     assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
811*166fe960SKevin Wolf 
812*166fe960SKevin Wolf     return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
813*166fe960SKevin Wolf }
814*166fe960SKevin Wolf 
81561007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs,
81661007b31SStefan Hajnoczi         int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
81761007b31SStefan Hajnoczi {
81861007b31SStefan Hajnoczi     /* Perform I/O through a temporary buffer so that users who scribble over
81961007b31SStefan Hajnoczi      * their read buffer while the operation is in progress do not end up
82061007b31SStefan Hajnoczi      * modifying the image file.  This is critical for zero-copy guest I/O
82161007b31SStefan Hajnoczi      * where anything might happen inside guest memory.
82261007b31SStefan Hajnoczi      */
82361007b31SStefan Hajnoczi     void *bounce_buffer;
82461007b31SStefan Hajnoczi 
82561007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
82661007b31SStefan Hajnoczi     struct iovec iov;
82761007b31SStefan Hajnoczi     QEMUIOVector bounce_qiov;
82861007b31SStefan Hajnoczi     int64_t cluster_sector_num;
82961007b31SStefan Hajnoczi     int cluster_nb_sectors;
83061007b31SStefan Hajnoczi     size_t skip_bytes;
83161007b31SStefan Hajnoczi     int ret;
83261007b31SStefan Hajnoczi 
83361007b31SStefan Hajnoczi     /* Cover entire cluster so no additional backing file I/O is required when
83461007b31SStefan Hajnoczi      * allocating cluster in the image file.
83561007b31SStefan Hajnoczi      */
83661007b31SStefan Hajnoczi     bdrv_round_to_clusters(bs, sector_num, nb_sectors,
83761007b31SStefan Hajnoczi                            &cluster_sector_num, &cluster_nb_sectors);
83861007b31SStefan Hajnoczi 
83961007b31SStefan Hajnoczi     trace_bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors,
84061007b31SStefan Hajnoczi                                    cluster_sector_num, cluster_nb_sectors);
84161007b31SStefan Hajnoczi 
84261007b31SStefan Hajnoczi     iov.iov_len = cluster_nb_sectors * BDRV_SECTOR_SIZE;
84361007b31SStefan Hajnoczi     iov.iov_base = bounce_buffer = qemu_try_blockalign(bs, iov.iov_len);
84461007b31SStefan Hajnoczi     if (bounce_buffer == NULL) {
84561007b31SStefan Hajnoczi         ret = -ENOMEM;
84661007b31SStefan Hajnoczi         goto err;
84761007b31SStefan Hajnoczi     }
84861007b31SStefan Hajnoczi 
84961007b31SStefan Hajnoczi     qemu_iovec_init_external(&bounce_qiov, &iov, 1);
85061007b31SStefan Hajnoczi 
851*166fe960SKevin Wolf     ret = bdrv_driver_preadv(bs, cluster_sector_num * BDRV_SECTOR_SIZE,
852*166fe960SKevin Wolf                              cluster_nb_sectors * BDRV_SECTOR_SIZE,
853*166fe960SKevin Wolf                              &bounce_qiov, 0);
85461007b31SStefan Hajnoczi     if (ret < 0) {
85561007b31SStefan Hajnoczi         goto err;
85661007b31SStefan Hajnoczi     }
85761007b31SStefan Hajnoczi 
85861007b31SStefan Hajnoczi     if (drv->bdrv_co_write_zeroes &&
85961007b31SStefan Hajnoczi         buffer_is_zero(bounce_buffer, iov.iov_len)) {
86061007b31SStefan Hajnoczi         ret = bdrv_co_do_write_zeroes(bs, cluster_sector_num,
86161007b31SStefan Hajnoczi                                       cluster_nb_sectors, 0);
86261007b31SStefan Hajnoczi     } else {
86361007b31SStefan Hajnoczi         /* This does not change the data on the disk, it is not necessary
86461007b31SStefan Hajnoczi          * to flush even in cache=writethrough mode.
86561007b31SStefan Hajnoczi          */
86661007b31SStefan Hajnoczi         ret = drv->bdrv_co_writev(bs, cluster_sector_num, cluster_nb_sectors,
86761007b31SStefan Hajnoczi                                   &bounce_qiov);
86861007b31SStefan Hajnoczi     }
86961007b31SStefan Hajnoczi 
87061007b31SStefan Hajnoczi     if (ret < 0) {
87161007b31SStefan Hajnoczi         /* It might be okay to ignore write errors for guest requests.  If this
87261007b31SStefan Hajnoczi          * is a deliberate copy-on-read then we don't want to ignore the error.
87361007b31SStefan Hajnoczi          * Simply report it in all cases.
87461007b31SStefan Hajnoczi          */
87561007b31SStefan Hajnoczi         goto err;
87661007b31SStefan Hajnoczi     }
87761007b31SStefan Hajnoczi 
87861007b31SStefan Hajnoczi     skip_bytes = (sector_num - cluster_sector_num) * BDRV_SECTOR_SIZE;
87961007b31SStefan Hajnoczi     qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes,
88061007b31SStefan Hajnoczi                         nb_sectors * BDRV_SECTOR_SIZE);
88161007b31SStefan Hajnoczi 
88261007b31SStefan Hajnoczi err:
88361007b31SStefan Hajnoczi     qemu_vfree(bounce_buffer);
88461007b31SStefan Hajnoczi     return ret;
88561007b31SStefan Hajnoczi }
88661007b31SStefan Hajnoczi 
88761007b31SStefan Hajnoczi /*
88861007b31SStefan Hajnoczi  * Forwards an already correctly aligned request to the BlockDriver. This
88961007b31SStefan Hajnoczi  * handles copy on read and zeroing after EOF; any other features must be
89061007b31SStefan Hajnoczi  * implemented by the caller.
89161007b31SStefan Hajnoczi  */
89261007b31SStefan Hajnoczi static int coroutine_fn bdrv_aligned_preadv(BlockDriverState *bs,
89361007b31SStefan Hajnoczi     BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
89461007b31SStefan Hajnoczi     int64_t align, QEMUIOVector *qiov, int flags)
89561007b31SStefan Hajnoczi {
89661007b31SStefan Hajnoczi     int ret;
89761007b31SStefan Hajnoczi 
89861007b31SStefan Hajnoczi     int64_t sector_num = offset >> BDRV_SECTOR_BITS;
89961007b31SStefan Hajnoczi     unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS;
90061007b31SStefan Hajnoczi 
90161007b31SStefan Hajnoczi     assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
90261007b31SStefan Hajnoczi     assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
90361007b31SStefan Hajnoczi     assert(!qiov || bytes == qiov->size);
904abb06c5aSDaniel P. Berrange     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
90561007b31SStefan Hajnoczi 
90661007b31SStefan Hajnoczi     /* Handle Copy on Read and associated serialisation */
90761007b31SStefan Hajnoczi     if (flags & BDRV_REQ_COPY_ON_READ) {
90861007b31SStefan Hajnoczi         /* If we touch the same cluster it counts as an overlap.  This
90961007b31SStefan Hajnoczi          * guarantees that allocating writes will be serialized and not race
91061007b31SStefan Hajnoczi          * with each other for the same cluster.  For example, in copy-on-read
91161007b31SStefan Hajnoczi          * it ensures that the CoR read and write operations are atomic and
91261007b31SStefan Hajnoczi          * guest writes cannot interleave between them. */
91361007b31SStefan Hajnoczi         mark_request_serialising(req, bdrv_get_cluster_size(bs));
91461007b31SStefan Hajnoczi     }
91561007b31SStefan Hajnoczi 
91661408b25SFam Zheng     if (!(flags & BDRV_REQ_NO_SERIALISING)) {
91761007b31SStefan Hajnoczi         wait_serialising_requests(req);
91861408b25SFam Zheng     }
91961007b31SStefan Hajnoczi 
92061007b31SStefan Hajnoczi     if (flags & BDRV_REQ_COPY_ON_READ) {
92161007b31SStefan Hajnoczi         int pnum;
92261007b31SStefan Hajnoczi 
92361007b31SStefan Hajnoczi         ret = bdrv_is_allocated(bs, sector_num, nb_sectors, &pnum);
92461007b31SStefan Hajnoczi         if (ret < 0) {
92561007b31SStefan Hajnoczi             goto out;
92661007b31SStefan Hajnoczi         }
92761007b31SStefan Hajnoczi 
92861007b31SStefan Hajnoczi         if (!ret || pnum != nb_sectors) {
92961007b31SStefan Hajnoczi             ret = bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors, qiov);
93061007b31SStefan Hajnoczi             goto out;
93161007b31SStefan Hajnoczi         }
93261007b31SStefan Hajnoczi     }
93361007b31SStefan Hajnoczi 
93461007b31SStefan Hajnoczi     /* Forward the request to the BlockDriver */
93561007b31SStefan Hajnoczi     if (!bs->zero_beyond_eof) {
936*166fe960SKevin Wolf         ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0);
93761007b31SStefan Hajnoczi     } else {
93861007b31SStefan Hajnoczi         /* Read zeros after EOF */
93961007b31SStefan Hajnoczi         int64_t total_sectors, max_nb_sectors;
94061007b31SStefan Hajnoczi 
94161007b31SStefan Hajnoczi         total_sectors = bdrv_nb_sectors(bs);
94261007b31SStefan Hajnoczi         if (total_sectors < 0) {
94361007b31SStefan Hajnoczi             ret = total_sectors;
94461007b31SStefan Hajnoczi             goto out;
94561007b31SStefan Hajnoczi         }
94661007b31SStefan Hajnoczi 
94761007b31SStefan Hajnoczi         max_nb_sectors = ROUND_UP(MAX(0, total_sectors - sector_num),
94861007b31SStefan Hajnoczi                                   align >> BDRV_SECTOR_BITS);
94961007b31SStefan Hajnoczi         if (nb_sectors < max_nb_sectors) {
950*166fe960SKevin Wolf             ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0);
95161007b31SStefan Hajnoczi         } else if (max_nb_sectors > 0) {
95261007b31SStefan Hajnoczi             QEMUIOVector local_qiov;
95361007b31SStefan Hajnoczi 
95461007b31SStefan Hajnoczi             qemu_iovec_init(&local_qiov, qiov->niov);
95561007b31SStefan Hajnoczi             qemu_iovec_concat(&local_qiov, qiov, 0,
95661007b31SStefan Hajnoczi                               max_nb_sectors * BDRV_SECTOR_SIZE);
95761007b31SStefan Hajnoczi 
958*166fe960SKevin Wolf             ret = bdrv_driver_preadv(bs, offset,
959*166fe960SKevin Wolf                                      max_nb_sectors * BDRV_SECTOR_SIZE,
960*166fe960SKevin Wolf                                      &local_qiov, 0);
96161007b31SStefan Hajnoczi 
96261007b31SStefan Hajnoczi             qemu_iovec_destroy(&local_qiov);
96361007b31SStefan Hajnoczi         } else {
96461007b31SStefan Hajnoczi             ret = 0;
96561007b31SStefan Hajnoczi         }
96661007b31SStefan Hajnoczi 
96761007b31SStefan Hajnoczi         /* Reading beyond end of file is supposed to produce zeroes */
96861007b31SStefan Hajnoczi         if (ret == 0 && total_sectors < sector_num + nb_sectors) {
96961007b31SStefan Hajnoczi             uint64_t offset = MAX(0, total_sectors - sector_num);
97061007b31SStefan Hajnoczi             uint64_t bytes = (sector_num + nb_sectors - offset) *
97161007b31SStefan Hajnoczi                               BDRV_SECTOR_SIZE;
97261007b31SStefan Hajnoczi             qemu_iovec_memset(qiov, offset * BDRV_SECTOR_SIZE, 0, bytes);
97361007b31SStefan Hajnoczi         }
97461007b31SStefan Hajnoczi     }
97561007b31SStefan Hajnoczi 
97661007b31SStefan Hajnoczi out:
97761007b31SStefan Hajnoczi     return ret;
97861007b31SStefan Hajnoczi }
97961007b31SStefan Hajnoczi 
98061007b31SStefan Hajnoczi /*
98161007b31SStefan Hajnoczi  * Handle a read request in coroutine context
98261007b31SStefan Hajnoczi  */
9831bf1cbc9SKevin Wolf int coroutine_fn bdrv_co_do_preadv(BlockDriverState *bs,
98461007b31SStefan Hajnoczi     int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
98561007b31SStefan Hajnoczi     BdrvRequestFlags flags)
98661007b31SStefan Hajnoczi {
98761007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
98861007b31SStefan Hajnoczi     BdrvTrackedRequest req;
98961007b31SStefan Hajnoczi 
990d01c07f2SFam Zheng     /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */
991d01c07f2SFam Zheng     uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
99261007b31SStefan Hajnoczi     uint8_t *head_buf = NULL;
99361007b31SStefan Hajnoczi     uint8_t *tail_buf = NULL;
99461007b31SStefan Hajnoczi     QEMUIOVector local_qiov;
99561007b31SStefan Hajnoczi     bool use_local_qiov = false;
99661007b31SStefan Hajnoczi     int ret;
99761007b31SStefan Hajnoczi 
99861007b31SStefan Hajnoczi     if (!drv) {
99961007b31SStefan Hajnoczi         return -ENOMEDIUM;
100061007b31SStefan Hajnoczi     }
100161007b31SStefan Hajnoczi 
100261007b31SStefan Hajnoczi     ret = bdrv_check_byte_request(bs, offset, bytes);
100361007b31SStefan Hajnoczi     if (ret < 0) {
100461007b31SStefan Hajnoczi         return ret;
100561007b31SStefan Hajnoczi     }
100661007b31SStefan Hajnoczi 
10079568b511SWen Congyang     /* Don't do copy-on-read if we read data before write operation */
100861408b25SFam Zheng     if (bs->copy_on_read && !(flags & BDRV_REQ_NO_SERIALISING)) {
100961007b31SStefan Hajnoczi         flags |= BDRV_REQ_COPY_ON_READ;
101061007b31SStefan Hajnoczi     }
101161007b31SStefan Hajnoczi 
101261007b31SStefan Hajnoczi     /* throttling disk I/O */
1013ce0f1412SPaolo Bonzini     if (bs->throttle_state) {
101476f4afb4SAlberto Garcia         throttle_group_co_io_limits_intercept(bs, bytes, false);
101561007b31SStefan Hajnoczi     }
101661007b31SStefan Hajnoczi 
101761007b31SStefan Hajnoczi     /* Align read if necessary by padding qiov */
101861007b31SStefan Hajnoczi     if (offset & (align - 1)) {
101961007b31SStefan Hajnoczi         head_buf = qemu_blockalign(bs, align);
102061007b31SStefan Hajnoczi         qemu_iovec_init(&local_qiov, qiov->niov + 2);
102161007b31SStefan Hajnoczi         qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
102261007b31SStefan Hajnoczi         qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
102361007b31SStefan Hajnoczi         use_local_qiov = true;
102461007b31SStefan Hajnoczi 
102561007b31SStefan Hajnoczi         bytes += offset & (align - 1);
102661007b31SStefan Hajnoczi         offset = offset & ~(align - 1);
102761007b31SStefan Hajnoczi     }
102861007b31SStefan Hajnoczi 
102961007b31SStefan Hajnoczi     if ((offset + bytes) & (align - 1)) {
103061007b31SStefan Hajnoczi         if (!use_local_qiov) {
103161007b31SStefan Hajnoczi             qemu_iovec_init(&local_qiov, qiov->niov + 1);
103261007b31SStefan Hajnoczi             qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
103361007b31SStefan Hajnoczi             use_local_qiov = true;
103461007b31SStefan Hajnoczi         }
103561007b31SStefan Hajnoczi         tail_buf = qemu_blockalign(bs, align);
103661007b31SStefan Hajnoczi         qemu_iovec_add(&local_qiov, tail_buf,
103761007b31SStefan Hajnoczi                        align - ((offset + bytes) & (align - 1)));
103861007b31SStefan Hajnoczi 
103961007b31SStefan Hajnoczi         bytes = ROUND_UP(bytes, align);
104061007b31SStefan Hajnoczi     }
104161007b31SStefan Hajnoczi 
1042ebde595cSFam Zheng     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
104361007b31SStefan Hajnoczi     ret = bdrv_aligned_preadv(bs, &req, offset, bytes, align,
104461007b31SStefan Hajnoczi                               use_local_qiov ? &local_qiov : qiov,
104561007b31SStefan Hajnoczi                               flags);
104661007b31SStefan Hajnoczi     tracked_request_end(&req);
104761007b31SStefan Hajnoczi 
104861007b31SStefan Hajnoczi     if (use_local_qiov) {
104961007b31SStefan Hajnoczi         qemu_iovec_destroy(&local_qiov);
105061007b31SStefan Hajnoczi         qemu_vfree(head_buf);
105161007b31SStefan Hajnoczi         qemu_vfree(tail_buf);
105261007b31SStefan Hajnoczi     }
105361007b31SStefan Hajnoczi 
105461007b31SStefan Hajnoczi     return ret;
105561007b31SStefan Hajnoczi }
105661007b31SStefan Hajnoczi 
105761007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_do_readv(BlockDriverState *bs,
105861007b31SStefan Hajnoczi     int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
105961007b31SStefan Hajnoczi     BdrvRequestFlags flags)
106061007b31SStefan Hajnoczi {
106161007b31SStefan Hajnoczi     if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
106261007b31SStefan Hajnoczi         return -EINVAL;
106361007b31SStefan Hajnoczi     }
106461007b31SStefan Hajnoczi 
106561007b31SStefan Hajnoczi     return bdrv_co_do_preadv(bs, sector_num << BDRV_SECTOR_BITS,
106661007b31SStefan Hajnoczi                              nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
106761007b31SStefan Hajnoczi }
106861007b31SStefan Hajnoczi 
106961007b31SStefan Hajnoczi int coroutine_fn bdrv_co_readv(BlockDriverState *bs, int64_t sector_num,
107061007b31SStefan Hajnoczi     int nb_sectors, QEMUIOVector *qiov)
107161007b31SStefan Hajnoczi {
107261007b31SStefan Hajnoczi     trace_bdrv_co_readv(bs, sector_num, nb_sectors);
107361007b31SStefan Hajnoczi 
107461007b31SStefan Hajnoczi     return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 0);
107561007b31SStefan Hajnoczi }
107661007b31SStefan Hajnoczi 
107761408b25SFam Zheng int coroutine_fn bdrv_co_readv_no_serialising(BlockDriverState *bs,
10789568b511SWen Congyang     int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
10799568b511SWen Congyang {
108061408b25SFam Zheng     trace_bdrv_co_readv_no_serialising(bs, sector_num, nb_sectors);
10819568b511SWen Congyang 
10829568b511SWen Congyang     return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov,
108361408b25SFam Zheng                             BDRV_REQ_NO_SERIALISING);
10849568b511SWen Congyang }
10859568b511SWen Congyang 
108661007b31SStefan Hajnoczi int coroutine_fn bdrv_co_copy_on_readv(BlockDriverState *bs,
108761007b31SStefan Hajnoczi     int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
108861007b31SStefan Hajnoczi {
108961007b31SStefan Hajnoczi     trace_bdrv_co_copy_on_readv(bs, sector_num, nb_sectors);
109061007b31SStefan Hajnoczi 
109161007b31SStefan Hajnoczi     return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov,
109261007b31SStefan Hajnoczi                             BDRV_REQ_COPY_ON_READ);
109361007b31SStefan Hajnoczi }
109461007b31SStefan Hajnoczi 
109561007b31SStefan Hajnoczi #define MAX_WRITE_ZEROES_BOUNCE_BUFFER 32768
109661007b31SStefan Hajnoczi 
109761007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
109861007b31SStefan Hajnoczi     int64_t sector_num, int nb_sectors, BdrvRequestFlags flags)
109961007b31SStefan Hajnoczi {
110061007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
110161007b31SStefan Hajnoczi     QEMUIOVector qiov;
110261007b31SStefan Hajnoczi     struct iovec iov = {0};
110361007b31SStefan Hajnoczi     int ret = 0;
110461007b31SStefan Hajnoczi 
110561007b31SStefan Hajnoczi     int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_write_zeroes,
110661007b31SStefan Hajnoczi                                         BDRV_REQUEST_MAX_SECTORS);
110761007b31SStefan Hajnoczi 
110861007b31SStefan Hajnoczi     while (nb_sectors > 0 && !ret) {
110961007b31SStefan Hajnoczi         int num = nb_sectors;
111061007b31SStefan Hajnoczi 
111161007b31SStefan Hajnoczi         /* Align request.  Block drivers can expect the "bulk" of the request
111261007b31SStefan Hajnoczi          * to be aligned.
111361007b31SStefan Hajnoczi          */
111461007b31SStefan Hajnoczi         if (bs->bl.write_zeroes_alignment
111561007b31SStefan Hajnoczi             && num > bs->bl.write_zeroes_alignment) {
111661007b31SStefan Hajnoczi             if (sector_num % bs->bl.write_zeroes_alignment != 0) {
111761007b31SStefan Hajnoczi                 /* Make a small request up to the first aligned sector.  */
111861007b31SStefan Hajnoczi                 num = bs->bl.write_zeroes_alignment;
111961007b31SStefan Hajnoczi                 num -= sector_num % bs->bl.write_zeroes_alignment;
112061007b31SStefan Hajnoczi             } else if ((sector_num + num) % bs->bl.write_zeroes_alignment != 0) {
112161007b31SStefan Hajnoczi                 /* Shorten the request to the last aligned sector.  num cannot
112261007b31SStefan Hajnoczi                  * underflow because num > bs->bl.write_zeroes_alignment.
112361007b31SStefan Hajnoczi                  */
112461007b31SStefan Hajnoczi                 num -= (sector_num + num) % bs->bl.write_zeroes_alignment;
112561007b31SStefan Hajnoczi             }
112661007b31SStefan Hajnoczi         }
112761007b31SStefan Hajnoczi 
112861007b31SStefan Hajnoczi         /* limit request size */
112961007b31SStefan Hajnoczi         if (num > max_write_zeroes) {
113061007b31SStefan Hajnoczi             num = max_write_zeroes;
113161007b31SStefan Hajnoczi         }
113261007b31SStefan Hajnoczi 
113361007b31SStefan Hajnoczi         ret = -ENOTSUP;
113461007b31SStefan Hajnoczi         /* First try the efficient write zeroes operation */
113561007b31SStefan Hajnoczi         if (drv->bdrv_co_write_zeroes) {
113661007b31SStefan Hajnoczi             ret = drv->bdrv_co_write_zeroes(bs, sector_num, num, flags);
113761007b31SStefan Hajnoczi         }
113861007b31SStefan Hajnoczi 
113961007b31SStefan Hajnoczi         if (ret == -ENOTSUP) {
114061007b31SStefan Hajnoczi             /* Fall back to bounce buffer if write zeroes is unsupported */
114161007b31SStefan Hajnoczi             int max_xfer_len = MIN_NON_ZERO(bs->bl.max_transfer_length,
114261007b31SStefan Hajnoczi                                             MAX_WRITE_ZEROES_BOUNCE_BUFFER);
114361007b31SStefan Hajnoczi             num = MIN(num, max_xfer_len);
114461007b31SStefan Hajnoczi             iov.iov_len = num * BDRV_SECTOR_SIZE;
114561007b31SStefan Hajnoczi             if (iov.iov_base == NULL) {
114661007b31SStefan Hajnoczi                 iov.iov_base = qemu_try_blockalign(bs, num * BDRV_SECTOR_SIZE);
114761007b31SStefan Hajnoczi                 if (iov.iov_base == NULL) {
114861007b31SStefan Hajnoczi                     ret = -ENOMEM;
114961007b31SStefan Hajnoczi                     goto fail;
115061007b31SStefan Hajnoczi                 }
115161007b31SStefan Hajnoczi                 memset(iov.iov_base, 0, num * BDRV_SECTOR_SIZE);
115261007b31SStefan Hajnoczi             }
115361007b31SStefan Hajnoczi             qemu_iovec_init_external(&qiov, &iov, 1);
115461007b31SStefan Hajnoczi 
115561007b31SStefan Hajnoczi             ret = drv->bdrv_co_writev(bs, sector_num, num, &qiov);
115661007b31SStefan Hajnoczi 
115761007b31SStefan Hajnoczi             /* Keep bounce buffer around if it is big enough for all
115861007b31SStefan Hajnoczi              * all future requests.
115961007b31SStefan Hajnoczi              */
116061007b31SStefan Hajnoczi             if (num < max_xfer_len) {
116161007b31SStefan Hajnoczi                 qemu_vfree(iov.iov_base);
116261007b31SStefan Hajnoczi                 iov.iov_base = NULL;
116361007b31SStefan Hajnoczi             }
116461007b31SStefan Hajnoczi         }
116561007b31SStefan Hajnoczi 
116661007b31SStefan Hajnoczi         sector_num += num;
116761007b31SStefan Hajnoczi         nb_sectors -= num;
116861007b31SStefan Hajnoczi     }
116961007b31SStefan Hajnoczi 
117061007b31SStefan Hajnoczi fail:
117161007b31SStefan Hajnoczi     qemu_vfree(iov.iov_base);
117261007b31SStefan Hajnoczi     return ret;
117361007b31SStefan Hajnoczi }
117461007b31SStefan Hajnoczi 
117561007b31SStefan Hajnoczi /*
117661007b31SStefan Hajnoczi  * Forwards an already correctly aligned write request to the BlockDriver.
117761007b31SStefan Hajnoczi  */
117861007b31SStefan Hajnoczi static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs,
117961007b31SStefan Hajnoczi     BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
118061007b31SStefan Hajnoczi     QEMUIOVector *qiov, int flags)
118161007b31SStefan Hajnoczi {
118261007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
118361007b31SStefan Hajnoczi     bool waited;
118461007b31SStefan Hajnoczi     int ret;
118561007b31SStefan Hajnoczi 
118661007b31SStefan Hajnoczi     int64_t sector_num = offset >> BDRV_SECTOR_BITS;
118761007b31SStefan Hajnoczi     unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS;
118861007b31SStefan Hajnoczi 
118961007b31SStefan Hajnoczi     assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
119061007b31SStefan Hajnoczi     assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
119161007b31SStefan Hajnoczi     assert(!qiov || bytes == qiov->size);
1192abb06c5aSDaniel P. Berrange     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
119361007b31SStefan Hajnoczi 
119461007b31SStefan Hajnoczi     waited = wait_serialising_requests(req);
119561007b31SStefan Hajnoczi     assert(!waited || !req->serialising);
119661007b31SStefan Hajnoczi     assert(req->overlap_offset <= offset);
119761007b31SStefan Hajnoczi     assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
119861007b31SStefan Hajnoczi 
119961007b31SStefan Hajnoczi     ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req);
120061007b31SStefan Hajnoczi 
120161007b31SStefan Hajnoczi     if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
120261007b31SStefan Hajnoczi         !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_write_zeroes &&
120361007b31SStefan Hajnoczi         qemu_iovec_is_zero(qiov)) {
120461007b31SStefan Hajnoczi         flags |= BDRV_REQ_ZERO_WRITE;
120561007b31SStefan Hajnoczi         if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
120661007b31SStefan Hajnoczi             flags |= BDRV_REQ_MAY_UNMAP;
120761007b31SStefan Hajnoczi         }
120861007b31SStefan Hajnoczi     }
120961007b31SStefan Hajnoczi 
121061007b31SStefan Hajnoczi     if (ret < 0) {
121161007b31SStefan Hajnoczi         /* Do nothing, write notifier decided to fail this request */
121261007b31SStefan Hajnoczi     } else if (flags & BDRV_REQ_ZERO_WRITE) {
12139a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
121461007b31SStefan Hajnoczi         ret = bdrv_co_do_write_zeroes(bs, sector_num, nb_sectors, flags);
121593f5e6d8SKevin Wolf     } else if (drv->bdrv_co_writev_flags) {
121693f5e6d8SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV);
121793f5e6d8SKevin Wolf         ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov,
121893f5e6d8SKevin Wolf                                         flags);
121961007b31SStefan Hajnoczi     } else {
122093f5e6d8SKevin Wolf         assert(drv->supported_write_flags == 0);
12219a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV);
122261007b31SStefan Hajnoczi         ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov);
122361007b31SStefan Hajnoczi     }
12249a4f4c31SKevin Wolf     bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
122561007b31SStefan Hajnoczi 
122693f5e6d8SKevin Wolf     if (ret == 0 && (flags & BDRV_REQ_FUA) &&
122793f5e6d8SKevin Wolf         !(drv->supported_write_flags & BDRV_REQ_FUA))
122893f5e6d8SKevin Wolf     {
122961007b31SStefan Hajnoczi         ret = bdrv_co_flush(bs);
123061007b31SStefan Hajnoczi     }
123161007b31SStefan Hajnoczi 
123261007b31SStefan Hajnoczi     bdrv_set_dirty(bs, sector_num, nb_sectors);
123361007b31SStefan Hajnoczi 
123453d8f9d8SMax Reitz     if (bs->wr_highest_offset < offset + bytes) {
123553d8f9d8SMax Reitz         bs->wr_highest_offset = offset + bytes;
123653d8f9d8SMax Reitz     }
123761007b31SStefan Hajnoczi 
123861007b31SStefan Hajnoczi     if (ret >= 0) {
123961007b31SStefan Hajnoczi         bs->total_sectors = MAX(bs->total_sectors, sector_num + nb_sectors);
124061007b31SStefan Hajnoczi     }
124161007b31SStefan Hajnoczi 
124261007b31SStefan Hajnoczi     return ret;
124361007b31SStefan Hajnoczi }
124461007b31SStefan Hajnoczi 
12459eeb6dd1SFam Zheng static int coroutine_fn bdrv_co_do_zero_pwritev(BlockDriverState *bs,
12469eeb6dd1SFam Zheng                                                 int64_t offset,
12479eeb6dd1SFam Zheng                                                 unsigned int bytes,
12489eeb6dd1SFam Zheng                                                 BdrvRequestFlags flags,
12499eeb6dd1SFam Zheng                                                 BdrvTrackedRequest *req)
12509eeb6dd1SFam Zheng {
12519eeb6dd1SFam Zheng     uint8_t *buf = NULL;
12529eeb6dd1SFam Zheng     QEMUIOVector local_qiov;
12539eeb6dd1SFam Zheng     struct iovec iov;
12549eeb6dd1SFam Zheng     uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
12559eeb6dd1SFam Zheng     unsigned int head_padding_bytes, tail_padding_bytes;
12569eeb6dd1SFam Zheng     int ret = 0;
12579eeb6dd1SFam Zheng 
12589eeb6dd1SFam Zheng     head_padding_bytes = offset & (align - 1);
12599eeb6dd1SFam Zheng     tail_padding_bytes = align - ((offset + bytes) & (align - 1));
12609eeb6dd1SFam Zheng 
12619eeb6dd1SFam Zheng 
12629eeb6dd1SFam Zheng     assert(flags & BDRV_REQ_ZERO_WRITE);
12639eeb6dd1SFam Zheng     if (head_padding_bytes || tail_padding_bytes) {
12649eeb6dd1SFam Zheng         buf = qemu_blockalign(bs, align);
12659eeb6dd1SFam Zheng         iov = (struct iovec) {
12669eeb6dd1SFam Zheng             .iov_base   = buf,
12679eeb6dd1SFam Zheng             .iov_len    = align,
12689eeb6dd1SFam Zheng         };
12699eeb6dd1SFam Zheng         qemu_iovec_init_external(&local_qiov, &iov, 1);
12709eeb6dd1SFam Zheng     }
12719eeb6dd1SFam Zheng     if (head_padding_bytes) {
12729eeb6dd1SFam Zheng         uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes);
12739eeb6dd1SFam Zheng 
12749eeb6dd1SFam Zheng         /* RMW the unaligned part before head. */
12759eeb6dd1SFam Zheng         mark_request_serialising(req, align);
12769eeb6dd1SFam Zheng         wait_serialising_requests(req);
12779a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
12789eeb6dd1SFam Zheng         ret = bdrv_aligned_preadv(bs, req, offset & ~(align - 1), align,
12799eeb6dd1SFam Zheng                                   align, &local_qiov, 0);
12809eeb6dd1SFam Zheng         if (ret < 0) {
12819eeb6dd1SFam Zheng             goto fail;
12829eeb6dd1SFam Zheng         }
12839a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
12849eeb6dd1SFam Zheng 
12859eeb6dd1SFam Zheng         memset(buf + head_padding_bytes, 0, zero_bytes);
12869eeb6dd1SFam Zheng         ret = bdrv_aligned_pwritev(bs, req, offset & ~(align - 1), align,
12879eeb6dd1SFam Zheng                                    &local_qiov,
12889eeb6dd1SFam Zheng                                    flags & ~BDRV_REQ_ZERO_WRITE);
12899eeb6dd1SFam Zheng         if (ret < 0) {
12909eeb6dd1SFam Zheng             goto fail;
12919eeb6dd1SFam Zheng         }
12929eeb6dd1SFam Zheng         offset += zero_bytes;
12939eeb6dd1SFam Zheng         bytes -= zero_bytes;
12949eeb6dd1SFam Zheng     }
12959eeb6dd1SFam Zheng 
12969eeb6dd1SFam Zheng     assert(!bytes || (offset & (align - 1)) == 0);
12979eeb6dd1SFam Zheng     if (bytes >= align) {
12989eeb6dd1SFam Zheng         /* Write the aligned part in the middle. */
12999eeb6dd1SFam Zheng         uint64_t aligned_bytes = bytes & ~(align - 1);
13009eeb6dd1SFam Zheng         ret = bdrv_aligned_pwritev(bs, req, offset, aligned_bytes,
13019eeb6dd1SFam Zheng                                    NULL, flags);
13029eeb6dd1SFam Zheng         if (ret < 0) {
13039eeb6dd1SFam Zheng             goto fail;
13049eeb6dd1SFam Zheng         }
13059eeb6dd1SFam Zheng         bytes -= aligned_bytes;
13069eeb6dd1SFam Zheng         offset += aligned_bytes;
13079eeb6dd1SFam Zheng     }
13089eeb6dd1SFam Zheng 
13099eeb6dd1SFam Zheng     assert(!bytes || (offset & (align - 1)) == 0);
13109eeb6dd1SFam Zheng     if (bytes) {
13119eeb6dd1SFam Zheng         assert(align == tail_padding_bytes + bytes);
13129eeb6dd1SFam Zheng         /* RMW the unaligned part after tail. */
13139eeb6dd1SFam Zheng         mark_request_serialising(req, align);
13149eeb6dd1SFam Zheng         wait_serialising_requests(req);
13159a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
13169eeb6dd1SFam Zheng         ret = bdrv_aligned_preadv(bs, req, offset, align,
13179eeb6dd1SFam Zheng                                   align, &local_qiov, 0);
13189eeb6dd1SFam Zheng         if (ret < 0) {
13199eeb6dd1SFam Zheng             goto fail;
13209eeb6dd1SFam Zheng         }
13219a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
13229eeb6dd1SFam Zheng 
13239eeb6dd1SFam Zheng         memset(buf, 0, bytes);
13249eeb6dd1SFam Zheng         ret = bdrv_aligned_pwritev(bs, req, offset, align,
13259eeb6dd1SFam Zheng                                    &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE);
13269eeb6dd1SFam Zheng     }
13279eeb6dd1SFam Zheng fail:
13289eeb6dd1SFam Zheng     qemu_vfree(buf);
13299eeb6dd1SFam Zheng     return ret;
13309eeb6dd1SFam Zheng 
13319eeb6dd1SFam Zheng }
13329eeb6dd1SFam Zheng 
133361007b31SStefan Hajnoczi /*
133461007b31SStefan Hajnoczi  * Handle a write request in coroutine context
133561007b31SStefan Hajnoczi  */
1336a8823a3bSKevin Wolf int coroutine_fn bdrv_co_do_pwritev(BlockDriverState *bs,
133761007b31SStefan Hajnoczi     int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
133861007b31SStefan Hajnoczi     BdrvRequestFlags flags)
133961007b31SStefan Hajnoczi {
134061007b31SStefan Hajnoczi     BdrvTrackedRequest req;
1341d01c07f2SFam Zheng     /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */
1342d01c07f2SFam Zheng     uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
134361007b31SStefan Hajnoczi     uint8_t *head_buf = NULL;
134461007b31SStefan Hajnoczi     uint8_t *tail_buf = NULL;
134561007b31SStefan Hajnoczi     QEMUIOVector local_qiov;
134661007b31SStefan Hajnoczi     bool use_local_qiov = false;
134761007b31SStefan Hajnoczi     int ret;
134861007b31SStefan Hajnoczi 
134961007b31SStefan Hajnoczi     if (!bs->drv) {
135061007b31SStefan Hajnoczi         return -ENOMEDIUM;
135161007b31SStefan Hajnoczi     }
135261007b31SStefan Hajnoczi     if (bs->read_only) {
1353eaf5fe2dSPaolo Bonzini         return -EPERM;
135461007b31SStefan Hajnoczi     }
135504c01a5cSKevin Wolf     assert(!(bs->open_flags & BDRV_O_INACTIVE));
135661007b31SStefan Hajnoczi 
135761007b31SStefan Hajnoczi     ret = bdrv_check_byte_request(bs, offset, bytes);
135861007b31SStefan Hajnoczi     if (ret < 0) {
135961007b31SStefan Hajnoczi         return ret;
136061007b31SStefan Hajnoczi     }
136161007b31SStefan Hajnoczi 
136261007b31SStefan Hajnoczi     /* throttling disk I/O */
1363ce0f1412SPaolo Bonzini     if (bs->throttle_state) {
136476f4afb4SAlberto Garcia         throttle_group_co_io_limits_intercept(bs, bytes, true);
136561007b31SStefan Hajnoczi     }
136661007b31SStefan Hajnoczi 
136761007b31SStefan Hajnoczi     /*
136861007b31SStefan Hajnoczi      * Align write if necessary by performing a read-modify-write cycle.
136961007b31SStefan Hajnoczi      * Pad qiov with the read parts and be sure to have a tracked request not
137061007b31SStefan Hajnoczi      * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle.
137161007b31SStefan Hajnoczi      */
1372ebde595cSFam Zheng     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
137361007b31SStefan Hajnoczi 
13749eeb6dd1SFam Zheng     if (!qiov) {
13759eeb6dd1SFam Zheng         ret = bdrv_co_do_zero_pwritev(bs, offset, bytes, flags, &req);
13769eeb6dd1SFam Zheng         goto out;
13779eeb6dd1SFam Zheng     }
13789eeb6dd1SFam Zheng 
137961007b31SStefan Hajnoczi     if (offset & (align - 1)) {
138061007b31SStefan Hajnoczi         QEMUIOVector head_qiov;
138161007b31SStefan Hajnoczi         struct iovec head_iov;
138261007b31SStefan Hajnoczi 
138361007b31SStefan Hajnoczi         mark_request_serialising(&req, align);
138461007b31SStefan Hajnoczi         wait_serialising_requests(&req);
138561007b31SStefan Hajnoczi 
138661007b31SStefan Hajnoczi         head_buf = qemu_blockalign(bs, align);
138761007b31SStefan Hajnoczi         head_iov = (struct iovec) {
138861007b31SStefan Hajnoczi             .iov_base   = head_buf,
138961007b31SStefan Hajnoczi             .iov_len    = align,
139061007b31SStefan Hajnoczi         };
139161007b31SStefan Hajnoczi         qemu_iovec_init_external(&head_qiov, &head_iov, 1);
139261007b31SStefan Hajnoczi 
13939a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
139461007b31SStefan Hajnoczi         ret = bdrv_aligned_preadv(bs, &req, offset & ~(align - 1), align,
139561007b31SStefan Hajnoczi                                   align, &head_qiov, 0);
139661007b31SStefan Hajnoczi         if (ret < 0) {
139761007b31SStefan Hajnoczi             goto fail;
139861007b31SStefan Hajnoczi         }
13999a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
140061007b31SStefan Hajnoczi 
140161007b31SStefan Hajnoczi         qemu_iovec_init(&local_qiov, qiov->niov + 2);
140261007b31SStefan Hajnoczi         qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
140361007b31SStefan Hajnoczi         qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
140461007b31SStefan Hajnoczi         use_local_qiov = true;
140561007b31SStefan Hajnoczi 
140661007b31SStefan Hajnoczi         bytes += offset & (align - 1);
140761007b31SStefan Hajnoczi         offset = offset & ~(align - 1);
140861007b31SStefan Hajnoczi     }
140961007b31SStefan Hajnoczi 
141061007b31SStefan Hajnoczi     if ((offset + bytes) & (align - 1)) {
141161007b31SStefan Hajnoczi         QEMUIOVector tail_qiov;
141261007b31SStefan Hajnoczi         struct iovec tail_iov;
141361007b31SStefan Hajnoczi         size_t tail_bytes;
141461007b31SStefan Hajnoczi         bool waited;
141561007b31SStefan Hajnoczi 
141661007b31SStefan Hajnoczi         mark_request_serialising(&req, align);
141761007b31SStefan Hajnoczi         waited = wait_serialising_requests(&req);
141861007b31SStefan Hajnoczi         assert(!waited || !use_local_qiov);
141961007b31SStefan Hajnoczi 
142061007b31SStefan Hajnoczi         tail_buf = qemu_blockalign(bs, align);
142161007b31SStefan Hajnoczi         tail_iov = (struct iovec) {
142261007b31SStefan Hajnoczi             .iov_base   = tail_buf,
142361007b31SStefan Hajnoczi             .iov_len    = align,
142461007b31SStefan Hajnoczi         };
142561007b31SStefan Hajnoczi         qemu_iovec_init_external(&tail_qiov, &tail_iov, 1);
142661007b31SStefan Hajnoczi 
14279a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
142861007b31SStefan Hajnoczi         ret = bdrv_aligned_preadv(bs, &req, (offset + bytes) & ~(align - 1), align,
142961007b31SStefan Hajnoczi                                   align, &tail_qiov, 0);
143061007b31SStefan Hajnoczi         if (ret < 0) {
143161007b31SStefan Hajnoczi             goto fail;
143261007b31SStefan Hajnoczi         }
14339a4f4c31SKevin Wolf         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
143461007b31SStefan Hajnoczi 
143561007b31SStefan Hajnoczi         if (!use_local_qiov) {
143661007b31SStefan Hajnoczi             qemu_iovec_init(&local_qiov, qiov->niov + 1);
143761007b31SStefan Hajnoczi             qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
143861007b31SStefan Hajnoczi             use_local_qiov = true;
143961007b31SStefan Hajnoczi         }
144061007b31SStefan Hajnoczi 
144161007b31SStefan Hajnoczi         tail_bytes = (offset + bytes) & (align - 1);
144261007b31SStefan Hajnoczi         qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes);
144361007b31SStefan Hajnoczi 
144461007b31SStefan Hajnoczi         bytes = ROUND_UP(bytes, align);
144561007b31SStefan Hajnoczi     }
144661007b31SStefan Hajnoczi 
144761007b31SStefan Hajnoczi     ret = bdrv_aligned_pwritev(bs, &req, offset, bytes,
144861007b31SStefan Hajnoczi                                use_local_qiov ? &local_qiov : qiov,
144961007b31SStefan Hajnoczi                                flags);
145061007b31SStefan Hajnoczi 
145161007b31SStefan Hajnoczi fail:
145261007b31SStefan Hajnoczi 
145361007b31SStefan Hajnoczi     if (use_local_qiov) {
145461007b31SStefan Hajnoczi         qemu_iovec_destroy(&local_qiov);
145561007b31SStefan Hajnoczi     }
145661007b31SStefan Hajnoczi     qemu_vfree(head_buf);
145761007b31SStefan Hajnoczi     qemu_vfree(tail_buf);
14589eeb6dd1SFam Zheng out:
14599eeb6dd1SFam Zheng     tracked_request_end(&req);
146061007b31SStefan Hajnoczi     return ret;
146161007b31SStefan Hajnoczi }
146261007b31SStefan Hajnoczi 
146361007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_do_writev(BlockDriverState *bs,
146461007b31SStefan Hajnoczi     int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
146561007b31SStefan Hajnoczi     BdrvRequestFlags flags)
146661007b31SStefan Hajnoczi {
146761007b31SStefan Hajnoczi     if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
146861007b31SStefan Hajnoczi         return -EINVAL;
146961007b31SStefan Hajnoczi     }
147061007b31SStefan Hajnoczi 
147161007b31SStefan Hajnoczi     return bdrv_co_do_pwritev(bs, sector_num << BDRV_SECTOR_BITS,
147261007b31SStefan Hajnoczi                               nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
147361007b31SStefan Hajnoczi }
147461007b31SStefan Hajnoczi 
147561007b31SStefan Hajnoczi int coroutine_fn bdrv_co_writev(BlockDriverState *bs, int64_t sector_num,
147661007b31SStefan Hajnoczi     int nb_sectors, QEMUIOVector *qiov)
147761007b31SStefan Hajnoczi {
147861007b31SStefan Hajnoczi     trace_bdrv_co_writev(bs, sector_num, nb_sectors);
147961007b31SStefan Hajnoczi 
148061007b31SStefan Hajnoczi     return bdrv_co_do_writev(bs, sector_num, nb_sectors, qiov, 0);
148161007b31SStefan Hajnoczi }
148261007b31SStefan Hajnoczi 
148361007b31SStefan Hajnoczi int coroutine_fn bdrv_co_write_zeroes(BlockDriverState *bs,
148461007b31SStefan Hajnoczi                                       int64_t sector_num, int nb_sectors,
148561007b31SStefan Hajnoczi                                       BdrvRequestFlags flags)
148661007b31SStefan Hajnoczi {
148761007b31SStefan Hajnoczi     trace_bdrv_co_write_zeroes(bs, sector_num, nb_sectors, flags);
148861007b31SStefan Hajnoczi 
148961007b31SStefan Hajnoczi     if (!(bs->open_flags & BDRV_O_UNMAP)) {
149061007b31SStefan Hajnoczi         flags &= ~BDRV_REQ_MAY_UNMAP;
149161007b31SStefan Hajnoczi     }
149261007b31SStefan Hajnoczi 
1493d01c07f2SFam Zheng     return bdrv_co_do_writev(bs, sector_num, nb_sectors, NULL,
149461007b31SStefan Hajnoczi                              BDRV_REQ_ZERO_WRITE | flags);
149561007b31SStefan Hajnoczi }
149661007b31SStefan Hajnoczi 
149761007b31SStefan Hajnoczi typedef struct BdrvCoGetBlockStatusData {
149861007b31SStefan Hajnoczi     BlockDriverState *bs;
149961007b31SStefan Hajnoczi     BlockDriverState *base;
150067a0fd2aSFam Zheng     BlockDriverState **file;
150161007b31SStefan Hajnoczi     int64_t sector_num;
150261007b31SStefan Hajnoczi     int nb_sectors;
150361007b31SStefan Hajnoczi     int *pnum;
150461007b31SStefan Hajnoczi     int64_t ret;
150561007b31SStefan Hajnoczi     bool done;
150661007b31SStefan Hajnoczi } BdrvCoGetBlockStatusData;
150761007b31SStefan Hajnoczi 
150861007b31SStefan Hajnoczi /*
150961007b31SStefan Hajnoczi  * Returns the allocation status of the specified sectors.
151061007b31SStefan Hajnoczi  * Drivers not implementing the functionality are assumed to not support
151161007b31SStefan Hajnoczi  * backing files, hence all their sectors are reported as allocated.
151261007b31SStefan Hajnoczi  *
151361007b31SStefan Hajnoczi  * If 'sector_num' is beyond the end of the disk image the return value is 0
151461007b31SStefan Hajnoczi  * and 'pnum' is set to 0.
151561007b31SStefan Hajnoczi  *
151661007b31SStefan Hajnoczi  * 'pnum' is set to the number of sectors (including and immediately following
151761007b31SStefan Hajnoczi  * the specified sector) that are known to be in the same
151861007b31SStefan Hajnoczi  * allocated/unallocated state.
151961007b31SStefan Hajnoczi  *
152061007b31SStefan Hajnoczi  * 'nb_sectors' is the max value 'pnum' should be set to.  If nb_sectors goes
152161007b31SStefan Hajnoczi  * beyond the end of the disk image it will be clamped.
152267a0fd2aSFam Zheng  *
152367a0fd2aSFam Zheng  * If returned value is positive and BDRV_BLOCK_OFFSET_VALID bit is set, 'file'
152467a0fd2aSFam Zheng  * points to the BDS which the sector range is allocated in.
152561007b31SStefan Hajnoczi  */
152661007b31SStefan Hajnoczi static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
152761007b31SStefan Hajnoczi                                                      int64_t sector_num,
152867a0fd2aSFam Zheng                                                      int nb_sectors, int *pnum,
152967a0fd2aSFam Zheng                                                      BlockDriverState **file)
153061007b31SStefan Hajnoczi {
153161007b31SStefan Hajnoczi     int64_t total_sectors;
153261007b31SStefan Hajnoczi     int64_t n;
153361007b31SStefan Hajnoczi     int64_t ret, ret2;
153461007b31SStefan Hajnoczi 
153561007b31SStefan Hajnoczi     total_sectors = bdrv_nb_sectors(bs);
153661007b31SStefan Hajnoczi     if (total_sectors < 0) {
153761007b31SStefan Hajnoczi         return total_sectors;
153861007b31SStefan Hajnoczi     }
153961007b31SStefan Hajnoczi 
154061007b31SStefan Hajnoczi     if (sector_num >= total_sectors) {
154161007b31SStefan Hajnoczi         *pnum = 0;
154261007b31SStefan Hajnoczi         return 0;
154361007b31SStefan Hajnoczi     }
154461007b31SStefan Hajnoczi 
154561007b31SStefan Hajnoczi     n = total_sectors - sector_num;
154661007b31SStefan Hajnoczi     if (n < nb_sectors) {
154761007b31SStefan Hajnoczi         nb_sectors = n;
154861007b31SStefan Hajnoczi     }
154961007b31SStefan Hajnoczi 
155061007b31SStefan Hajnoczi     if (!bs->drv->bdrv_co_get_block_status) {
155161007b31SStefan Hajnoczi         *pnum = nb_sectors;
155261007b31SStefan Hajnoczi         ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
155361007b31SStefan Hajnoczi         if (bs->drv->protocol_name) {
155461007b31SStefan Hajnoczi             ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE);
155561007b31SStefan Hajnoczi         }
155661007b31SStefan Hajnoczi         return ret;
155761007b31SStefan Hajnoczi     }
155861007b31SStefan Hajnoczi 
155967a0fd2aSFam Zheng     *file = NULL;
156067a0fd2aSFam Zheng     ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum,
156167a0fd2aSFam Zheng                                             file);
156261007b31SStefan Hajnoczi     if (ret < 0) {
156361007b31SStefan Hajnoczi         *pnum = 0;
156461007b31SStefan Hajnoczi         return ret;
156561007b31SStefan Hajnoczi     }
156661007b31SStefan Hajnoczi 
156761007b31SStefan Hajnoczi     if (ret & BDRV_BLOCK_RAW) {
156861007b31SStefan Hajnoczi         assert(ret & BDRV_BLOCK_OFFSET_VALID);
15699a4f4c31SKevin Wolf         return bdrv_get_block_status(bs->file->bs, ret >> BDRV_SECTOR_BITS,
157067a0fd2aSFam Zheng                                      *pnum, pnum, file);
157161007b31SStefan Hajnoczi     }
157261007b31SStefan Hajnoczi 
157361007b31SStefan Hajnoczi     if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
157461007b31SStefan Hajnoczi         ret |= BDRV_BLOCK_ALLOCATED;
1575a53f1a95SPaolo Bonzini     } else {
157661007b31SStefan Hajnoczi         if (bdrv_unallocated_blocks_are_zero(bs)) {
157761007b31SStefan Hajnoczi             ret |= BDRV_BLOCK_ZERO;
1578760e0063SKevin Wolf         } else if (bs->backing) {
1579760e0063SKevin Wolf             BlockDriverState *bs2 = bs->backing->bs;
158061007b31SStefan Hajnoczi             int64_t nb_sectors2 = bdrv_nb_sectors(bs2);
158161007b31SStefan Hajnoczi             if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) {
158261007b31SStefan Hajnoczi                 ret |= BDRV_BLOCK_ZERO;
158361007b31SStefan Hajnoczi             }
158461007b31SStefan Hajnoczi         }
158561007b31SStefan Hajnoczi     }
158661007b31SStefan Hajnoczi 
1587ac987b30SFam Zheng     if (*file && *file != bs &&
158861007b31SStefan Hajnoczi         (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
158961007b31SStefan Hajnoczi         (ret & BDRV_BLOCK_OFFSET_VALID)) {
159067a0fd2aSFam Zheng         BlockDriverState *file2;
159161007b31SStefan Hajnoczi         int file_pnum;
159261007b31SStefan Hajnoczi 
1593ac987b30SFam Zheng         ret2 = bdrv_co_get_block_status(*file, ret >> BDRV_SECTOR_BITS,
159467a0fd2aSFam Zheng                                         *pnum, &file_pnum, &file2);
159561007b31SStefan Hajnoczi         if (ret2 >= 0) {
159661007b31SStefan Hajnoczi             /* Ignore errors.  This is just providing extra information, it
159761007b31SStefan Hajnoczi              * is useful but not necessary.
159861007b31SStefan Hajnoczi              */
159961007b31SStefan Hajnoczi             if (!file_pnum) {
160061007b31SStefan Hajnoczi                 /* !file_pnum indicates an offset at or beyond the EOF; it is
160161007b31SStefan Hajnoczi                  * perfectly valid for the format block driver to point to such
160261007b31SStefan Hajnoczi                  * offsets, so catch it and mark everything as zero */
160361007b31SStefan Hajnoczi                 ret |= BDRV_BLOCK_ZERO;
160461007b31SStefan Hajnoczi             } else {
160561007b31SStefan Hajnoczi                 /* Limit request to the range reported by the protocol driver */
160661007b31SStefan Hajnoczi                 *pnum = file_pnum;
160761007b31SStefan Hajnoczi                 ret |= (ret2 & BDRV_BLOCK_ZERO);
160861007b31SStefan Hajnoczi             }
160961007b31SStefan Hajnoczi         }
161061007b31SStefan Hajnoczi     }
161161007b31SStefan Hajnoczi 
161261007b31SStefan Hajnoczi     return ret;
161361007b31SStefan Hajnoczi }
161461007b31SStefan Hajnoczi 
1615ba3f0e25SFam Zheng static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs,
1616ba3f0e25SFam Zheng         BlockDriverState *base,
1617ba3f0e25SFam Zheng         int64_t sector_num,
1618ba3f0e25SFam Zheng         int nb_sectors,
161967a0fd2aSFam Zheng         int *pnum,
162067a0fd2aSFam Zheng         BlockDriverState **file)
1621ba3f0e25SFam Zheng {
1622ba3f0e25SFam Zheng     BlockDriverState *p;
1623ba3f0e25SFam Zheng     int64_t ret = 0;
1624ba3f0e25SFam Zheng 
1625ba3f0e25SFam Zheng     assert(bs != base);
1626760e0063SKevin Wolf     for (p = bs; p != base; p = backing_bs(p)) {
162767a0fd2aSFam Zheng         ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum, file);
1628ba3f0e25SFam Zheng         if (ret < 0 || ret & BDRV_BLOCK_ALLOCATED) {
1629ba3f0e25SFam Zheng             break;
1630ba3f0e25SFam Zheng         }
1631ba3f0e25SFam Zheng         /* [sector_num, pnum] unallocated on this layer, which could be only
1632ba3f0e25SFam Zheng          * the first part of [sector_num, nb_sectors].  */
1633ba3f0e25SFam Zheng         nb_sectors = MIN(nb_sectors, *pnum);
1634ba3f0e25SFam Zheng     }
1635ba3f0e25SFam Zheng     return ret;
1636ba3f0e25SFam Zheng }
1637ba3f0e25SFam Zheng 
1638ba3f0e25SFam Zheng /* Coroutine wrapper for bdrv_get_block_status_above() */
1639ba3f0e25SFam Zheng static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque)
164061007b31SStefan Hajnoczi {
164161007b31SStefan Hajnoczi     BdrvCoGetBlockStatusData *data = opaque;
164261007b31SStefan Hajnoczi 
1643ba3f0e25SFam Zheng     data->ret = bdrv_co_get_block_status_above(data->bs, data->base,
1644ba3f0e25SFam Zheng                                                data->sector_num,
1645ba3f0e25SFam Zheng                                                data->nb_sectors,
164667a0fd2aSFam Zheng                                                data->pnum,
164767a0fd2aSFam Zheng                                                data->file);
164861007b31SStefan Hajnoczi     data->done = true;
164961007b31SStefan Hajnoczi }
165061007b31SStefan Hajnoczi 
165161007b31SStefan Hajnoczi /*
1652ba3f0e25SFam Zheng  * Synchronous wrapper around bdrv_co_get_block_status_above().
165361007b31SStefan Hajnoczi  *
1654ba3f0e25SFam Zheng  * See bdrv_co_get_block_status_above() for details.
165561007b31SStefan Hajnoczi  */
1656ba3f0e25SFam Zheng int64_t bdrv_get_block_status_above(BlockDriverState *bs,
1657ba3f0e25SFam Zheng                                     BlockDriverState *base,
1658ba3f0e25SFam Zheng                                     int64_t sector_num,
165967a0fd2aSFam Zheng                                     int nb_sectors, int *pnum,
166067a0fd2aSFam Zheng                                     BlockDriverState **file)
166161007b31SStefan Hajnoczi {
166261007b31SStefan Hajnoczi     Coroutine *co;
166361007b31SStefan Hajnoczi     BdrvCoGetBlockStatusData data = {
166461007b31SStefan Hajnoczi         .bs = bs,
1665ba3f0e25SFam Zheng         .base = base,
166667a0fd2aSFam Zheng         .file = file,
166761007b31SStefan Hajnoczi         .sector_num = sector_num,
166861007b31SStefan Hajnoczi         .nb_sectors = nb_sectors,
166961007b31SStefan Hajnoczi         .pnum = pnum,
167061007b31SStefan Hajnoczi         .done = false,
167161007b31SStefan Hajnoczi     };
167261007b31SStefan Hajnoczi 
167361007b31SStefan Hajnoczi     if (qemu_in_coroutine()) {
167461007b31SStefan Hajnoczi         /* Fast-path if already in coroutine context */
1675ba3f0e25SFam Zheng         bdrv_get_block_status_above_co_entry(&data);
167661007b31SStefan Hajnoczi     } else {
167761007b31SStefan Hajnoczi         AioContext *aio_context = bdrv_get_aio_context(bs);
167861007b31SStefan Hajnoczi 
1679ba3f0e25SFam Zheng         co = qemu_coroutine_create(bdrv_get_block_status_above_co_entry);
168061007b31SStefan Hajnoczi         qemu_coroutine_enter(co, &data);
168161007b31SStefan Hajnoczi         while (!data.done) {
168261007b31SStefan Hajnoczi             aio_poll(aio_context, true);
168361007b31SStefan Hajnoczi         }
168461007b31SStefan Hajnoczi     }
168561007b31SStefan Hajnoczi     return data.ret;
168661007b31SStefan Hajnoczi }
168761007b31SStefan Hajnoczi 
1688ba3f0e25SFam Zheng int64_t bdrv_get_block_status(BlockDriverState *bs,
1689ba3f0e25SFam Zheng                               int64_t sector_num,
169067a0fd2aSFam Zheng                               int nb_sectors, int *pnum,
169167a0fd2aSFam Zheng                               BlockDriverState **file)
1692ba3f0e25SFam Zheng {
1693760e0063SKevin Wolf     return bdrv_get_block_status_above(bs, backing_bs(bs),
169467a0fd2aSFam Zheng                                        sector_num, nb_sectors, pnum, file);
1695ba3f0e25SFam Zheng }
1696ba3f0e25SFam Zheng 
169761007b31SStefan Hajnoczi int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num,
169861007b31SStefan Hajnoczi                                    int nb_sectors, int *pnum)
169961007b31SStefan Hajnoczi {
170067a0fd2aSFam Zheng     BlockDriverState *file;
170167a0fd2aSFam Zheng     int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum,
170267a0fd2aSFam Zheng                                         &file);
170361007b31SStefan Hajnoczi     if (ret < 0) {
170461007b31SStefan Hajnoczi         return ret;
170561007b31SStefan Hajnoczi     }
170661007b31SStefan Hajnoczi     return !!(ret & BDRV_BLOCK_ALLOCATED);
170761007b31SStefan Hajnoczi }
170861007b31SStefan Hajnoczi 
170961007b31SStefan Hajnoczi /*
171061007b31SStefan Hajnoczi  * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
171161007b31SStefan Hajnoczi  *
171261007b31SStefan Hajnoczi  * Return true if the given sector is allocated in any image between
171361007b31SStefan Hajnoczi  * BASE and TOP (inclusive).  BASE can be NULL to check if the given
171461007b31SStefan Hajnoczi  * sector is allocated in any image of the chain.  Return false otherwise.
171561007b31SStefan Hajnoczi  *
171661007b31SStefan Hajnoczi  * 'pnum' is set to the number of sectors (including and immediately following
171761007b31SStefan Hajnoczi  *  the specified sector) that are known to be in the same
171861007b31SStefan Hajnoczi  *  allocated/unallocated state.
171961007b31SStefan Hajnoczi  *
172061007b31SStefan Hajnoczi  */
172161007b31SStefan Hajnoczi int bdrv_is_allocated_above(BlockDriverState *top,
172261007b31SStefan Hajnoczi                             BlockDriverState *base,
172361007b31SStefan Hajnoczi                             int64_t sector_num,
172461007b31SStefan Hajnoczi                             int nb_sectors, int *pnum)
172561007b31SStefan Hajnoczi {
172661007b31SStefan Hajnoczi     BlockDriverState *intermediate;
172761007b31SStefan Hajnoczi     int ret, n = nb_sectors;
172861007b31SStefan Hajnoczi 
172961007b31SStefan Hajnoczi     intermediate = top;
173061007b31SStefan Hajnoczi     while (intermediate && intermediate != base) {
173161007b31SStefan Hajnoczi         int pnum_inter;
173261007b31SStefan Hajnoczi         ret = bdrv_is_allocated(intermediate, sector_num, nb_sectors,
173361007b31SStefan Hajnoczi                                 &pnum_inter);
173461007b31SStefan Hajnoczi         if (ret < 0) {
173561007b31SStefan Hajnoczi             return ret;
173661007b31SStefan Hajnoczi         } else if (ret) {
173761007b31SStefan Hajnoczi             *pnum = pnum_inter;
173861007b31SStefan Hajnoczi             return 1;
173961007b31SStefan Hajnoczi         }
174061007b31SStefan Hajnoczi 
174161007b31SStefan Hajnoczi         /*
174261007b31SStefan Hajnoczi          * [sector_num, nb_sectors] is unallocated on top but intermediate
174361007b31SStefan Hajnoczi          * might have
174461007b31SStefan Hajnoczi          *
174561007b31SStefan Hajnoczi          * [sector_num+x, nr_sectors] allocated.
174661007b31SStefan Hajnoczi          */
174761007b31SStefan Hajnoczi         if (n > pnum_inter &&
174861007b31SStefan Hajnoczi             (intermediate == top ||
174961007b31SStefan Hajnoczi              sector_num + pnum_inter < intermediate->total_sectors)) {
175061007b31SStefan Hajnoczi             n = pnum_inter;
175161007b31SStefan Hajnoczi         }
175261007b31SStefan Hajnoczi 
1753760e0063SKevin Wolf         intermediate = backing_bs(intermediate);
175461007b31SStefan Hajnoczi     }
175561007b31SStefan Hajnoczi 
175661007b31SStefan Hajnoczi     *pnum = n;
175761007b31SStefan Hajnoczi     return 0;
175861007b31SStefan Hajnoczi }
175961007b31SStefan Hajnoczi 
176061007b31SStefan Hajnoczi int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num,
176161007b31SStefan Hajnoczi                           const uint8_t *buf, int nb_sectors)
176261007b31SStefan Hajnoczi {
176361007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
176461007b31SStefan Hajnoczi     int ret;
176561007b31SStefan Hajnoczi 
176661007b31SStefan Hajnoczi     if (!drv) {
176761007b31SStefan Hajnoczi         return -ENOMEDIUM;
176861007b31SStefan Hajnoczi     }
176961007b31SStefan Hajnoczi     if (!drv->bdrv_write_compressed) {
177061007b31SStefan Hajnoczi         return -ENOTSUP;
177161007b31SStefan Hajnoczi     }
177261007b31SStefan Hajnoczi     ret = bdrv_check_request(bs, sector_num, nb_sectors);
177361007b31SStefan Hajnoczi     if (ret < 0) {
177461007b31SStefan Hajnoczi         return ret;
177561007b31SStefan Hajnoczi     }
177661007b31SStefan Hajnoczi 
177761007b31SStefan Hajnoczi     assert(QLIST_EMPTY(&bs->dirty_bitmaps));
177861007b31SStefan Hajnoczi 
177961007b31SStefan Hajnoczi     return drv->bdrv_write_compressed(bs, sector_num, buf, nb_sectors);
178061007b31SStefan Hajnoczi }
178161007b31SStefan Hajnoczi 
178261007b31SStefan Hajnoczi int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
178361007b31SStefan Hajnoczi                       int64_t pos, int size)
178461007b31SStefan Hajnoczi {
178561007b31SStefan Hajnoczi     QEMUIOVector qiov;
178661007b31SStefan Hajnoczi     struct iovec iov = {
178761007b31SStefan Hajnoczi         .iov_base   = (void *) buf,
178861007b31SStefan Hajnoczi         .iov_len    = size,
178961007b31SStefan Hajnoczi     };
179061007b31SStefan Hajnoczi 
179161007b31SStefan Hajnoczi     qemu_iovec_init_external(&qiov, &iov, 1);
179261007b31SStefan Hajnoczi     return bdrv_writev_vmstate(bs, &qiov, pos);
179361007b31SStefan Hajnoczi }
179461007b31SStefan Hajnoczi 
179561007b31SStefan Hajnoczi int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
179661007b31SStefan Hajnoczi {
179761007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
179861007b31SStefan Hajnoczi 
179961007b31SStefan Hajnoczi     if (!drv) {
180061007b31SStefan Hajnoczi         return -ENOMEDIUM;
180161007b31SStefan Hajnoczi     } else if (drv->bdrv_save_vmstate) {
180261007b31SStefan Hajnoczi         return drv->bdrv_save_vmstate(bs, qiov, pos);
180361007b31SStefan Hajnoczi     } else if (bs->file) {
18049a4f4c31SKevin Wolf         return bdrv_writev_vmstate(bs->file->bs, qiov, pos);
180561007b31SStefan Hajnoczi     }
180661007b31SStefan Hajnoczi 
180761007b31SStefan Hajnoczi     return -ENOTSUP;
180861007b31SStefan Hajnoczi }
180961007b31SStefan Hajnoczi 
181061007b31SStefan Hajnoczi int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
181161007b31SStefan Hajnoczi                       int64_t pos, int size)
181261007b31SStefan Hajnoczi {
181361007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
181461007b31SStefan Hajnoczi     if (!drv)
181561007b31SStefan Hajnoczi         return -ENOMEDIUM;
181661007b31SStefan Hajnoczi     if (drv->bdrv_load_vmstate)
181761007b31SStefan Hajnoczi         return drv->bdrv_load_vmstate(bs, buf, pos, size);
181861007b31SStefan Hajnoczi     if (bs->file)
18199a4f4c31SKevin Wolf         return bdrv_load_vmstate(bs->file->bs, buf, pos, size);
182061007b31SStefan Hajnoczi     return -ENOTSUP;
182161007b31SStefan Hajnoczi }
182261007b31SStefan Hajnoczi 
182361007b31SStefan Hajnoczi /**************************************************************/
182461007b31SStefan Hajnoczi /* async I/Os */
182561007b31SStefan Hajnoczi 
182661007b31SStefan Hajnoczi BlockAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num,
182761007b31SStefan Hajnoczi                            QEMUIOVector *qiov, int nb_sectors,
182861007b31SStefan Hajnoczi                            BlockCompletionFunc *cb, void *opaque)
182961007b31SStefan Hajnoczi {
183061007b31SStefan Hajnoczi     trace_bdrv_aio_readv(bs, sector_num, nb_sectors, opaque);
183161007b31SStefan Hajnoczi 
183261007b31SStefan Hajnoczi     return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0,
183361007b31SStefan Hajnoczi                                  cb, opaque, false);
183461007b31SStefan Hajnoczi }
183561007b31SStefan Hajnoczi 
183661007b31SStefan Hajnoczi BlockAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num,
183761007b31SStefan Hajnoczi                             QEMUIOVector *qiov, int nb_sectors,
183861007b31SStefan Hajnoczi                             BlockCompletionFunc *cb, void *opaque)
183961007b31SStefan Hajnoczi {
184061007b31SStefan Hajnoczi     trace_bdrv_aio_writev(bs, sector_num, nb_sectors, opaque);
184161007b31SStefan Hajnoczi 
184261007b31SStefan Hajnoczi     return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0,
184361007b31SStefan Hajnoczi                                  cb, opaque, true);
184461007b31SStefan Hajnoczi }
184561007b31SStefan Hajnoczi 
184661007b31SStefan Hajnoczi BlockAIOCB *bdrv_aio_write_zeroes(BlockDriverState *bs,
184761007b31SStefan Hajnoczi         int64_t sector_num, int nb_sectors, BdrvRequestFlags flags,
184861007b31SStefan Hajnoczi         BlockCompletionFunc *cb, void *opaque)
184961007b31SStefan Hajnoczi {
185061007b31SStefan Hajnoczi     trace_bdrv_aio_write_zeroes(bs, sector_num, nb_sectors, flags, opaque);
185161007b31SStefan Hajnoczi 
185261007b31SStefan Hajnoczi     return bdrv_co_aio_rw_vector(bs, sector_num, NULL, nb_sectors,
185361007b31SStefan Hajnoczi                                  BDRV_REQ_ZERO_WRITE | flags,
185461007b31SStefan Hajnoczi                                  cb, opaque, true);
185561007b31SStefan Hajnoczi }
185661007b31SStefan Hajnoczi 
185761007b31SStefan Hajnoczi 
185861007b31SStefan Hajnoczi typedef struct MultiwriteCB {
185961007b31SStefan Hajnoczi     int error;
186061007b31SStefan Hajnoczi     int num_requests;
186161007b31SStefan Hajnoczi     int num_callbacks;
186261007b31SStefan Hajnoczi     struct {
186361007b31SStefan Hajnoczi         BlockCompletionFunc *cb;
186461007b31SStefan Hajnoczi         void *opaque;
186561007b31SStefan Hajnoczi         QEMUIOVector *free_qiov;
186661007b31SStefan Hajnoczi     } callbacks[];
186761007b31SStefan Hajnoczi } MultiwriteCB;
186861007b31SStefan Hajnoczi 
186961007b31SStefan Hajnoczi static void multiwrite_user_cb(MultiwriteCB *mcb)
187061007b31SStefan Hajnoczi {
187161007b31SStefan Hajnoczi     int i;
187261007b31SStefan Hajnoczi 
187361007b31SStefan Hajnoczi     for (i = 0; i < mcb->num_callbacks; i++) {
187461007b31SStefan Hajnoczi         mcb->callbacks[i].cb(mcb->callbacks[i].opaque, mcb->error);
187561007b31SStefan Hajnoczi         if (mcb->callbacks[i].free_qiov) {
187661007b31SStefan Hajnoczi             qemu_iovec_destroy(mcb->callbacks[i].free_qiov);
187761007b31SStefan Hajnoczi         }
187861007b31SStefan Hajnoczi         g_free(mcb->callbacks[i].free_qiov);
187961007b31SStefan Hajnoczi     }
188061007b31SStefan Hajnoczi }
188161007b31SStefan Hajnoczi 
188261007b31SStefan Hajnoczi static void multiwrite_cb(void *opaque, int ret)
188361007b31SStefan Hajnoczi {
188461007b31SStefan Hajnoczi     MultiwriteCB *mcb = opaque;
188561007b31SStefan Hajnoczi 
188661007b31SStefan Hajnoczi     trace_multiwrite_cb(mcb, ret);
188761007b31SStefan Hajnoczi 
188861007b31SStefan Hajnoczi     if (ret < 0 && !mcb->error) {
188961007b31SStefan Hajnoczi         mcb->error = ret;
189061007b31SStefan Hajnoczi     }
189161007b31SStefan Hajnoczi 
189261007b31SStefan Hajnoczi     mcb->num_requests--;
189361007b31SStefan Hajnoczi     if (mcb->num_requests == 0) {
189461007b31SStefan Hajnoczi         multiwrite_user_cb(mcb);
189561007b31SStefan Hajnoczi         g_free(mcb);
189661007b31SStefan Hajnoczi     }
189761007b31SStefan Hajnoczi }
189861007b31SStefan Hajnoczi 
189961007b31SStefan Hajnoczi static int multiwrite_req_compare(const void *a, const void *b)
190061007b31SStefan Hajnoczi {
190161007b31SStefan Hajnoczi     const BlockRequest *req1 = a, *req2 = b;
190261007b31SStefan Hajnoczi 
190361007b31SStefan Hajnoczi     /*
190461007b31SStefan Hajnoczi      * Note that we can't simply subtract req2->sector from req1->sector
190561007b31SStefan Hajnoczi      * here as that could overflow the return value.
190661007b31SStefan Hajnoczi      */
190761007b31SStefan Hajnoczi     if (req1->sector > req2->sector) {
190861007b31SStefan Hajnoczi         return 1;
190961007b31SStefan Hajnoczi     } else if (req1->sector < req2->sector) {
191061007b31SStefan Hajnoczi         return -1;
191161007b31SStefan Hajnoczi     } else {
191261007b31SStefan Hajnoczi         return 0;
191361007b31SStefan Hajnoczi     }
191461007b31SStefan Hajnoczi }
191561007b31SStefan Hajnoczi 
191661007b31SStefan Hajnoczi /*
191761007b31SStefan Hajnoczi  * Takes a bunch of requests and tries to merge them. Returns the number of
191861007b31SStefan Hajnoczi  * requests that remain after merging.
191961007b31SStefan Hajnoczi  */
192061007b31SStefan Hajnoczi static int multiwrite_merge(BlockDriverState *bs, BlockRequest *reqs,
192161007b31SStefan Hajnoczi     int num_reqs, MultiwriteCB *mcb)
192261007b31SStefan Hajnoczi {
192361007b31SStefan Hajnoczi     int i, outidx;
192461007b31SStefan Hajnoczi 
192561007b31SStefan Hajnoczi     // Sort requests by start sector
192661007b31SStefan Hajnoczi     qsort(reqs, num_reqs, sizeof(*reqs), &multiwrite_req_compare);
192761007b31SStefan Hajnoczi 
192861007b31SStefan Hajnoczi     // Check if adjacent requests touch the same clusters. If so, combine them,
192961007b31SStefan Hajnoczi     // filling up gaps with zero sectors.
193061007b31SStefan Hajnoczi     outidx = 0;
193161007b31SStefan Hajnoczi     for (i = 1; i < num_reqs; i++) {
193261007b31SStefan Hajnoczi         int merge = 0;
193361007b31SStefan Hajnoczi         int64_t oldreq_last = reqs[outidx].sector + reqs[outidx].nb_sectors;
193461007b31SStefan Hajnoczi 
193561007b31SStefan Hajnoczi         // Handle exactly sequential writes and overlapping writes.
193661007b31SStefan Hajnoczi         if (reqs[i].sector <= oldreq_last) {
193761007b31SStefan Hajnoczi             merge = 1;
193861007b31SStefan Hajnoczi         }
193961007b31SStefan Hajnoczi 
1940222565f6SStefan Hajnoczi         if (reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1 >
1941222565f6SStefan Hajnoczi             bs->bl.max_iov) {
194261007b31SStefan Hajnoczi             merge = 0;
194361007b31SStefan Hajnoczi         }
194461007b31SStefan Hajnoczi 
194561007b31SStefan Hajnoczi         if (bs->bl.max_transfer_length && reqs[outidx].nb_sectors +
194661007b31SStefan Hajnoczi             reqs[i].nb_sectors > bs->bl.max_transfer_length) {
194761007b31SStefan Hajnoczi             merge = 0;
194861007b31SStefan Hajnoczi         }
194961007b31SStefan Hajnoczi 
195061007b31SStefan Hajnoczi         if (merge) {
195161007b31SStefan Hajnoczi             size_t size;
195261007b31SStefan Hajnoczi             QEMUIOVector *qiov = g_malloc0(sizeof(*qiov));
195361007b31SStefan Hajnoczi             qemu_iovec_init(qiov,
195461007b31SStefan Hajnoczi                 reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1);
195561007b31SStefan Hajnoczi 
195661007b31SStefan Hajnoczi             // Add the first request to the merged one. If the requests are
195761007b31SStefan Hajnoczi             // overlapping, drop the last sectors of the first request.
195861007b31SStefan Hajnoczi             size = (reqs[i].sector - reqs[outidx].sector) << 9;
195961007b31SStefan Hajnoczi             qemu_iovec_concat(qiov, reqs[outidx].qiov, 0, size);
196061007b31SStefan Hajnoczi 
196161007b31SStefan Hajnoczi             // We should need to add any zeros between the two requests
196261007b31SStefan Hajnoczi             assert (reqs[i].sector <= oldreq_last);
196361007b31SStefan Hajnoczi 
196461007b31SStefan Hajnoczi             // Add the second request
196561007b31SStefan Hajnoczi             qemu_iovec_concat(qiov, reqs[i].qiov, 0, reqs[i].qiov->size);
196661007b31SStefan Hajnoczi 
196761007b31SStefan Hajnoczi             // Add tail of first request, if necessary
196861007b31SStefan Hajnoczi             if (qiov->size < reqs[outidx].qiov->size) {
196961007b31SStefan Hajnoczi                 qemu_iovec_concat(qiov, reqs[outidx].qiov, qiov->size,
197061007b31SStefan Hajnoczi                                   reqs[outidx].qiov->size - qiov->size);
197161007b31SStefan Hajnoczi             }
197261007b31SStefan Hajnoczi 
197361007b31SStefan Hajnoczi             reqs[outidx].nb_sectors = qiov->size >> 9;
197461007b31SStefan Hajnoczi             reqs[outidx].qiov = qiov;
197561007b31SStefan Hajnoczi 
197661007b31SStefan Hajnoczi             mcb->callbacks[i].free_qiov = reqs[outidx].qiov;
197761007b31SStefan Hajnoczi         } else {
197861007b31SStefan Hajnoczi             outidx++;
197961007b31SStefan Hajnoczi             reqs[outidx].sector     = reqs[i].sector;
198061007b31SStefan Hajnoczi             reqs[outidx].nb_sectors = reqs[i].nb_sectors;
198161007b31SStefan Hajnoczi             reqs[outidx].qiov       = reqs[i].qiov;
198261007b31SStefan Hajnoczi         }
198361007b31SStefan Hajnoczi     }
198461007b31SStefan Hajnoczi 
19857f0e9da6SMax Reitz     if (bs->blk) {
19867f0e9da6SMax Reitz         block_acct_merge_done(blk_get_stats(bs->blk), BLOCK_ACCT_WRITE,
19877f0e9da6SMax Reitz                               num_reqs - outidx - 1);
19887f0e9da6SMax Reitz     }
198961007b31SStefan Hajnoczi 
199061007b31SStefan Hajnoczi     return outidx + 1;
199161007b31SStefan Hajnoczi }
199261007b31SStefan Hajnoczi 
199361007b31SStefan Hajnoczi /*
199461007b31SStefan Hajnoczi  * Submit multiple AIO write requests at once.
199561007b31SStefan Hajnoczi  *
199661007b31SStefan Hajnoczi  * On success, the function returns 0 and all requests in the reqs array have
199761007b31SStefan Hajnoczi  * been submitted. In error case this function returns -1, and any of the
199861007b31SStefan Hajnoczi  * requests may or may not be submitted yet. In particular, this means that the
199961007b31SStefan Hajnoczi  * callback will be called for some of the requests, for others it won't. The
200061007b31SStefan Hajnoczi  * caller must check the error field of the BlockRequest to wait for the right
200161007b31SStefan Hajnoczi  * callbacks (if error != 0, no callback will be called).
200261007b31SStefan Hajnoczi  *
200361007b31SStefan Hajnoczi  * The implementation may modify the contents of the reqs array, e.g. to merge
200461007b31SStefan Hajnoczi  * requests. However, the fields opaque and error are left unmodified as they
200561007b31SStefan Hajnoczi  * are used to signal failure for a single request to the caller.
200661007b31SStefan Hajnoczi  */
200761007b31SStefan Hajnoczi int bdrv_aio_multiwrite(BlockDriverState *bs, BlockRequest *reqs, int num_reqs)
200861007b31SStefan Hajnoczi {
200961007b31SStefan Hajnoczi     MultiwriteCB *mcb;
201061007b31SStefan Hajnoczi     int i;
201161007b31SStefan Hajnoczi 
201261007b31SStefan Hajnoczi     /* don't submit writes if we don't have a medium */
201361007b31SStefan Hajnoczi     if (bs->drv == NULL) {
201461007b31SStefan Hajnoczi         for (i = 0; i < num_reqs; i++) {
201561007b31SStefan Hajnoczi             reqs[i].error = -ENOMEDIUM;
201661007b31SStefan Hajnoczi         }
201761007b31SStefan Hajnoczi         return -1;
201861007b31SStefan Hajnoczi     }
201961007b31SStefan Hajnoczi 
202061007b31SStefan Hajnoczi     if (num_reqs == 0) {
202161007b31SStefan Hajnoczi         return 0;
202261007b31SStefan Hajnoczi     }
202361007b31SStefan Hajnoczi 
202461007b31SStefan Hajnoczi     // Create MultiwriteCB structure
202561007b31SStefan Hajnoczi     mcb = g_malloc0(sizeof(*mcb) + num_reqs * sizeof(*mcb->callbacks));
202661007b31SStefan Hajnoczi     mcb->num_requests = 0;
202761007b31SStefan Hajnoczi     mcb->num_callbacks = num_reqs;
202861007b31SStefan Hajnoczi 
202961007b31SStefan Hajnoczi     for (i = 0; i < num_reqs; i++) {
203061007b31SStefan Hajnoczi         mcb->callbacks[i].cb = reqs[i].cb;
203161007b31SStefan Hajnoczi         mcb->callbacks[i].opaque = reqs[i].opaque;
203261007b31SStefan Hajnoczi     }
203361007b31SStefan Hajnoczi 
203461007b31SStefan Hajnoczi     // Check for mergable requests
203561007b31SStefan Hajnoczi     num_reqs = multiwrite_merge(bs, reqs, num_reqs, mcb);
203661007b31SStefan Hajnoczi 
203761007b31SStefan Hajnoczi     trace_bdrv_aio_multiwrite(mcb, mcb->num_callbacks, num_reqs);
203861007b31SStefan Hajnoczi 
203961007b31SStefan Hajnoczi     /* Run the aio requests. */
204061007b31SStefan Hajnoczi     mcb->num_requests = num_reqs;
204161007b31SStefan Hajnoczi     for (i = 0; i < num_reqs; i++) {
204261007b31SStefan Hajnoczi         bdrv_co_aio_rw_vector(bs, reqs[i].sector, reqs[i].qiov,
204361007b31SStefan Hajnoczi                               reqs[i].nb_sectors, reqs[i].flags,
204461007b31SStefan Hajnoczi                               multiwrite_cb, mcb,
204561007b31SStefan Hajnoczi                               true);
204661007b31SStefan Hajnoczi     }
204761007b31SStefan Hajnoczi 
204861007b31SStefan Hajnoczi     return 0;
204961007b31SStefan Hajnoczi }
205061007b31SStefan Hajnoczi 
205161007b31SStefan Hajnoczi void bdrv_aio_cancel(BlockAIOCB *acb)
205261007b31SStefan Hajnoczi {
205361007b31SStefan Hajnoczi     qemu_aio_ref(acb);
205461007b31SStefan Hajnoczi     bdrv_aio_cancel_async(acb);
205561007b31SStefan Hajnoczi     while (acb->refcnt > 1) {
205661007b31SStefan Hajnoczi         if (acb->aiocb_info->get_aio_context) {
205761007b31SStefan Hajnoczi             aio_poll(acb->aiocb_info->get_aio_context(acb), true);
205861007b31SStefan Hajnoczi         } else if (acb->bs) {
205961007b31SStefan Hajnoczi             aio_poll(bdrv_get_aio_context(acb->bs), true);
206061007b31SStefan Hajnoczi         } else {
206161007b31SStefan Hajnoczi             abort();
206261007b31SStefan Hajnoczi         }
206361007b31SStefan Hajnoczi     }
206461007b31SStefan Hajnoczi     qemu_aio_unref(acb);
206561007b31SStefan Hajnoczi }
206661007b31SStefan Hajnoczi 
206761007b31SStefan Hajnoczi /* Async version of aio cancel. The caller is not blocked if the acb implements
206861007b31SStefan Hajnoczi  * cancel_async, otherwise we do nothing and let the request normally complete.
206961007b31SStefan Hajnoczi  * In either case the completion callback must be called. */
207061007b31SStefan Hajnoczi void bdrv_aio_cancel_async(BlockAIOCB *acb)
207161007b31SStefan Hajnoczi {
207261007b31SStefan Hajnoczi     if (acb->aiocb_info->cancel_async) {
207361007b31SStefan Hajnoczi         acb->aiocb_info->cancel_async(acb);
207461007b31SStefan Hajnoczi     }
207561007b31SStefan Hajnoczi }
207661007b31SStefan Hajnoczi 
207761007b31SStefan Hajnoczi /**************************************************************/
207861007b31SStefan Hajnoczi /* async block device emulation */
207961007b31SStefan Hajnoczi 
208061007b31SStefan Hajnoczi typedef struct BlockAIOCBSync {
208161007b31SStefan Hajnoczi     BlockAIOCB common;
208261007b31SStefan Hajnoczi     QEMUBH *bh;
208361007b31SStefan Hajnoczi     int ret;
208461007b31SStefan Hajnoczi     /* vector translation state */
208561007b31SStefan Hajnoczi     QEMUIOVector *qiov;
208661007b31SStefan Hajnoczi     uint8_t *bounce;
208761007b31SStefan Hajnoczi     int is_write;
208861007b31SStefan Hajnoczi } BlockAIOCBSync;
208961007b31SStefan Hajnoczi 
209061007b31SStefan Hajnoczi static const AIOCBInfo bdrv_em_aiocb_info = {
209161007b31SStefan Hajnoczi     .aiocb_size         = sizeof(BlockAIOCBSync),
209261007b31SStefan Hajnoczi };
209361007b31SStefan Hajnoczi 
209461007b31SStefan Hajnoczi static void bdrv_aio_bh_cb(void *opaque)
209561007b31SStefan Hajnoczi {
209661007b31SStefan Hajnoczi     BlockAIOCBSync *acb = opaque;
209761007b31SStefan Hajnoczi 
209861007b31SStefan Hajnoczi     if (!acb->is_write && acb->ret >= 0) {
209961007b31SStefan Hajnoczi         qemu_iovec_from_buf(acb->qiov, 0, acb->bounce, acb->qiov->size);
210061007b31SStefan Hajnoczi     }
210161007b31SStefan Hajnoczi     qemu_vfree(acb->bounce);
210261007b31SStefan Hajnoczi     acb->common.cb(acb->common.opaque, acb->ret);
210361007b31SStefan Hajnoczi     qemu_bh_delete(acb->bh);
210461007b31SStefan Hajnoczi     acb->bh = NULL;
210561007b31SStefan Hajnoczi     qemu_aio_unref(acb);
210661007b31SStefan Hajnoczi }
210761007b31SStefan Hajnoczi 
210861007b31SStefan Hajnoczi static BlockAIOCB *bdrv_aio_rw_vector(BlockDriverState *bs,
210961007b31SStefan Hajnoczi                                       int64_t sector_num,
211061007b31SStefan Hajnoczi                                       QEMUIOVector *qiov,
211161007b31SStefan Hajnoczi                                       int nb_sectors,
211261007b31SStefan Hajnoczi                                       BlockCompletionFunc *cb,
211361007b31SStefan Hajnoczi                                       void *opaque,
211461007b31SStefan Hajnoczi                                       int is_write)
211561007b31SStefan Hajnoczi 
211661007b31SStefan Hajnoczi {
211761007b31SStefan Hajnoczi     BlockAIOCBSync *acb;
211861007b31SStefan Hajnoczi 
211961007b31SStefan Hajnoczi     acb = qemu_aio_get(&bdrv_em_aiocb_info, bs, cb, opaque);
212061007b31SStefan Hajnoczi     acb->is_write = is_write;
212161007b31SStefan Hajnoczi     acb->qiov = qiov;
212261007b31SStefan Hajnoczi     acb->bounce = qemu_try_blockalign(bs, qiov->size);
212361007b31SStefan Hajnoczi     acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_aio_bh_cb, acb);
212461007b31SStefan Hajnoczi 
212561007b31SStefan Hajnoczi     if (acb->bounce == NULL) {
212661007b31SStefan Hajnoczi         acb->ret = -ENOMEM;
212761007b31SStefan Hajnoczi     } else if (is_write) {
212861007b31SStefan Hajnoczi         qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size);
212961007b31SStefan Hajnoczi         acb->ret = bs->drv->bdrv_write(bs, sector_num, acb->bounce, nb_sectors);
213061007b31SStefan Hajnoczi     } else {
213161007b31SStefan Hajnoczi         acb->ret = bs->drv->bdrv_read(bs, sector_num, acb->bounce, nb_sectors);
213261007b31SStefan Hajnoczi     }
213361007b31SStefan Hajnoczi 
213461007b31SStefan Hajnoczi     qemu_bh_schedule(acb->bh);
213561007b31SStefan Hajnoczi 
213661007b31SStefan Hajnoczi     return &acb->common;
213761007b31SStefan Hajnoczi }
213861007b31SStefan Hajnoczi 
213961007b31SStefan Hajnoczi static BlockAIOCB *bdrv_aio_readv_em(BlockDriverState *bs,
214061007b31SStefan Hajnoczi         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
214161007b31SStefan Hajnoczi         BlockCompletionFunc *cb, void *opaque)
214261007b31SStefan Hajnoczi {
214361007b31SStefan Hajnoczi     return bdrv_aio_rw_vector(bs, sector_num, qiov, nb_sectors, cb, opaque, 0);
214461007b31SStefan Hajnoczi }
214561007b31SStefan Hajnoczi 
214661007b31SStefan Hajnoczi static BlockAIOCB *bdrv_aio_writev_em(BlockDriverState *bs,
214761007b31SStefan Hajnoczi         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
214861007b31SStefan Hajnoczi         BlockCompletionFunc *cb, void *opaque)
214961007b31SStefan Hajnoczi {
215061007b31SStefan Hajnoczi     return bdrv_aio_rw_vector(bs, sector_num, qiov, nb_sectors, cb, opaque, 1);
215161007b31SStefan Hajnoczi }
215261007b31SStefan Hajnoczi 
215361007b31SStefan Hajnoczi 
215461007b31SStefan Hajnoczi typedef struct BlockAIOCBCoroutine {
215561007b31SStefan Hajnoczi     BlockAIOCB common;
215661007b31SStefan Hajnoczi     BlockRequest req;
215761007b31SStefan Hajnoczi     bool is_write;
215861007b31SStefan Hajnoczi     bool need_bh;
215961007b31SStefan Hajnoczi     bool *done;
216061007b31SStefan Hajnoczi     QEMUBH* bh;
216161007b31SStefan Hajnoczi } BlockAIOCBCoroutine;
216261007b31SStefan Hajnoczi 
216361007b31SStefan Hajnoczi static const AIOCBInfo bdrv_em_co_aiocb_info = {
216461007b31SStefan Hajnoczi     .aiocb_size         = sizeof(BlockAIOCBCoroutine),
216561007b31SStefan Hajnoczi };
216661007b31SStefan Hajnoczi 
216761007b31SStefan Hajnoczi static void bdrv_co_complete(BlockAIOCBCoroutine *acb)
216861007b31SStefan Hajnoczi {
216961007b31SStefan Hajnoczi     if (!acb->need_bh) {
217061007b31SStefan Hajnoczi         acb->common.cb(acb->common.opaque, acb->req.error);
217161007b31SStefan Hajnoczi         qemu_aio_unref(acb);
217261007b31SStefan Hajnoczi     }
217361007b31SStefan Hajnoczi }
217461007b31SStefan Hajnoczi 
217561007b31SStefan Hajnoczi static void bdrv_co_em_bh(void *opaque)
217661007b31SStefan Hajnoczi {
217761007b31SStefan Hajnoczi     BlockAIOCBCoroutine *acb = opaque;
217861007b31SStefan Hajnoczi 
217961007b31SStefan Hajnoczi     assert(!acb->need_bh);
218061007b31SStefan Hajnoczi     qemu_bh_delete(acb->bh);
218161007b31SStefan Hajnoczi     bdrv_co_complete(acb);
218261007b31SStefan Hajnoczi }
218361007b31SStefan Hajnoczi 
218461007b31SStefan Hajnoczi static void bdrv_co_maybe_schedule_bh(BlockAIOCBCoroutine *acb)
218561007b31SStefan Hajnoczi {
218661007b31SStefan Hajnoczi     acb->need_bh = false;
218761007b31SStefan Hajnoczi     if (acb->req.error != -EINPROGRESS) {
218861007b31SStefan Hajnoczi         BlockDriverState *bs = acb->common.bs;
218961007b31SStefan Hajnoczi 
219061007b31SStefan Hajnoczi         acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb);
219161007b31SStefan Hajnoczi         qemu_bh_schedule(acb->bh);
219261007b31SStefan Hajnoczi     }
219361007b31SStefan Hajnoczi }
219461007b31SStefan Hajnoczi 
219561007b31SStefan Hajnoczi /* Invoke bdrv_co_do_readv/bdrv_co_do_writev */
219661007b31SStefan Hajnoczi static void coroutine_fn bdrv_co_do_rw(void *opaque)
219761007b31SStefan Hajnoczi {
219861007b31SStefan Hajnoczi     BlockAIOCBCoroutine *acb = opaque;
219961007b31SStefan Hajnoczi     BlockDriverState *bs = acb->common.bs;
220061007b31SStefan Hajnoczi 
220161007b31SStefan Hajnoczi     if (!acb->is_write) {
220261007b31SStefan Hajnoczi         acb->req.error = bdrv_co_do_readv(bs, acb->req.sector,
220361007b31SStefan Hajnoczi             acb->req.nb_sectors, acb->req.qiov, acb->req.flags);
220461007b31SStefan Hajnoczi     } else {
220561007b31SStefan Hajnoczi         acb->req.error = bdrv_co_do_writev(bs, acb->req.sector,
220661007b31SStefan Hajnoczi             acb->req.nb_sectors, acb->req.qiov, acb->req.flags);
220761007b31SStefan Hajnoczi     }
220861007b31SStefan Hajnoczi 
220961007b31SStefan Hajnoczi     bdrv_co_complete(acb);
221061007b31SStefan Hajnoczi }
221161007b31SStefan Hajnoczi 
221261007b31SStefan Hajnoczi static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
221361007b31SStefan Hajnoczi                                          int64_t sector_num,
221461007b31SStefan Hajnoczi                                          QEMUIOVector *qiov,
221561007b31SStefan Hajnoczi                                          int nb_sectors,
221661007b31SStefan Hajnoczi                                          BdrvRequestFlags flags,
221761007b31SStefan Hajnoczi                                          BlockCompletionFunc *cb,
221861007b31SStefan Hajnoczi                                          void *opaque,
221961007b31SStefan Hajnoczi                                          bool is_write)
222061007b31SStefan Hajnoczi {
222161007b31SStefan Hajnoczi     Coroutine *co;
222261007b31SStefan Hajnoczi     BlockAIOCBCoroutine *acb;
222361007b31SStefan Hajnoczi 
222461007b31SStefan Hajnoczi     acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
222561007b31SStefan Hajnoczi     acb->need_bh = true;
222661007b31SStefan Hajnoczi     acb->req.error = -EINPROGRESS;
222761007b31SStefan Hajnoczi     acb->req.sector = sector_num;
222861007b31SStefan Hajnoczi     acb->req.nb_sectors = nb_sectors;
222961007b31SStefan Hajnoczi     acb->req.qiov = qiov;
223061007b31SStefan Hajnoczi     acb->req.flags = flags;
223161007b31SStefan Hajnoczi     acb->is_write = is_write;
223261007b31SStefan Hajnoczi 
223361007b31SStefan Hajnoczi     co = qemu_coroutine_create(bdrv_co_do_rw);
223461007b31SStefan Hajnoczi     qemu_coroutine_enter(co, acb);
223561007b31SStefan Hajnoczi 
223661007b31SStefan Hajnoczi     bdrv_co_maybe_schedule_bh(acb);
223761007b31SStefan Hajnoczi     return &acb->common;
223861007b31SStefan Hajnoczi }
223961007b31SStefan Hajnoczi 
224061007b31SStefan Hajnoczi static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque)
224161007b31SStefan Hajnoczi {
224261007b31SStefan Hajnoczi     BlockAIOCBCoroutine *acb = opaque;
224361007b31SStefan Hajnoczi     BlockDriverState *bs = acb->common.bs;
224461007b31SStefan Hajnoczi 
224561007b31SStefan Hajnoczi     acb->req.error = bdrv_co_flush(bs);
224661007b31SStefan Hajnoczi     bdrv_co_complete(acb);
224761007b31SStefan Hajnoczi }
224861007b31SStefan Hajnoczi 
224961007b31SStefan Hajnoczi BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs,
225061007b31SStefan Hajnoczi         BlockCompletionFunc *cb, void *opaque)
225161007b31SStefan Hajnoczi {
225261007b31SStefan Hajnoczi     trace_bdrv_aio_flush(bs, opaque);
225361007b31SStefan Hajnoczi 
225461007b31SStefan Hajnoczi     Coroutine *co;
225561007b31SStefan Hajnoczi     BlockAIOCBCoroutine *acb;
225661007b31SStefan Hajnoczi 
225761007b31SStefan Hajnoczi     acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
225861007b31SStefan Hajnoczi     acb->need_bh = true;
225961007b31SStefan Hajnoczi     acb->req.error = -EINPROGRESS;
226061007b31SStefan Hajnoczi 
226161007b31SStefan Hajnoczi     co = qemu_coroutine_create(bdrv_aio_flush_co_entry);
226261007b31SStefan Hajnoczi     qemu_coroutine_enter(co, acb);
226361007b31SStefan Hajnoczi 
226461007b31SStefan Hajnoczi     bdrv_co_maybe_schedule_bh(acb);
226561007b31SStefan Hajnoczi     return &acb->common;
226661007b31SStefan Hajnoczi }
226761007b31SStefan Hajnoczi 
226861007b31SStefan Hajnoczi static void coroutine_fn bdrv_aio_discard_co_entry(void *opaque)
226961007b31SStefan Hajnoczi {
227061007b31SStefan Hajnoczi     BlockAIOCBCoroutine *acb = opaque;
227161007b31SStefan Hajnoczi     BlockDriverState *bs = acb->common.bs;
227261007b31SStefan Hajnoczi 
227361007b31SStefan Hajnoczi     acb->req.error = bdrv_co_discard(bs, acb->req.sector, acb->req.nb_sectors);
227461007b31SStefan Hajnoczi     bdrv_co_complete(acb);
227561007b31SStefan Hajnoczi }
227661007b31SStefan Hajnoczi 
227761007b31SStefan Hajnoczi BlockAIOCB *bdrv_aio_discard(BlockDriverState *bs,
227861007b31SStefan Hajnoczi         int64_t sector_num, int nb_sectors,
227961007b31SStefan Hajnoczi         BlockCompletionFunc *cb, void *opaque)
228061007b31SStefan Hajnoczi {
228161007b31SStefan Hajnoczi     Coroutine *co;
228261007b31SStefan Hajnoczi     BlockAIOCBCoroutine *acb;
228361007b31SStefan Hajnoczi 
228461007b31SStefan Hajnoczi     trace_bdrv_aio_discard(bs, sector_num, nb_sectors, opaque);
228561007b31SStefan Hajnoczi 
228661007b31SStefan Hajnoczi     acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
228761007b31SStefan Hajnoczi     acb->need_bh = true;
228861007b31SStefan Hajnoczi     acb->req.error = -EINPROGRESS;
228961007b31SStefan Hajnoczi     acb->req.sector = sector_num;
229061007b31SStefan Hajnoczi     acb->req.nb_sectors = nb_sectors;
229161007b31SStefan Hajnoczi     co = qemu_coroutine_create(bdrv_aio_discard_co_entry);
229261007b31SStefan Hajnoczi     qemu_coroutine_enter(co, acb);
229361007b31SStefan Hajnoczi 
229461007b31SStefan Hajnoczi     bdrv_co_maybe_schedule_bh(acb);
229561007b31SStefan Hajnoczi     return &acb->common;
229661007b31SStefan Hajnoczi }
229761007b31SStefan Hajnoczi 
229861007b31SStefan Hajnoczi void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
229961007b31SStefan Hajnoczi                    BlockCompletionFunc *cb, void *opaque)
230061007b31SStefan Hajnoczi {
230161007b31SStefan Hajnoczi     BlockAIOCB *acb;
230261007b31SStefan Hajnoczi 
2303c84b3192SPaolo Bonzini     acb = g_malloc(aiocb_info->aiocb_size);
230461007b31SStefan Hajnoczi     acb->aiocb_info = aiocb_info;
230561007b31SStefan Hajnoczi     acb->bs = bs;
230661007b31SStefan Hajnoczi     acb->cb = cb;
230761007b31SStefan Hajnoczi     acb->opaque = opaque;
230861007b31SStefan Hajnoczi     acb->refcnt = 1;
230961007b31SStefan Hajnoczi     return acb;
231061007b31SStefan Hajnoczi }
231161007b31SStefan Hajnoczi 
231261007b31SStefan Hajnoczi void qemu_aio_ref(void *p)
231361007b31SStefan Hajnoczi {
231461007b31SStefan Hajnoczi     BlockAIOCB *acb = p;
231561007b31SStefan Hajnoczi     acb->refcnt++;
231661007b31SStefan Hajnoczi }
231761007b31SStefan Hajnoczi 
231861007b31SStefan Hajnoczi void qemu_aio_unref(void *p)
231961007b31SStefan Hajnoczi {
232061007b31SStefan Hajnoczi     BlockAIOCB *acb = p;
232161007b31SStefan Hajnoczi     assert(acb->refcnt > 0);
232261007b31SStefan Hajnoczi     if (--acb->refcnt == 0) {
2323c84b3192SPaolo Bonzini         g_free(acb);
232461007b31SStefan Hajnoczi     }
232561007b31SStefan Hajnoczi }
232661007b31SStefan Hajnoczi 
232761007b31SStefan Hajnoczi /**************************************************************/
232861007b31SStefan Hajnoczi /* Coroutine block device emulation */
232961007b31SStefan Hajnoczi 
233061007b31SStefan Hajnoczi typedef struct CoroutineIOCompletion {
233161007b31SStefan Hajnoczi     Coroutine *coroutine;
233261007b31SStefan Hajnoczi     int ret;
233361007b31SStefan Hajnoczi } CoroutineIOCompletion;
233461007b31SStefan Hajnoczi 
233561007b31SStefan Hajnoczi static void bdrv_co_io_em_complete(void *opaque, int ret)
233661007b31SStefan Hajnoczi {
233761007b31SStefan Hajnoczi     CoroutineIOCompletion *co = opaque;
233861007b31SStefan Hajnoczi 
233961007b31SStefan Hajnoczi     co->ret = ret;
234061007b31SStefan Hajnoczi     qemu_coroutine_enter(co->coroutine, NULL);
234161007b31SStefan Hajnoczi }
234261007b31SStefan Hajnoczi 
234361007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_io_em(BlockDriverState *bs, int64_t sector_num,
234461007b31SStefan Hajnoczi                                       int nb_sectors, QEMUIOVector *iov,
234561007b31SStefan Hajnoczi                                       bool is_write)
234661007b31SStefan Hajnoczi {
234761007b31SStefan Hajnoczi     CoroutineIOCompletion co = {
234861007b31SStefan Hajnoczi         .coroutine = qemu_coroutine_self(),
234961007b31SStefan Hajnoczi     };
235061007b31SStefan Hajnoczi     BlockAIOCB *acb;
235161007b31SStefan Hajnoczi 
235261007b31SStefan Hajnoczi     if (is_write) {
235361007b31SStefan Hajnoczi         acb = bs->drv->bdrv_aio_writev(bs, sector_num, iov, nb_sectors,
235461007b31SStefan Hajnoczi                                        bdrv_co_io_em_complete, &co);
235561007b31SStefan Hajnoczi     } else {
235661007b31SStefan Hajnoczi         acb = bs->drv->bdrv_aio_readv(bs, sector_num, iov, nb_sectors,
235761007b31SStefan Hajnoczi                                       bdrv_co_io_em_complete, &co);
235861007b31SStefan Hajnoczi     }
235961007b31SStefan Hajnoczi 
236061007b31SStefan Hajnoczi     trace_bdrv_co_io_em(bs, sector_num, nb_sectors, is_write, acb);
236161007b31SStefan Hajnoczi     if (!acb) {
236261007b31SStefan Hajnoczi         return -EIO;
236361007b31SStefan Hajnoczi     }
236461007b31SStefan Hajnoczi     qemu_coroutine_yield();
236561007b31SStefan Hajnoczi 
236661007b31SStefan Hajnoczi     return co.ret;
236761007b31SStefan Hajnoczi }
236861007b31SStefan Hajnoczi 
236961007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_readv_em(BlockDriverState *bs,
237061007b31SStefan Hajnoczi                                          int64_t sector_num, int nb_sectors,
237161007b31SStefan Hajnoczi                                          QEMUIOVector *iov)
237261007b31SStefan Hajnoczi {
237361007b31SStefan Hajnoczi     return bdrv_co_io_em(bs, sector_num, nb_sectors, iov, false);
237461007b31SStefan Hajnoczi }
237561007b31SStefan Hajnoczi 
237661007b31SStefan Hajnoczi static int coroutine_fn bdrv_co_writev_em(BlockDriverState *bs,
237761007b31SStefan Hajnoczi                                          int64_t sector_num, int nb_sectors,
237861007b31SStefan Hajnoczi                                          QEMUIOVector *iov)
237961007b31SStefan Hajnoczi {
238061007b31SStefan Hajnoczi     return bdrv_co_io_em(bs, sector_num, nb_sectors, iov, true);
238161007b31SStefan Hajnoczi }
238261007b31SStefan Hajnoczi 
238361007b31SStefan Hajnoczi static void coroutine_fn bdrv_flush_co_entry(void *opaque)
238461007b31SStefan Hajnoczi {
238561007b31SStefan Hajnoczi     RwCo *rwco = opaque;
238661007b31SStefan Hajnoczi 
238761007b31SStefan Hajnoczi     rwco->ret = bdrv_co_flush(rwco->bs);
238861007b31SStefan Hajnoczi }
238961007b31SStefan Hajnoczi 
239061007b31SStefan Hajnoczi int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
239161007b31SStefan Hajnoczi {
239261007b31SStefan Hajnoczi     int ret;
2393cdb5e315SFam Zheng     BdrvTrackedRequest req;
239461007b31SStefan Hajnoczi 
23951b6bc94dSDimitris Aragiorgis     if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
23961b6bc94dSDimitris Aragiorgis         bdrv_is_sg(bs)) {
239761007b31SStefan Hajnoczi         return 0;
239861007b31SStefan Hajnoczi     }
239961007b31SStefan Hajnoczi 
2400cdb5e315SFam Zheng     tracked_request_begin(&req, bs, 0, 0, BDRV_TRACKED_FLUSH);
2401c32b82afSPavel Dovgalyuk 
2402c32b82afSPavel Dovgalyuk     /* Write back all layers by calling one driver function */
2403c32b82afSPavel Dovgalyuk     if (bs->drv->bdrv_co_flush) {
2404c32b82afSPavel Dovgalyuk         ret = bs->drv->bdrv_co_flush(bs);
2405c32b82afSPavel Dovgalyuk         goto out;
2406c32b82afSPavel Dovgalyuk     }
2407c32b82afSPavel Dovgalyuk 
240861007b31SStefan Hajnoczi     /* Write back cached data to the OS even with cache=unsafe */
240961007b31SStefan Hajnoczi     BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS);
241061007b31SStefan Hajnoczi     if (bs->drv->bdrv_co_flush_to_os) {
241161007b31SStefan Hajnoczi         ret = bs->drv->bdrv_co_flush_to_os(bs);
241261007b31SStefan Hajnoczi         if (ret < 0) {
2413cdb5e315SFam Zheng             goto out;
241461007b31SStefan Hajnoczi         }
241561007b31SStefan Hajnoczi     }
241661007b31SStefan Hajnoczi 
241761007b31SStefan Hajnoczi     /* But don't actually force it to the disk with cache=unsafe */
241861007b31SStefan Hajnoczi     if (bs->open_flags & BDRV_O_NO_FLUSH) {
241961007b31SStefan Hajnoczi         goto flush_parent;
242061007b31SStefan Hajnoczi     }
242161007b31SStefan Hajnoczi 
242261007b31SStefan Hajnoczi     BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK);
242361007b31SStefan Hajnoczi     if (bs->drv->bdrv_co_flush_to_disk) {
242461007b31SStefan Hajnoczi         ret = bs->drv->bdrv_co_flush_to_disk(bs);
242561007b31SStefan Hajnoczi     } else if (bs->drv->bdrv_aio_flush) {
242661007b31SStefan Hajnoczi         BlockAIOCB *acb;
242761007b31SStefan Hajnoczi         CoroutineIOCompletion co = {
242861007b31SStefan Hajnoczi             .coroutine = qemu_coroutine_self(),
242961007b31SStefan Hajnoczi         };
243061007b31SStefan Hajnoczi 
243161007b31SStefan Hajnoczi         acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
243261007b31SStefan Hajnoczi         if (acb == NULL) {
243361007b31SStefan Hajnoczi             ret = -EIO;
243461007b31SStefan Hajnoczi         } else {
243561007b31SStefan Hajnoczi             qemu_coroutine_yield();
243661007b31SStefan Hajnoczi             ret = co.ret;
243761007b31SStefan Hajnoczi         }
243861007b31SStefan Hajnoczi     } else {
243961007b31SStefan Hajnoczi         /*
244061007b31SStefan Hajnoczi          * Some block drivers always operate in either writethrough or unsafe
244161007b31SStefan Hajnoczi          * mode and don't support bdrv_flush therefore. Usually qemu doesn't
244261007b31SStefan Hajnoczi          * know how the server works (because the behaviour is hardcoded or
244361007b31SStefan Hajnoczi          * depends on server-side configuration), so we can't ensure that
244461007b31SStefan Hajnoczi          * everything is safe on disk. Returning an error doesn't work because
244561007b31SStefan Hajnoczi          * that would break guests even if the server operates in writethrough
244661007b31SStefan Hajnoczi          * mode.
244761007b31SStefan Hajnoczi          *
244861007b31SStefan Hajnoczi          * Let's hope the user knows what he's doing.
244961007b31SStefan Hajnoczi          */
245061007b31SStefan Hajnoczi         ret = 0;
245161007b31SStefan Hajnoczi     }
245261007b31SStefan Hajnoczi     if (ret < 0) {
2453cdb5e315SFam Zheng         goto out;
245461007b31SStefan Hajnoczi     }
245561007b31SStefan Hajnoczi 
245661007b31SStefan Hajnoczi     /* Now flush the underlying protocol.  It will also have BDRV_O_NO_FLUSH
245761007b31SStefan Hajnoczi      * in the case of cache=unsafe, so there are no useless flushes.
245861007b31SStefan Hajnoczi      */
245961007b31SStefan Hajnoczi flush_parent:
2460cdb5e315SFam Zheng     ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0;
2461cdb5e315SFam Zheng out:
2462cdb5e315SFam Zheng     tracked_request_end(&req);
2463cdb5e315SFam Zheng     return ret;
246461007b31SStefan Hajnoczi }
246561007b31SStefan Hajnoczi 
246661007b31SStefan Hajnoczi int bdrv_flush(BlockDriverState *bs)
246761007b31SStefan Hajnoczi {
246861007b31SStefan Hajnoczi     Coroutine *co;
246961007b31SStefan Hajnoczi     RwCo rwco = {
247061007b31SStefan Hajnoczi         .bs = bs,
247161007b31SStefan Hajnoczi         .ret = NOT_DONE,
247261007b31SStefan Hajnoczi     };
247361007b31SStefan Hajnoczi 
247461007b31SStefan Hajnoczi     if (qemu_in_coroutine()) {
247561007b31SStefan Hajnoczi         /* Fast-path if already in coroutine context */
247661007b31SStefan Hajnoczi         bdrv_flush_co_entry(&rwco);
247761007b31SStefan Hajnoczi     } else {
247861007b31SStefan Hajnoczi         AioContext *aio_context = bdrv_get_aio_context(bs);
247961007b31SStefan Hajnoczi 
248061007b31SStefan Hajnoczi         co = qemu_coroutine_create(bdrv_flush_co_entry);
248161007b31SStefan Hajnoczi         qemu_coroutine_enter(co, &rwco);
248261007b31SStefan Hajnoczi         while (rwco.ret == NOT_DONE) {
248361007b31SStefan Hajnoczi             aio_poll(aio_context, true);
248461007b31SStefan Hajnoczi         }
248561007b31SStefan Hajnoczi     }
248661007b31SStefan Hajnoczi 
248761007b31SStefan Hajnoczi     return rwco.ret;
248861007b31SStefan Hajnoczi }
248961007b31SStefan Hajnoczi 
249061007b31SStefan Hajnoczi typedef struct DiscardCo {
249161007b31SStefan Hajnoczi     BlockDriverState *bs;
249261007b31SStefan Hajnoczi     int64_t sector_num;
249361007b31SStefan Hajnoczi     int nb_sectors;
249461007b31SStefan Hajnoczi     int ret;
249561007b31SStefan Hajnoczi } DiscardCo;
249661007b31SStefan Hajnoczi static void coroutine_fn bdrv_discard_co_entry(void *opaque)
249761007b31SStefan Hajnoczi {
249861007b31SStefan Hajnoczi     DiscardCo *rwco = opaque;
249961007b31SStefan Hajnoczi 
250061007b31SStefan Hajnoczi     rwco->ret = bdrv_co_discard(rwco->bs, rwco->sector_num, rwco->nb_sectors);
250161007b31SStefan Hajnoczi }
250261007b31SStefan Hajnoczi 
250361007b31SStefan Hajnoczi int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
250461007b31SStefan Hajnoczi                                  int nb_sectors)
250561007b31SStefan Hajnoczi {
2506b1066c87SFam Zheng     BdrvTrackedRequest req;
250761007b31SStefan Hajnoczi     int max_discard, ret;
250861007b31SStefan Hajnoczi 
250961007b31SStefan Hajnoczi     if (!bs->drv) {
251061007b31SStefan Hajnoczi         return -ENOMEDIUM;
251161007b31SStefan Hajnoczi     }
251261007b31SStefan Hajnoczi 
251361007b31SStefan Hajnoczi     ret = bdrv_check_request(bs, sector_num, nb_sectors);
251461007b31SStefan Hajnoczi     if (ret < 0) {
251561007b31SStefan Hajnoczi         return ret;
251661007b31SStefan Hajnoczi     } else if (bs->read_only) {
2517eaf5fe2dSPaolo Bonzini         return -EPERM;
251861007b31SStefan Hajnoczi     }
251904c01a5cSKevin Wolf     assert(!(bs->open_flags & BDRV_O_INACTIVE));
252061007b31SStefan Hajnoczi 
252161007b31SStefan Hajnoczi     /* Do nothing if disabled.  */
252261007b31SStefan Hajnoczi     if (!(bs->open_flags & BDRV_O_UNMAP)) {
252361007b31SStefan Hajnoczi         return 0;
252461007b31SStefan Hajnoczi     }
252561007b31SStefan Hajnoczi 
252661007b31SStefan Hajnoczi     if (!bs->drv->bdrv_co_discard && !bs->drv->bdrv_aio_discard) {
252761007b31SStefan Hajnoczi         return 0;
252861007b31SStefan Hajnoczi     }
252961007b31SStefan Hajnoczi 
2530b1066c87SFam Zheng     tracked_request_begin(&req, bs, sector_num, nb_sectors,
2531b1066c87SFam Zheng                           BDRV_TRACKED_DISCARD);
253250824995SFam Zheng     bdrv_set_dirty(bs, sector_num, nb_sectors);
253350824995SFam Zheng 
253461007b31SStefan Hajnoczi     max_discard = MIN_NON_ZERO(bs->bl.max_discard, BDRV_REQUEST_MAX_SECTORS);
253561007b31SStefan Hajnoczi     while (nb_sectors > 0) {
253661007b31SStefan Hajnoczi         int ret;
253761007b31SStefan Hajnoczi         int num = nb_sectors;
253861007b31SStefan Hajnoczi 
253961007b31SStefan Hajnoczi         /* align request */
254061007b31SStefan Hajnoczi         if (bs->bl.discard_alignment &&
254161007b31SStefan Hajnoczi             num >= bs->bl.discard_alignment &&
254261007b31SStefan Hajnoczi             sector_num % bs->bl.discard_alignment) {
254361007b31SStefan Hajnoczi             if (num > bs->bl.discard_alignment) {
254461007b31SStefan Hajnoczi                 num = bs->bl.discard_alignment;
254561007b31SStefan Hajnoczi             }
254661007b31SStefan Hajnoczi             num -= sector_num % bs->bl.discard_alignment;
254761007b31SStefan Hajnoczi         }
254861007b31SStefan Hajnoczi 
254961007b31SStefan Hajnoczi         /* limit request size */
255061007b31SStefan Hajnoczi         if (num > max_discard) {
255161007b31SStefan Hajnoczi             num = max_discard;
255261007b31SStefan Hajnoczi         }
255361007b31SStefan Hajnoczi 
255461007b31SStefan Hajnoczi         if (bs->drv->bdrv_co_discard) {
255561007b31SStefan Hajnoczi             ret = bs->drv->bdrv_co_discard(bs, sector_num, num);
255661007b31SStefan Hajnoczi         } else {
255761007b31SStefan Hajnoczi             BlockAIOCB *acb;
255861007b31SStefan Hajnoczi             CoroutineIOCompletion co = {
255961007b31SStefan Hajnoczi                 .coroutine = qemu_coroutine_self(),
256061007b31SStefan Hajnoczi             };
256161007b31SStefan Hajnoczi 
256261007b31SStefan Hajnoczi             acb = bs->drv->bdrv_aio_discard(bs, sector_num, nb_sectors,
256361007b31SStefan Hajnoczi                                             bdrv_co_io_em_complete, &co);
256461007b31SStefan Hajnoczi             if (acb == NULL) {
2565b1066c87SFam Zheng                 ret = -EIO;
2566b1066c87SFam Zheng                 goto out;
256761007b31SStefan Hajnoczi             } else {
256861007b31SStefan Hajnoczi                 qemu_coroutine_yield();
256961007b31SStefan Hajnoczi                 ret = co.ret;
257061007b31SStefan Hajnoczi             }
257161007b31SStefan Hajnoczi         }
257261007b31SStefan Hajnoczi         if (ret && ret != -ENOTSUP) {
2573b1066c87SFam Zheng             goto out;
257461007b31SStefan Hajnoczi         }
257561007b31SStefan Hajnoczi 
257661007b31SStefan Hajnoczi         sector_num += num;
257761007b31SStefan Hajnoczi         nb_sectors -= num;
257861007b31SStefan Hajnoczi     }
2579b1066c87SFam Zheng     ret = 0;
2580b1066c87SFam Zheng out:
2581b1066c87SFam Zheng     tracked_request_end(&req);
2582b1066c87SFam Zheng     return ret;
258361007b31SStefan Hajnoczi }
258461007b31SStefan Hajnoczi 
258561007b31SStefan Hajnoczi int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors)
258661007b31SStefan Hajnoczi {
258761007b31SStefan Hajnoczi     Coroutine *co;
258861007b31SStefan Hajnoczi     DiscardCo rwco = {
258961007b31SStefan Hajnoczi         .bs = bs,
259061007b31SStefan Hajnoczi         .sector_num = sector_num,
259161007b31SStefan Hajnoczi         .nb_sectors = nb_sectors,
259261007b31SStefan Hajnoczi         .ret = NOT_DONE,
259361007b31SStefan Hajnoczi     };
259461007b31SStefan Hajnoczi 
259561007b31SStefan Hajnoczi     if (qemu_in_coroutine()) {
259661007b31SStefan Hajnoczi         /* Fast-path if already in coroutine context */
259761007b31SStefan Hajnoczi         bdrv_discard_co_entry(&rwco);
259861007b31SStefan Hajnoczi     } else {
259961007b31SStefan Hajnoczi         AioContext *aio_context = bdrv_get_aio_context(bs);
260061007b31SStefan Hajnoczi 
260161007b31SStefan Hajnoczi         co = qemu_coroutine_create(bdrv_discard_co_entry);
260261007b31SStefan Hajnoczi         qemu_coroutine_enter(co, &rwco);
260361007b31SStefan Hajnoczi         while (rwco.ret == NOT_DONE) {
260461007b31SStefan Hajnoczi             aio_poll(aio_context, true);
260561007b31SStefan Hajnoczi         }
260661007b31SStefan Hajnoczi     }
260761007b31SStefan Hajnoczi 
260861007b31SStefan Hajnoczi     return rwco.ret;
260961007b31SStefan Hajnoczi }
261061007b31SStefan Hajnoczi 
26115c5ae76aSFam Zheng typedef struct {
26125c5ae76aSFam Zheng     CoroutineIOCompletion *co;
26135c5ae76aSFam Zheng     QEMUBH *bh;
26145c5ae76aSFam Zheng } BdrvIoctlCompletionData;
261561007b31SStefan Hajnoczi 
26165c5ae76aSFam Zheng static void bdrv_ioctl_bh_cb(void *opaque)
26175c5ae76aSFam Zheng {
26185c5ae76aSFam Zheng     BdrvIoctlCompletionData *data = opaque;
26195c5ae76aSFam Zheng 
26205c5ae76aSFam Zheng     bdrv_co_io_em_complete(data->co, -ENOTSUP);
26215c5ae76aSFam Zheng     qemu_bh_delete(data->bh);
26225c5ae76aSFam Zheng }
26235c5ae76aSFam Zheng 
26245c5ae76aSFam Zheng static int bdrv_co_do_ioctl(BlockDriverState *bs, int req, void *buf)
262561007b31SStefan Hajnoczi {
262661007b31SStefan Hajnoczi     BlockDriver *drv = bs->drv;
26275c5ae76aSFam Zheng     BdrvTrackedRequest tracked_req;
26285c5ae76aSFam Zheng     CoroutineIOCompletion co = {
26295c5ae76aSFam Zheng         .coroutine = qemu_coroutine_self(),
26305c5ae76aSFam Zheng     };
26315c5ae76aSFam Zheng     BlockAIOCB *acb;
263261007b31SStefan Hajnoczi 
26335c5ae76aSFam Zheng     tracked_request_begin(&tracked_req, bs, 0, 0, BDRV_TRACKED_IOCTL);
26345c5ae76aSFam Zheng     if (!drv || !drv->bdrv_aio_ioctl) {
26355c5ae76aSFam Zheng         co.ret = -ENOTSUP;
26365c5ae76aSFam Zheng         goto out;
26375c5ae76aSFam Zheng     }
26385c5ae76aSFam Zheng 
26395c5ae76aSFam Zheng     acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
26405c5ae76aSFam Zheng     if (!acb) {
26415c5ae76aSFam Zheng         BdrvIoctlCompletionData *data = g_new(BdrvIoctlCompletionData, 1);
26425c5ae76aSFam Zheng         data->bh = aio_bh_new(bdrv_get_aio_context(bs),
26435c5ae76aSFam Zheng                                 bdrv_ioctl_bh_cb, data);
26445c5ae76aSFam Zheng         data->co = &co;
26455c5ae76aSFam Zheng         qemu_bh_schedule(data->bh);
26465c5ae76aSFam Zheng     }
26475c5ae76aSFam Zheng     qemu_coroutine_yield();
26485c5ae76aSFam Zheng out:
26495c5ae76aSFam Zheng     tracked_request_end(&tracked_req);
26505c5ae76aSFam Zheng     return co.ret;
26515c5ae76aSFam Zheng }
26525c5ae76aSFam Zheng 
26535c5ae76aSFam Zheng typedef struct {
26545c5ae76aSFam Zheng     BlockDriverState *bs;
26555c5ae76aSFam Zheng     int req;
26565c5ae76aSFam Zheng     void *buf;
26575c5ae76aSFam Zheng     int ret;
26585c5ae76aSFam Zheng } BdrvIoctlCoData;
26595c5ae76aSFam Zheng 
26605c5ae76aSFam Zheng static void coroutine_fn bdrv_co_ioctl_entry(void *opaque)
26615c5ae76aSFam Zheng {
26625c5ae76aSFam Zheng     BdrvIoctlCoData *data = opaque;
26635c5ae76aSFam Zheng     data->ret = bdrv_co_do_ioctl(data->bs, data->req, data->buf);
26645c5ae76aSFam Zheng }
26655c5ae76aSFam Zheng 
26665c5ae76aSFam Zheng /* needed for generic scsi interface */
26675c5ae76aSFam Zheng int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
26685c5ae76aSFam Zheng {
26695c5ae76aSFam Zheng     BdrvIoctlCoData data = {
26705c5ae76aSFam Zheng         .bs = bs,
26715c5ae76aSFam Zheng         .req = req,
26725c5ae76aSFam Zheng         .buf = buf,
26735c5ae76aSFam Zheng         .ret = -EINPROGRESS,
26745c5ae76aSFam Zheng     };
26755c5ae76aSFam Zheng 
26765c5ae76aSFam Zheng     if (qemu_in_coroutine()) {
26775c5ae76aSFam Zheng         /* Fast-path if already in coroutine context */
26785c5ae76aSFam Zheng         bdrv_co_ioctl_entry(&data);
26795c5ae76aSFam Zheng     } else {
26805c5ae76aSFam Zheng         Coroutine *co = qemu_coroutine_create(bdrv_co_ioctl_entry);
2681ba889444SPaolo Bonzini 
26825c5ae76aSFam Zheng         qemu_coroutine_enter(co, &data);
26835c5ae76aSFam Zheng         while (data.ret == -EINPROGRESS) {
26845c5ae76aSFam Zheng             aio_poll(bdrv_get_aio_context(bs), true);
26855c5ae76aSFam Zheng         }
2686ba889444SPaolo Bonzini     }
26875c5ae76aSFam Zheng     return data.ret;
26885c5ae76aSFam Zheng }
26895c5ae76aSFam Zheng 
26905c5ae76aSFam Zheng static void coroutine_fn bdrv_co_aio_ioctl_entry(void *opaque)
26915c5ae76aSFam Zheng {
26925c5ae76aSFam Zheng     BlockAIOCBCoroutine *acb = opaque;
26935c5ae76aSFam Zheng     acb->req.error = bdrv_co_do_ioctl(acb->common.bs,
26945c5ae76aSFam Zheng                                       acb->req.req, acb->req.buf);
26955c5ae76aSFam Zheng     bdrv_co_complete(acb);
269661007b31SStefan Hajnoczi }
269761007b31SStefan Hajnoczi 
269861007b31SStefan Hajnoczi BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
269961007b31SStefan Hajnoczi         unsigned long int req, void *buf,
270061007b31SStefan Hajnoczi         BlockCompletionFunc *cb, void *opaque)
270161007b31SStefan Hajnoczi {
27025c5ae76aSFam Zheng     BlockAIOCBCoroutine *acb = qemu_aio_get(&bdrv_em_co_aiocb_info,
27035c5ae76aSFam Zheng                                             bs, cb, opaque);
27045c5ae76aSFam Zheng     Coroutine *co;
270561007b31SStefan Hajnoczi 
27065c5ae76aSFam Zheng     acb->need_bh = true;
27075c5ae76aSFam Zheng     acb->req.error = -EINPROGRESS;
27085c5ae76aSFam Zheng     acb->req.req = req;
27095c5ae76aSFam Zheng     acb->req.buf = buf;
27105c5ae76aSFam Zheng     co = qemu_coroutine_create(bdrv_co_aio_ioctl_entry);
27115c5ae76aSFam Zheng     qemu_coroutine_enter(co, acb);
27125c5ae76aSFam Zheng 
27135c5ae76aSFam Zheng     bdrv_co_maybe_schedule_bh(acb);
27145c5ae76aSFam Zheng     return &acb->common;
271561007b31SStefan Hajnoczi }
271661007b31SStefan Hajnoczi 
271761007b31SStefan Hajnoczi void *qemu_blockalign(BlockDriverState *bs, size_t size)
271861007b31SStefan Hajnoczi {
271961007b31SStefan Hajnoczi     return qemu_memalign(bdrv_opt_mem_align(bs), size);
272061007b31SStefan Hajnoczi }
272161007b31SStefan Hajnoczi 
272261007b31SStefan Hajnoczi void *qemu_blockalign0(BlockDriverState *bs, size_t size)
272361007b31SStefan Hajnoczi {
272461007b31SStefan Hajnoczi     return memset(qemu_blockalign(bs, size), 0, size);
272561007b31SStefan Hajnoczi }
272661007b31SStefan Hajnoczi 
272761007b31SStefan Hajnoczi void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
272861007b31SStefan Hajnoczi {
272961007b31SStefan Hajnoczi     size_t align = bdrv_opt_mem_align(bs);
273061007b31SStefan Hajnoczi 
273161007b31SStefan Hajnoczi     /* Ensure that NULL is never returned on success */
273261007b31SStefan Hajnoczi     assert(align > 0);
273361007b31SStefan Hajnoczi     if (size == 0) {
273461007b31SStefan Hajnoczi         size = align;
273561007b31SStefan Hajnoczi     }
273661007b31SStefan Hajnoczi 
273761007b31SStefan Hajnoczi     return qemu_try_memalign(align, size);
273861007b31SStefan Hajnoczi }
273961007b31SStefan Hajnoczi 
274061007b31SStefan Hajnoczi void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
274161007b31SStefan Hajnoczi {
274261007b31SStefan Hajnoczi     void *mem = qemu_try_blockalign(bs, size);
274361007b31SStefan Hajnoczi 
274461007b31SStefan Hajnoczi     if (mem) {
274561007b31SStefan Hajnoczi         memset(mem, 0, size);
274661007b31SStefan Hajnoczi     }
274761007b31SStefan Hajnoczi 
274861007b31SStefan Hajnoczi     return mem;
274961007b31SStefan Hajnoczi }
275061007b31SStefan Hajnoczi 
275161007b31SStefan Hajnoczi /*
275261007b31SStefan Hajnoczi  * Check if all memory in this vector is sector aligned.
275361007b31SStefan Hajnoczi  */
275461007b31SStefan Hajnoczi bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
275561007b31SStefan Hajnoczi {
275661007b31SStefan Hajnoczi     int i;
27574196d2f0SDenis V. Lunev     size_t alignment = bdrv_min_mem_align(bs);
275861007b31SStefan Hajnoczi 
275961007b31SStefan Hajnoczi     for (i = 0; i < qiov->niov; i++) {
276061007b31SStefan Hajnoczi         if ((uintptr_t) qiov->iov[i].iov_base % alignment) {
276161007b31SStefan Hajnoczi             return false;
276261007b31SStefan Hajnoczi         }
276361007b31SStefan Hajnoczi         if (qiov->iov[i].iov_len % alignment) {
276461007b31SStefan Hajnoczi             return false;
276561007b31SStefan Hajnoczi         }
276661007b31SStefan Hajnoczi     }
276761007b31SStefan Hajnoczi 
276861007b31SStefan Hajnoczi     return true;
276961007b31SStefan Hajnoczi }
277061007b31SStefan Hajnoczi 
277161007b31SStefan Hajnoczi void bdrv_add_before_write_notifier(BlockDriverState *bs,
277261007b31SStefan Hajnoczi                                     NotifierWithReturn *notifier)
277361007b31SStefan Hajnoczi {
277461007b31SStefan Hajnoczi     notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
277561007b31SStefan Hajnoczi }
277661007b31SStefan Hajnoczi 
277761007b31SStefan Hajnoczi void bdrv_io_plug(BlockDriverState *bs)
277861007b31SStefan Hajnoczi {
27796b98bd64SPaolo Bonzini     BdrvChild *child;
27806b98bd64SPaolo Bonzini 
27816b98bd64SPaolo Bonzini     QLIST_FOREACH(child, &bs->children, next) {
27826b98bd64SPaolo Bonzini         bdrv_io_plug(child->bs);
27836b98bd64SPaolo Bonzini     }
27846b98bd64SPaolo Bonzini 
27856b98bd64SPaolo Bonzini     if (bs->io_plugged++ == 0 && bs->io_plug_disabled == 0) {
278661007b31SStefan Hajnoczi         BlockDriver *drv = bs->drv;
278761007b31SStefan Hajnoczi         if (drv && drv->bdrv_io_plug) {
278861007b31SStefan Hajnoczi             drv->bdrv_io_plug(bs);
27896b98bd64SPaolo Bonzini         }
279061007b31SStefan Hajnoczi     }
279161007b31SStefan Hajnoczi }
279261007b31SStefan Hajnoczi 
279361007b31SStefan Hajnoczi void bdrv_io_unplug(BlockDriverState *bs)
279461007b31SStefan Hajnoczi {
27956b98bd64SPaolo Bonzini     BdrvChild *child;
27966b98bd64SPaolo Bonzini 
27976b98bd64SPaolo Bonzini     assert(bs->io_plugged);
27986b98bd64SPaolo Bonzini     if (--bs->io_plugged == 0 && bs->io_plug_disabled == 0) {
279961007b31SStefan Hajnoczi         BlockDriver *drv = bs->drv;
280061007b31SStefan Hajnoczi         if (drv && drv->bdrv_io_unplug) {
280161007b31SStefan Hajnoczi             drv->bdrv_io_unplug(bs);
280261007b31SStefan Hajnoczi         }
280361007b31SStefan Hajnoczi     }
280461007b31SStefan Hajnoczi 
28056b98bd64SPaolo Bonzini     QLIST_FOREACH(child, &bs->children, next) {
28066b98bd64SPaolo Bonzini         bdrv_io_unplug(child->bs);
28076b98bd64SPaolo Bonzini     }
28086b98bd64SPaolo Bonzini }
28096b98bd64SPaolo Bonzini 
28106b98bd64SPaolo Bonzini void bdrv_io_unplugged_begin(BlockDriverState *bs)
281161007b31SStefan Hajnoczi {
28126b98bd64SPaolo Bonzini     BdrvChild *child;
28136b98bd64SPaolo Bonzini 
28146b98bd64SPaolo Bonzini     if (bs->io_plug_disabled++ == 0 && bs->io_plugged > 0) {
281561007b31SStefan Hajnoczi         BlockDriver *drv = bs->drv;
28166b98bd64SPaolo Bonzini         if (drv && drv->bdrv_io_unplug) {
28176b98bd64SPaolo Bonzini             drv->bdrv_io_unplug(bs);
28186b98bd64SPaolo Bonzini         }
28196b98bd64SPaolo Bonzini     }
28206b98bd64SPaolo Bonzini 
28216b98bd64SPaolo Bonzini     QLIST_FOREACH(child, &bs->children, next) {
28226b98bd64SPaolo Bonzini         bdrv_io_unplugged_begin(child->bs);
28236b98bd64SPaolo Bonzini     }
28246b98bd64SPaolo Bonzini }
28256b98bd64SPaolo Bonzini 
28266b98bd64SPaolo Bonzini void bdrv_io_unplugged_end(BlockDriverState *bs)
28276b98bd64SPaolo Bonzini {
28286b98bd64SPaolo Bonzini     BdrvChild *child;
28296b98bd64SPaolo Bonzini 
28306b98bd64SPaolo Bonzini     assert(bs->io_plug_disabled);
28316b98bd64SPaolo Bonzini     QLIST_FOREACH(child, &bs->children, next) {
28326b98bd64SPaolo Bonzini         bdrv_io_unplugged_end(child->bs);
28336b98bd64SPaolo Bonzini     }
28346b98bd64SPaolo Bonzini 
28356b98bd64SPaolo Bonzini     if (--bs->io_plug_disabled == 0 && bs->io_plugged > 0) {
28366b98bd64SPaolo Bonzini         BlockDriver *drv = bs->drv;
28376b98bd64SPaolo Bonzini         if (drv && drv->bdrv_io_plug) {
28386b98bd64SPaolo Bonzini             drv->bdrv_io_plug(bs);
28396b98bd64SPaolo Bonzini         }
284061007b31SStefan Hajnoczi     }
284161007b31SStefan Hajnoczi }
284251288d79SFam Zheng 
284351288d79SFam Zheng void bdrv_drained_begin(BlockDriverState *bs)
284451288d79SFam Zheng {
284551288d79SFam Zheng     if (!bs->quiesce_counter++) {
284651288d79SFam Zheng         aio_disable_external(bdrv_get_aio_context(bs));
284751288d79SFam Zheng     }
284851288d79SFam Zheng     bdrv_drain(bs);
284951288d79SFam Zheng }
285051288d79SFam Zheng 
285151288d79SFam Zheng void bdrv_drained_end(BlockDriverState *bs)
285251288d79SFam Zheng {
285351288d79SFam Zheng     assert(bs->quiesce_counter > 0);
285451288d79SFam Zheng     if (--bs->quiesce_counter > 0) {
285551288d79SFam Zheng         return;
285651288d79SFam Zheng     }
285751288d79SFam Zheng     aio_enable_external(bdrv_get_aio_context(bs));
285851288d79SFam Zheng }
2859