161007b31SStefan Hajnoczi /* 261007b31SStefan Hajnoczi * Block layer I/O functions 361007b31SStefan Hajnoczi * 461007b31SStefan Hajnoczi * Copyright (c) 2003 Fabrice Bellard 561007b31SStefan Hajnoczi * 661007b31SStefan Hajnoczi * Permission is hereby granted, free of charge, to any person obtaining a copy 761007b31SStefan Hajnoczi * of this software and associated documentation files (the "Software"), to deal 861007b31SStefan Hajnoczi * in the Software without restriction, including without limitation the rights 961007b31SStefan Hajnoczi * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 1061007b31SStefan Hajnoczi * copies of the Software, and to permit persons to whom the Software is 1161007b31SStefan Hajnoczi * furnished to do so, subject to the following conditions: 1261007b31SStefan Hajnoczi * 1361007b31SStefan Hajnoczi * The above copyright notice and this permission notice shall be included in 1461007b31SStefan Hajnoczi * all copies or substantial portions of the Software. 1561007b31SStefan Hajnoczi * 1661007b31SStefan Hajnoczi * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 1761007b31SStefan Hajnoczi * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 1861007b31SStefan Hajnoczi * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 1961007b31SStefan Hajnoczi * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 2061007b31SStefan Hajnoczi * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 2161007b31SStefan Hajnoczi * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 2261007b31SStefan Hajnoczi * THE SOFTWARE. 2361007b31SStefan Hajnoczi */ 2461007b31SStefan Hajnoczi 2580c71a24SPeter Maydell #include "qemu/osdep.h" 2661007b31SStefan Hajnoczi #include "trace.h" 277f0e9da6SMax Reitz #include "sysemu/block-backend.h" 287719f3c9SStefan Hajnoczi #include "block/aio-wait.h" 2961007b31SStefan Hajnoczi #include "block/blockjob.h" 30f321dcb5SPaolo Bonzini #include "block/blockjob_int.h" 3161007b31SStefan Hajnoczi #include "block/block_int.h" 32f348b6d1SVeronia Bahaa #include "qemu/cutils.h" 33da34e65cSMarkus Armbruster #include "qapi/error.h" 34d49b6836SMarkus Armbruster #include "qemu/error-report.h" 3561007b31SStefan Hajnoczi 3661007b31SStefan Hajnoczi #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */ 3761007b31SStefan Hajnoczi 38cb2e2878SEric Blake /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */ 39cb2e2878SEric Blake #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS) 40cb2e2878SEric Blake 417f8f03efSFam Zheng static void bdrv_parent_cb_resize(BlockDriverState *bs); 42d05aa8bbSEric Blake static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 43f5a5ca79SManos Pitsidianakis int64_t offset, int bytes, BdrvRequestFlags flags); 4461007b31SStefan Hajnoczi 456cd5c9d7SKevin Wolf void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore, 466cd5c9d7SKevin Wolf bool ignore_bds_parents) 4761007b31SStefan Hajnoczi { 4802d21300SKevin Wolf BdrvChild *c, *next; 4927ccdd52SKevin Wolf 5002d21300SKevin Wolf QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) { 516cd5c9d7SKevin Wolf if (c == ignore || (ignore_bds_parents && c->role->parent_is_bds)) { 520152bf40SKevin Wolf continue; 530152bf40SKevin Wolf } 544be6a6d1SKevin Wolf bdrv_parent_drained_begin_single(c, false); 55ce0f1412SPaolo Bonzini } 56ce0f1412SPaolo Bonzini } 57ce0f1412SPaolo Bonzini 58804db8eaSMax Reitz void bdrv_parent_drained_end_single(BdrvChild *c) 59804db8eaSMax Reitz { 60804db8eaSMax Reitz assert(c->parent_quiesce_counter > 0); 61804db8eaSMax Reitz c->parent_quiesce_counter--; 62804db8eaSMax Reitz if (c->role->drained_end) { 63804db8eaSMax Reitz c->role->drained_end(c); 64804db8eaSMax Reitz } 65804db8eaSMax Reitz } 66804db8eaSMax Reitz 676cd5c9d7SKevin Wolf void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore, 686cd5c9d7SKevin Wolf bool ignore_bds_parents) 69ce0f1412SPaolo Bonzini { 7002d21300SKevin Wolf BdrvChild *c, *next; 7127ccdd52SKevin Wolf 7202d21300SKevin Wolf QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) { 736cd5c9d7SKevin Wolf if (c == ignore || (ignore_bds_parents && c->role->parent_is_bds)) { 740152bf40SKevin Wolf continue; 750152bf40SKevin Wolf } 76804db8eaSMax Reitz bdrv_parent_drained_end_single(c); 77c2066af0SKevin Wolf } 7861007b31SStefan Hajnoczi } 7961007b31SStefan Hajnoczi 804be6a6d1SKevin Wolf static bool bdrv_parent_drained_poll_single(BdrvChild *c) 814be6a6d1SKevin Wolf { 824be6a6d1SKevin Wolf if (c->role->drained_poll) { 834be6a6d1SKevin Wolf return c->role->drained_poll(c); 844be6a6d1SKevin Wolf } 854be6a6d1SKevin Wolf return false; 864be6a6d1SKevin Wolf } 874be6a6d1SKevin Wolf 886cd5c9d7SKevin Wolf static bool bdrv_parent_drained_poll(BlockDriverState *bs, BdrvChild *ignore, 896cd5c9d7SKevin Wolf bool ignore_bds_parents) 9089bd0305SKevin Wolf { 9189bd0305SKevin Wolf BdrvChild *c, *next; 9289bd0305SKevin Wolf bool busy = false; 9389bd0305SKevin Wolf 9489bd0305SKevin Wolf QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) { 956cd5c9d7SKevin Wolf if (c == ignore || (ignore_bds_parents && c->role->parent_is_bds)) { 9689bd0305SKevin Wolf continue; 9789bd0305SKevin Wolf } 984be6a6d1SKevin Wolf busy |= bdrv_parent_drained_poll_single(c); 9989bd0305SKevin Wolf } 10089bd0305SKevin Wolf 10189bd0305SKevin Wolf return busy; 10289bd0305SKevin Wolf } 10389bd0305SKevin Wolf 1044be6a6d1SKevin Wolf void bdrv_parent_drained_begin_single(BdrvChild *c, bool poll) 1054be6a6d1SKevin Wolf { 106804db8eaSMax Reitz c->parent_quiesce_counter++; 1074be6a6d1SKevin Wolf if (c->role->drained_begin) { 1084be6a6d1SKevin Wolf c->role->drained_begin(c); 1094be6a6d1SKevin Wolf } 1104be6a6d1SKevin Wolf if (poll) { 1114be6a6d1SKevin Wolf BDRV_POLL_WHILE(c->bs, bdrv_parent_drained_poll_single(c)); 1124be6a6d1SKevin Wolf } 1134be6a6d1SKevin Wolf } 1144be6a6d1SKevin Wolf 115d9e0dfa2SEric Blake static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src) 116d9e0dfa2SEric Blake { 117d9e0dfa2SEric Blake dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer); 118d9e0dfa2SEric Blake dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer); 119d9e0dfa2SEric Blake dst->opt_mem_alignment = MAX(dst->opt_mem_alignment, 120d9e0dfa2SEric Blake src->opt_mem_alignment); 121d9e0dfa2SEric Blake dst->min_mem_alignment = MAX(dst->min_mem_alignment, 122d9e0dfa2SEric Blake src->min_mem_alignment); 123d9e0dfa2SEric Blake dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov); 124d9e0dfa2SEric Blake } 125d9e0dfa2SEric Blake 12661007b31SStefan Hajnoczi void bdrv_refresh_limits(BlockDriverState *bs, Error **errp) 12761007b31SStefan Hajnoczi { 12861007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 12961007b31SStefan Hajnoczi Error *local_err = NULL; 13061007b31SStefan Hajnoczi 13161007b31SStefan Hajnoczi memset(&bs->bl, 0, sizeof(bs->bl)); 13261007b31SStefan Hajnoczi 13361007b31SStefan Hajnoczi if (!drv) { 13461007b31SStefan Hajnoczi return; 13561007b31SStefan Hajnoczi } 13661007b31SStefan Hajnoczi 13779ba8c98SEric Blake /* Default alignment based on whether driver has byte interface */ 138e31f6864SEric Blake bs->bl.request_alignment = (drv->bdrv_co_preadv || 139e31f6864SEric Blake drv->bdrv_aio_preadv) ? 1 : 512; 14079ba8c98SEric Blake 14161007b31SStefan Hajnoczi /* Take some limits from the children as a default */ 14261007b31SStefan Hajnoczi if (bs->file) { 1439a4f4c31SKevin Wolf bdrv_refresh_limits(bs->file->bs, &local_err); 14461007b31SStefan Hajnoczi if (local_err) { 14561007b31SStefan Hajnoczi error_propagate(errp, local_err); 14661007b31SStefan Hajnoczi return; 14761007b31SStefan Hajnoczi } 148d9e0dfa2SEric Blake bdrv_merge_limits(&bs->bl, &bs->file->bs->bl); 14961007b31SStefan Hajnoczi } else { 1504196d2f0SDenis V. Lunev bs->bl.min_mem_alignment = 512; 151459b4e66SDenis V. Lunev bs->bl.opt_mem_alignment = getpagesize(); 152bd44feb7SStefan Hajnoczi 153bd44feb7SStefan Hajnoczi /* Safe default since most protocols use readv()/writev()/etc */ 154bd44feb7SStefan Hajnoczi bs->bl.max_iov = IOV_MAX; 15561007b31SStefan Hajnoczi } 15661007b31SStefan Hajnoczi 157760e0063SKevin Wolf if (bs->backing) { 158760e0063SKevin Wolf bdrv_refresh_limits(bs->backing->bs, &local_err); 15961007b31SStefan Hajnoczi if (local_err) { 16061007b31SStefan Hajnoczi error_propagate(errp, local_err); 16161007b31SStefan Hajnoczi return; 16261007b31SStefan Hajnoczi } 163d9e0dfa2SEric Blake bdrv_merge_limits(&bs->bl, &bs->backing->bs->bl); 16461007b31SStefan Hajnoczi } 16561007b31SStefan Hajnoczi 16661007b31SStefan Hajnoczi /* Then let the driver override it */ 16761007b31SStefan Hajnoczi if (drv->bdrv_refresh_limits) { 16861007b31SStefan Hajnoczi drv->bdrv_refresh_limits(bs, errp); 16961007b31SStefan Hajnoczi } 17061007b31SStefan Hajnoczi } 17161007b31SStefan Hajnoczi 17261007b31SStefan Hajnoczi /** 17361007b31SStefan Hajnoczi * The copy-on-read flag is actually a reference count so multiple users may 17461007b31SStefan Hajnoczi * use the feature without worrying about clobbering its previous state. 17561007b31SStefan Hajnoczi * Copy-on-read stays enabled until all users have called to disable it. 17661007b31SStefan Hajnoczi */ 17761007b31SStefan Hajnoczi void bdrv_enable_copy_on_read(BlockDriverState *bs) 17861007b31SStefan Hajnoczi { 179d3faa13eSPaolo Bonzini atomic_inc(&bs->copy_on_read); 18061007b31SStefan Hajnoczi } 18161007b31SStefan Hajnoczi 18261007b31SStefan Hajnoczi void bdrv_disable_copy_on_read(BlockDriverState *bs) 18361007b31SStefan Hajnoczi { 184d3faa13eSPaolo Bonzini int old = atomic_fetch_dec(&bs->copy_on_read); 185d3faa13eSPaolo Bonzini assert(old >= 1); 18661007b31SStefan Hajnoczi } 18761007b31SStefan Hajnoczi 18861124f03SPaolo Bonzini typedef struct { 18961124f03SPaolo Bonzini Coroutine *co; 19061124f03SPaolo Bonzini BlockDriverState *bs; 19161124f03SPaolo Bonzini bool done; 192481cad48SManos Pitsidianakis bool begin; 193b0165585SKevin Wolf bool recursive; 194fe4f0614SKevin Wolf bool poll; 1950152bf40SKevin Wolf BdrvChild *parent; 1966cd5c9d7SKevin Wolf bool ignore_bds_parents; 197*8e1da77eSMax Reitz int *drained_end_counter; 19861124f03SPaolo Bonzini } BdrvCoDrainData; 19961124f03SPaolo Bonzini 20061124f03SPaolo Bonzini static void coroutine_fn bdrv_drain_invoke_entry(void *opaque) 20161124f03SPaolo Bonzini { 20261124f03SPaolo Bonzini BdrvCoDrainData *data = opaque; 20361124f03SPaolo Bonzini BlockDriverState *bs = data->bs; 20461124f03SPaolo Bonzini 205481cad48SManos Pitsidianakis if (data->begin) { 206f8ea8dacSManos Pitsidianakis bs->drv->bdrv_co_drain_begin(bs); 207481cad48SManos Pitsidianakis } else { 208481cad48SManos Pitsidianakis bs->drv->bdrv_co_drain_end(bs); 209481cad48SManos Pitsidianakis } 21061124f03SPaolo Bonzini 21161124f03SPaolo Bonzini /* Set data->done before reading bs->wakeup. */ 21261124f03SPaolo Bonzini atomic_mb_set(&data->done, true); 2130109e7e6SKevin Wolf bdrv_dec_in_flight(bs); 2140109e7e6SKevin Wolf 215*8e1da77eSMax Reitz if (data->drained_end_counter) { 216*8e1da77eSMax Reitz atomic_dec(data->drained_end_counter); 217*8e1da77eSMax Reitz } 218*8e1da77eSMax Reitz 219*8e1da77eSMax Reitz if (data->begin || data->drained_end_counter) { 2200109e7e6SKevin Wolf g_free(data); 2210109e7e6SKevin Wolf } 22261124f03SPaolo Bonzini } 22361124f03SPaolo Bonzini 224db0289b9SKevin Wolf /* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */ 225*8e1da77eSMax Reitz static void bdrv_drain_invoke(BlockDriverState *bs, bool begin, 226*8e1da77eSMax Reitz int *drained_end_counter) 22761124f03SPaolo Bonzini { 2280109e7e6SKevin Wolf BdrvCoDrainData *data; 22961124f03SPaolo Bonzini 230f8ea8dacSManos Pitsidianakis if (!bs->drv || (begin && !bs->drv->bdrv_co_drain_begin) || 231481cad48SManos Pitsidianakis (!begin && !bs->drv->bdrv_co_drain_end)) { 23261124f03SPaolo Bonzini return; 23361124f03SPaolo Bonzini } 23461124f03SPaolo Bonzini 2350109e7e6SKevin Wolf data = g_new(BdrvCoDrainData, 1); 2360109e7e6SKevin Wolf *data = (BdrvCoDrainData) { 2370109e7e6SKevin Wolf .bs = bs, 2380109e7e6SKevin Wolf .done = false, 239*8e1da77eSMax Reitz .begin = begin, 240*8e1da77eSMax Reitz .drained_end_counter = drained_end_counter, 2410109e7e6SKevin Wolf }; 2420109e7e6SKevin Wolf 243*8e1da77eSMax Reitz if (!begin && drained_end_counter) { 244*8e1da77eSMax Reitz atomic_inc(drained_end_counter); 245*8e1da77eSMax Reitz } 246*8e1da77eSMax Reitz 2470109e7e6SKevin Wolf /* Make sure the driver callback completes during the polling phase for 2480109e7e6SKevin Wolf * drain_begin. */ 2490109e7e6SKevin Wolf bdrv_inc_in_flight(bs); 2500109e7e6SKevin Wolf data->co = qemu_coroutine_create(bdrv_drain_invoke_entry, data); 2510109e7e6SKevin Wolf aio_co_schedule(bdrv_get_aio_context(bs), data->co); 2520109e7e6SKevin Wolf 253*8e1da77eSMax Reitz /* 254*8e1da77eSMax Reitz * TODO: Drop this and make callers pass @drained_end_counter and poll 255*8e1da77eSMax Reitz * themselves 256*8e1da77eSMax Reitz */ 257*8e1da77eSMax Reitz if (!begin && !drained_end_counter) { 2580109e7e6SKevin Wolf BDRV_POLL_WHILE(bs, !data->done); 2590109e7e6SKevin Wolf g_free(data); 2600109e7e6SKevin Wolf } 26161124f03SPaolo Bonzini } 26261124f03SPaolo Bonzini 2631cc8e54aSKevin Wolf /* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */ 264fe4f0614SKevin Wolf bool bdrv_drain_poll(BlockDriverState *bs, bool recursive, 2656cd5c9d7SKevin Wolf BdrvChild *ignore_parent, bool ignore_bds_parents) 26689bd0305SKevin Wolf { 267fe4f0614SKevin Wolf BdrvChild *child, *next; 268fe4f0614SKevin Wolf 2696cd5c9d7SKevin Wolf if (bdrv_parent_drained_poll(bs, ignore_parent, ignore_bds_parents)) { 27089bd0305SKevin Wolf return true; 27189bd0305SKevin Wolf } 27289bd0305SKevin Wolf 273fe4f0614SKevin Wolf if (atomic_read(&bs->in_flight)) { 274fe4f0614SKevin Wolf return true; 27589bd0305SKevin Wolf } 27689bd0305SKevin Wolf 277fe4f0614SKevin Wolf if (recursive) { 2786cd5c9d7SKevin Wolf assert(!ignore_bds_parents); 279fe4f0614SKevin Wolf QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 2806cd5c9d7SKevin Wolf if (bdrv_drain_poll(child->bs, recursive, child, false)) { 281fe4f0614SKevin Wolf return true; 282fe4f0614SKevin Wolf } 283fe4f0614SKevin Wolf } 284fe4f0614SKevin Wolf } 285fe4f0614SKevin Wolf 286fe4f0614SKevin Wolf return false; 287fe4f0614SKevin Wolf } 288fe4f0614SKevin Wolf 289fe4f0614SKevin Wolf static bool bdrv_drain_poll_top_level(BlockDriverState *bs, bool recursive, 29089bd0305SKevin Wolf BdrvChild *ignore_parent) 2911cc8e54aSKevin Wolf { 2926cd5c9d7SKevin Wolf return bdrv_drain_poll(bs, recursive, ignore_parent, false); 2931cc8e54aSKevin Wolf } 2941cc8e54aSKevin Wolf 295b0165585SKevin Wolf static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive, 2966cd5c9d7SKevin Wolf BdrvChild *parent, bool ignore_bds_parents, 2976cd5c9d7SKevin Wolf bool poll); 298b0165585SKevin Wolf static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive, 299*8e1da77eSMax Reitz BdrvChild *parent, bool ignore_bds_parents, 300*8e1da77eSMax Reitz int *drained_end_counter); 3010152bf40SKevin Wolf 302a77fd4bbSFam Zheng static void bdrv_co_drain_bh_cb(void *opaque) 303a77fd4bbSFam Zheng { 304a77fd4bbSFam Zheng BdrvCoDrainData *data = opaque; 305a77fd4bbSFam Zheng Coroutine *co = data->co; 30699723548SPaolo Bonzini BlockDriverState *bs = data->bs; 307a77fd4bbSFam Zheng 308c8ca33d0SKevin Wolf if (bs) { 309aa1361d5SKevin Wolf AioContext *ctx = bdrv_get_aio_context(bs); 310aa1361d5SKevin Wolf AioContext *co_ctx = qemu_coroutine_get_aio_context(co); 311aa1361d5SKevin Wolf 312aa1361d5SKevin Wolf /* 313aa1361d5SKevin Wolf * When the coroutine yielded, the lock for its home context was 314aa1361d5SKevin Wolf * released, so we need to re-acquire it here. If it explicitly 315aa1361d5SKevin Wolf * acquired a different context, the lock is still held and we don't 316aa1361d5SKevin Wolf * want to lock it a second time (or AIO_WAIT_WHILE() would hang). 317aa1361d5SKevin Wolf */ 318aa1361d5SKevin Wolf if (ctx == co_ctx) { 319aa1361d5SKevin Wolf aio_context_acquire(ctx); 320aa1361d5SKevin Wolf } 32199723548SPaolo Bonzini bdrv_dec_in_flight(bs); 322481cad48SManos Pitsidianakis if (data->begin) { 3236cd5c9d7SKevin Wolf bdrv_do_drained_begin(bs, data->recursive, data->parent, 3246cd5c9d7SKevin Wolf data->ignore_bds_parents, data->poll); 325481cad48SManos Pitsidianakis } else { 3266cd5c9d7SKevin Wolf bdrv_do_drained_end(bs, data->recursive, data->parent, 327*8e1da77eSMax Reitz data->ignore_bds_parents, 328*8e1da77eSMax Reitz data->drained_end_counter); 329481cad48SManos Pitsidianakis } 330aa1361d5SKevin Wolf if (ctx == co_ctx) { 331aa1361d5SKevin Wolf aio_context_release(ctx); 332aa1361d5SKevin Wolf } 333c8ca33d0SKevin Wolf } else { 334c8ca33d0SKevin Wolf assert(data->begin); 335c8ca33d0SKevin Wolf bdrv_drain_all_begin(); 336c8ca33d0SKevin Wolf } 337481cad48SManos Pitsidianakis 338a77fd4bbSFam Zheng data->done = true; 3391919631eSPaolo Bonzini aio_co_wake(co); 340a77fd4bbSFam Zheng } 341a77fd4bbSFam Zheng 342481cad48SManos Pitsidianakis static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs, 343b0165585SKevin Wolf bool begin, bool recursive, 3446cd5c9d7SKevin Wolf BdrvChild *parent, 3456cd5c9d7SKevin Wolf bool ignore_bds_parents, 346*8e1da77eSMax Reitz bool poll, 347*8e1da77eSMax Reitz int *drained_end_counter) 348a77fd4bbSFam Zheng { 349a77fd4bbSFam Zheng BdrvCoDrainData data; 350a77fd4bbSFam Zheng 351a77fd4bbSFam Zheng /* Calling bdrv_drain() from a BH ensures the current coroutine yields and 352c40a2545SStefan Hajnoczi * other coroutines run if they were queued by aio_co_enter(). */ 353a77fd4bbSFam Zheng 354a77fd4bbSFam Zheng assert(qemu_in_coroutine()); 355a77fd4bbSFam Zheng data = (BdrvCoDrainData) { 356a77fd4bbSFam Zheng .co = qemu_coroutine_self(), 357a77fd4bbSFam Zheng .bs = bs, 358a77fd4bbSFam Zheng .done = false, 359481cad48SManos Pitsidianakis .begin = begin, 360b0165585SKevin Wolf .recursive = recursive, 3610152bf40SKevin Wolf .parent = parent, 3626cd5c9d7SKevin Wolf .ignore_bds_parents = ignore_bds_parents, 363fe4f0614SKevin Wolf .poll = poll, 364*8e1da77eSMax Reitz .drained_end_counter = drained_end_counter, 365a77fd4bbSFam Zheng }; 366*8e1da77eSMax Reitz 367c8ca33d0SKevin Wolf if (bs) { 36899723548SPaolo Bonzini bdrv_inc_in_flight(bs); 369c8ca33d0SKevin Wolf } 370fffb6e12SPaolo Bonzini aio_bh_schedule_oneshot(bdrv_get_aio_context(bs), 371fffb6e12SPaolo Bonzini bdrv_co_drain_bh_cb, &data); 372a77fd4bbSFam Zheng 373a77fd4bbSFam Zheng qemu_coroutine_yield(); 374a77fd4bbSFam Zheng /* If we are resumed from some other event (such as an aio completion or a 375a77fd4bbSFam Zheng * timer callback), it is a bug in the caller that should be fixed. */ 376a77fd4bbSFam Zheng assert(data.done); 377a77fd4bbSFam Zheng } 378a77fd4bbSFam Zheng 379dcf94a23SKevin Wolf void bdrv_do_drained_begin_quiesce(BlockDriverState *bs, 3806cd5c9d7SKevin Wolf BdrvChild *parent, bool ignore_bds_parents) 381dcf94a23SKevin Wolf { 382dcf94a23SKevin Wolf assert(!qemu_in_coroutine()); 383dcf94a23SKevin Wolf 384dcf94a23SKevin Wolf /* Stop things in parent-to-child order */ 385dcf94a23SKevin Wolf if (atomic_fetch_inc(&bs->quiesce_counter) == 0) { 386dcf94a23SKevin Wolf aio_disable_external(bdrv_get_aio_context(bs)); 387dcf94a23SKevin Wolf } 388dcf94a23SKevin Wolf 3896cd5c9d7SKevin Wolf bdrv_parent_drained_begin(bs, parent, ignore_bds_parents); 390*8e1da77eSMax Reitz bdrv_drain_invoke(bs, true, NULL); 391dcf94a23SKevin Wolf } 392dcf94a23SKevin Wolf 393dcf94a23SKevin Wolf static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive, 3946cd5c9d7SKevin Wolf BdrvChild *parent, bool ignore_bds_parents, 3956cd5c9d7SKevin Wolf bool poll) 3966820643fSKevin Wolf { 397b0165585SKevin Wolf BdrvChild *child, *next; 398b0165585SKevin Wolf 399d42cf288SPaolo Bonzini if (qemu_in_coroutine()) { 4006cd5c9d7SKevin Wolf bdrv_co_yield_to_drain(bs, true, recursive, parent, ignore_bds_parents, 401*8e1da77eSMax Reitz poll, NULL); 402d42cf288SPaolo Bonzini return; 403d42cf288SPaolo Bonzini } 404d42cf288SPaolo Bonzini 4056cd5c9d7SKevin Wolf bdrv_do_drained_begin_quiesce(bs, parent, ignore_bds_parents); 406d30b8e64SKevin Wolf 407b0165585SKevin Wolf if (recursive) { 4086cd5c9d7SKevin Wolf assert(!ignore_bds_parents); 409d736f119SKevin Wolf bs->recursive_quiesce_counter++; 410b0165585SKevin Wolf QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 4116cd5c9d7SKevin Wolf bdrv_do_drained_begin(child->bs, true, child, ignore_bds_parents, 4126cd5c9d7SKevin Wolf false); 413b0165585SKevin Wolf } 414b0165585SKevin Wolf } 415fe4f0614SKevin Wolf 416fe4f0614SKevin Wolf /* 417fe4f0614SKevin Wolf * Wait for drained requests to finish. 418fe4f0614SKevin Wolf * 419fe4f0614SKevin Wolf * Calling BDRV_POLL_WHILE() only once for the top-level node is okay: The 420fe4f0614SKevin Wolf * call is needed so things in this AioContext can make progress even 421fe4f0614SKevin Wolf * though we don't return to the main AioContext loop - this automatically 422fe4f0614SKevin Wolf * includes other nodes in the same AioContext and therefore all child 423fe4f0614SKevin Wolf * nodes. 424fe4f0614SKevin Wolf */ 425fe4f0614SKevin Wolf if (poll) { 4266cd5c9d7SKevin Wolf assert(!ignore_bds_parents); 427fe4f0614SKevin Wolf BDRV_POLL_WHILE(bs, bdrv_drain_poll_top_level(bs, recursive, parent)); 428fe4f0614SKevin Wolf } 4296820643fSKevin Wolf } 4306820643fSKevin Wolf 4310152bf40SKevin Wolf void bdrv_drained_begin(BlockDriverState *bs) 4320152bf40SKevin Wolf { 4336cd5c9d7SKevin Wolf bdrv_do_drained_begin(bs, false, NULL, false, true); 4340152bf40SKevin Wolf } 4350152bf40SKevin Wolf 436b0165585SKevin Wolf void bdrv_subtree_drained_begin(BlockDriverState *bs) 4376820643fSKevin Wolf { 4386cd5c9d7SKevin Wolf bdrv_do_drained_begin(bs, true, NULL, false, true); 439b0165585SKevin Wolf } 440b0165585SKevin Wolf 4416cd5c9d7SKevin Wolf static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive, 442*8e1da77eSMax Reitz BdrvChild *parent, bool ignore_bds_parents, 443*8e1da77eSMax Reitz int *drained_end_counter) 444b0165585SKevin Wolf { 445b0165585SKevin Wolf BdrvChild *child, *next; 4460f115168SKevin Wolf int old_quiesce_counter; 4470f115168SKevin Wolf 448481cad48SManos Pitsidianakis if (qemu_in_coroutine()) { 4496cd5c9d7SKevin Wolf bdrv_co_yield_to_drain(bs, false, recursive, parent, ignore_bds_parents, 450*8e1da77eSMax Reitz false, drained_end_counter); 451481cad48SManos Pitsidianakis return; 452481cad48SManos Pitsidianakis } 4536820643fSKevin Wolf assert(bs->quiesce_counter > 0); 4546820643fSKevin Wolf 45560369b86SKevin Wolf /* Re-enable things in child-to-parent order */ 456*8e1da77eSMax Reitz bdrv_drain_invoke(bs, false, drained_end_counter); 4576cd5c9d7SKevin Wolf bdrv_parent_drained_end(bs, parent, ignore_bds_parents); 4585cb2737eSMax Reitz 4595cb2737eSMax Reitz old_quiesce_counter = atomic_fetch_dec(&bs->quiesce_counter); 4600f115168SKevin Wolf if (old_quiesce_counter == 1) { 4616820643fSKevin Wolf aio_enable_external(bdrv_get_aio_context(bs)); 4626820643fSKevin Wolf } 463b0165585SKevin Wolf 464b0165585SKevin Wolf if (recursive) { 4656cd5c9d7SKevin Wolf assert(!ignore_bds_parents); 466d736f119SKevin Wolf bs->recursive_quiesce_counter--; 467b0165585SKevin Wolf QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 468*8e1da77eSMax Reitz bdrv_do_drained_end(child->bs, true, child, ignore_bds_parents, 469*8e1da77eSMax Reitz drained_end_counter); 470b0165585SKevin Wolf } 471b0165585SKevin Wolf } 4720f115168SKevin Wolf } 4736820643fSKevin Wolf 4740152bf40SKevin Wolf void bdrv_drained_end(BlockDriverState *bs) 4750152bf40SKevin Wolf { 476*8e1da77eSMax Reitz bdrv_do_drained_end(bs, false, NULL, false, NULL); 477b0165585SKevin Wolf } 478b0165585SKevin Wolf 479b0165585SKevin Wolf void bdrv_subtree_drained_end(BlockDriverState *bs) 480b0165585SKevin Wolf { 481*8e1da77eSMax Reitz bdrv_do_drained_end(bs, true, NULL, false, NULL); 4820152bf40SKevin Wolf } 4830152bf40SKevin Wolf 484d736f119SKevin Wolf void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent) 485d736f119SKevin Wolf { 486d736f119SKevin Wolf int i; 487d736f119SKevin Wolf 488d736f119SKevin Wolf for (i = 0; i < new_parent->recursive_quiesce_counter; i++) { 4896cd5c9d7SKevin Wolf bdrv_do_drained_begin(child->bs, true, child, false, true); 490d736f119SKevin Wolf } 491d736f119SKevin Wolf } 492d736f119SKevin Wolf 493d736f119SKevin Wolf void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent) 494d736f119SKevin Wolf { 495d736f119SKevin Wolf int i; 496d736f119SKevin Wolf 497d736f119SKevin Wolf for (i = 0; i < old_parent->recursive_quiesce_counter; i++) { 498*8e1da77eSMax Reitz bdrv_do_drained_end(child->bs, true, child, false, NULL); 499d736f119SKevin Wolf } 500d736f119SKevin Wolf } 501d736f119SKevin Wolf 50261007b31SStefan Hajnoczi /* 50367da1dc5SFam Zheng * Wait for pending requests to complete on a single BlockDriverState subtree, 50467da1dc5SFam Zheng * and suspend block driver's internal I/O until next request arrives. 50561007b31SStefan Hajnoczi * 50661007b31SStefan Hajnoczi * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState 50761007b31SStefan Hajnoczi * AioContext. 50861007b31SStefan Hajnoczi */ 509b6e84c97SPaolo Bonzini void coroutine_fn bdrv_co_drain(BlockDriverState *bs) 510b6e84c97SPaolo Bonzini { 5116820643fSKevin Wolf assert(qemu_in_coroutine()); 5126820643fSKevin Wolf bdrv_drained_begin(bs); 5136820643fSKevin Wolf bdrv_drained_end(bs); 514b6e84c97SPaolo Bonzini } 515b6e84c97SPaolo Bonzini 51661007b31SStefan Hajnoczi void bdrv_drain(BlockDriverState *bs) 51761007b31SStefan Hajnoczi { 5186820643fSKevin Wolf bdrv_drained_begin(bs); 5196820643fSKevin Wolf bdrv_drained_end(bs); 52061007b31SStefan Hajnoczi } 52161007b31SStefan Hajnoczi 522c13ad59fSKevin Wolf static void bdrv_drain_assert_idle(BlockDriverState *bs) 523c13ad59fSKevin Wolf { 524c13ad59fSKevin Wolf BdrvChild *child, *next; 525c13ad59fSKevin Wolf 526c13ad59fSKevin Wolf assert(atomic_read(&bs->in_flight) == 0); 527c13ad59fSKevin Wolf QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 528c13ad59fSKevin Wolf bdrv_drain_assert_idle(child->bs); 529c13ad59fSKevin Wolf } 530c13ad59fSKevin Wolf } 531c13ad59fSKevin Wolf 5320f12264eSKevin Wolf unsigned int bdrv_drain_all_count = 0; 5330f12264eSKevin Wolf 5340f12264eSKevin Wolf static bool bdrv_drain_all_poll(void) 5350f12264eSKevin Wolf { 5360f12264eSKevin Wolf BlockDriverState *bs = NULL; 5370f12264eSKevin Wolf bool result = false; 5380f12264eSKevin Wolf 5390f12264eSKevin Wolf /* bdrv_drain_poll() can't make changes to the graph and we are holding the 5400f12264eSKevin Wolf * main AioContext lock, so iterating bdrv_next_all_states() is safe. */ 5410f12264eSKevin Wolf while ((bs = bdrv_next_all_states(bs))) { 5420f12264eSKevin Wolf AioContext *aio_context = bdrv_get_aio_context(bs); 5430f12264eSKevin Wolf aio_context_acquire(aio_context); 5440f12264eSKevin Wolf result |= bdrv_drain_poll(bs, false, NULL, true); 5450f12264eSKevin Wolf aio_context_release(aio_context); 5460f12264eSKevin Wolf } 5470f12264eSKevin Wolf 5480f12264eSKevin Wolf return result; 5490f12264eSKevin Wolf } 5500f12264eSKevin Wolf 55161007b31SStefan Hajnoczi /* 55261007b31SStefan Hajnoczi * Wait for pending requests to complete across all BlockDriverStates 55361007b31SStefan Hajnoczi * 55461007b31SStefan Hajnoczi * This function does not flush data to disk, use bdrv_flush_all() for that 55561007b31SStefan Hajnoczi * after calling this function. 556c0778f66SAlberto Garcia * 557c0778f66SAlberto Garcia * This pauses all block jobs and disables external clients. It must 558c0778f66SAlberto Garcia * be paired with bdrv_drain_all_end(). 559c0778f66SAlberto Garcia * 560c0778f66SAlberto Garcia * NOTE: no new block jobs or BlockDriverStates can be created between 561c0778f66SAlberto Garcia * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls. 56261007b31SStefan Hajnoczi */ 563c0778f66SAlberto Garcia void bdrv_drain_all_begin(void) 56461007b31SStefan Hajnoczi { 5650f12264eSKevin Wolf BlockDriverState *bs = NULL; 56661007b31SStefan Hajnoczi 567c8ca33d0SKevin Wolf if (qemu_in_coroutine()) { 568*8e1da77eSMax Reitz bdrv_co_yield_to_drain(NULL, true, false, NULL, true, true, NULL); 569c8ca33d0SKevin Wolf return; 570c8ca33d0SKevin Wolf } 571c8ca33d0SKevin Wolf 5720f12264eSKevin Wolf /* AIO_WAIT_WHILE() with a NULL context can only be called from the main 5730f12264eSKevin Wolf * loop AioContext, so make sure we're in the main context. */ 5749a7e86c8SKevin Wolf assert(qemu_get_current_aio_context() == qemu_get_aio_context()); 5750f12264eSKevin Wolf assert(bdrv_drain_all_count < INT_MAX); 5760f12264eSKevin Wolf bdrv_drain_all_count++; 5779a7e86c8SKevin Wolf 5780f12264eSKevin Wolf /* Quiesce all nodes, without polling in-flight requests yet. The graph 5790f12264eSKevin Wolf * cannot change during this loop. */ 5800f12264eSKevin Wolf while ((bs = bdrv_next_all_states(bs))) { 58161007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 58261007b31SStefan Hajnoczi 58361007b31SStefan Hajnoczi aio_context_acquire(aio_context); 5840f12264eSKevin Wolf bdrv_do_drained_begin(bs, false, NULL, true, false); 58561007b31SStefan Hajnoczi aio_context_release(aio_context); 58661007b31SStefan Hajnoczi } 58761007b31SStefan Hajnoczi 5880f12264eSKevin Wolf /* Now poll the in-flight requests */ 589cfe29d82SKevin Wolf AIO_WAIT_WHILE(NULL, bdrv_drain_all_poll()); 5900f12264eSKevin Wolf 5910f12264eSKevin Wolf while ((bs = bdrv_next_all_states(bs))) { 592c13ad59fSKevin Wolf bdrv_drain_assert_idle(bs); 593f406c03cSAlexander Yarygin } 594f406c03cSAlexander Yarygin } 595c0778f66SAlberto Garcia 596c0778f66SAlberto Garcia void bdrv_drain_all_end(void) 597c0778f66SAlberto Garcia { 5980f12264eSKevin Wolf BlockDriverState *bs = NULL; 599c0778f66SAlberto Garcia 6000f12264eSKevin Wolf while ((bs = bdrv_next_all_states(bs))) { 60161007b31SStefan Hajnoczi AioContext *aio_context = bdrv_get_aio_context(bs); 60261007b31SStefan Hajnoczi 60361007b31SStefan Hajnoczi aio_context_acquire(aio_context); 604*8e1da77eSMax Reitz bdrv_do_drained_end(bs, false, NULL, true, NULL); 60561007b31SStefan Hajnoczi aio_context_release(aio_context); 60661007b31SStefan Hajnoczi } 6070f12264eSKevin Wolf 6080f12264eSKevin Wolf assert(bdrv_drain_all_count > 0); 6090f12264eSKevin Wolf bdrv_drain_all_count--; 61061007b31SStefan Hajnoczi } 61161007b31SStefan Hajnoczi 612c0778f66SAlberto Garcia void bdrv_drain_all(void) 613c0778f66SAlberto Garcia { 614c0778f66SAlberto Garcia bdrv_drain_all_begin(); 615c0778f66SAlberto Garcia bdrv_drain_all_end(); 616c0778f66SAlberto Garcia } 617c0778f66SAlberto Garcia 61861007b31SStefan Hajnoczi /** 61961007b31SStefan Hajnoczi * Remove an active request from the tracked requests list 62061007b31SStefan Hajnoczi * 62161007b31SStefan Hajnoczi * This function should be called when a tracked request is completing. 62261007b31SStefan Hajnoczi */ 62361007b31SStefan Hajnoczi static void tracked_request_end(BdrvTrackedRequest *req) 62461007b31SStefan Hajnoczi { 62561007b31SStefan Hajnoczi if (req->serialising) { 62620fc71b2SPaolo Bonzini atomic_dec(&req->bs->serialising_in_flight); 62761007b31SStefan Hajnoczi } 62861007b31SStefan Hajnoczi 6293783fa3dSPaolo Bonzini qemu_co_mutex_lock(&req->bs->reqs_lock); 63061007b31SStefan Hajnoczi QLIST_REMOVE(req, list); 63161007b31SStefan Hajnoczi qemu_co_queue_restart_all(&req->wait_queue); 6323783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&req->bs->reqs_lock); 63361007b31SStefan Hajnoczi } 63461007b31SStefan Hajnoczi 63561007b31SStefan Hajnoczi /** 63661007b31SStefan Hajnoczi * Add an active request to the tracked requests list 63761007b31SStefan Hajnoczi */ 63861007b31SStefan Hajnoczi static void tracked_request_begin(BdrvTrackedRequest *req, 63961007b31SStefan Hajnoczi BlockDriverState *bs, 64061007b31SStefan Hajnoczi int64_t offset, 64122931a15SFam Zheng uint64_t bytes, 642ebde595cSFam Zheng enum BdrvTrackedRequestType type) 64361007b31SStefan Hajnoczi { 64422931a15SFam Zheng assert(bytes <= INT64_MAX && offset <= INT64_MAX - bytes); 64522931a15SFam Zheng 64661007b31SStefan Hajnoczi *req = (BdrvTrackedRequest){ 64761007b31SStefan Hajnoczi .bs = bs, 64861007b31SStefan Hajnoczi .offset = offset, 64961007b31SStefan Hajnoczi .bytes = bytes, 650ebde595cSFam Zheng .type = type, 65161007b31SStefan Hajnoczi .co = qemu_coroutine_self(), 65261007b31SStefan Hajnoczi .serialising = false, 65361007b31SStefan Hajnoczi .overlap_offset = offset, 65461007b31SStefan Hajnoczi .overlap_bytes = bytes, 65561007b31SStefan Hajnoczi }; 65661007b31SStefan Hajnoczi 65761007b31SStefan Hajnoczi qemu_co_queue_init(&req->wait_queue); 65861007b31SStefan Hajnoczi 6593783fa3dSPaolo Bonzini qemu_co_mutex_lock(&bs->reqs_lock); 66061007b31SStefan Hajnoczi QLIST_INSERT_HEAD(&bs->tracked_requests, req, list); 6613783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&bs->reqs_lock); 66261007b31SStefan Hajnoczi } 66361007b31SStefan Hajnoczi 66461007b31SStefan Hajnoczi static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align) 66561007b31SStefan Hajnoczi { 66661007b31SStefan Hajnoczi int64_t overlap_offset = req->offset & ~(align - 1); 66722931a15SFam Zheng uint64_t overlap_bytes = ROUND_UP(req->offset + req->bytes, align) 66861007b31SStefan Hajnoczi - overlap_offset; 66961007b31SStefan Hajnoczi 67061007b31SStefan Hajnoczi if (!req->serialising) { 67120fc71b2SPaolo Bonzini atomic_inc(&req->bs->serialising_in_flight); 67261007b31SStefan Hajnoczi req->serialising = true; 67361007b31SStefan Hajnoczi } 67461007b31SStefan Hajnoczi 67561007b31SStefan Hajnoczi req->overlap_offset = MIN(req->overlap_offset, overlap_offset); 67661007b31SStefan Hajnoczi req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes); 67761007b31SStefan Hajnoczi } 67861007b31SStefan Hajnoczi 67909d2f948SVladimir Sementsov-Ogievskiy static bool is_request_serialising_and_aligned(BdrvTrackedRequest *req) 68009d2f948SVladimir Sementsov-Ogievskiy { 68109d2f948SVladimir Sementsov-Ogievskiy /* 68209d2f948SVladimir Sementsov-Ogievskiy * If the request is serialising, overlap_offset and overlap_bytes are set, 68309d2f948SVladimir Sementsov-Ogievskiy * so we can check if the request is aligned. Otherwise, don't care and 68409d2f948SVladimir Sementsov-Ogievskiy * return false. 68509d2f948SVladimir Sementsov-Ogievskiy */ 68609d2f948SVladimir Sementsov-Ogievskiy 68709d2f948SVladimir Sementsov-Ogievskiy return req->serialising && (req->offset == req->overlap_offset) && 68809d2f948SVladimir Sementsov-Ogievskiy (req->bytes == req->overlap_bytes); 68909d2f948SVladimir Sementsov-Ogievskiy } 69009d2f948SVladimir Sementsov-Ogievskiy 69161007b31SStefan Hajnoczi /** 692244483e6SKevin Wolf * Round a region to cluster boundaries 693244483e6SKevin Wolf */ 694244483e6SKevin Wolf void bdrv_round_to_clusters(BlockDriverState *bs, 6957cfd5275SEric Blake int64_t offset, int64_t bytes, 696244483e6SKevin Wolf int64_t *cluster_offset, 6977cfd5275SEric Blake int64_t *cluster_bytes) 698244483e6SKevin Wolf { 699244483e6SKevin Wolf BlockDriverInfo bdi; 700244483e6SKevin Wolf 701244483e6SKevin Wolf if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 702244483e6SKevin Wolf *cluster_offset = offset; 703244483e6SKevin Wolf *cluster_bytes = bytes; 704244483e6SKevin Wolf } else { 705244483e6SKevin Wolf int64_t c = bdi.cluster_size; 706244483e6SKevin Wolf *cluster_offset = QEMU_ALIGN_DOWN(offset, c); 707244483e6SKevin Wolf *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c); 708244483e6SKevin Wolf } 709244483e6SKevin Wolf } 710244483e6SKevin Wolf 71161007b31SStefan Hajnoczi static int bdrv_get_cluster_size(BlockDriverState *bs) 71261007b31SStefan Hajnoczi { 71361007b31SStefan Hajnoczi BlockDriverInfo bdi; 71461007b31SStefan Hajnoczi int ret; 71561007b31SStefan Hajnoczi 71661007b31SStefan Hajnoczi ret = bdrv_get_info(bs, &bdi); 71761007b31SStefan Hajnoczi if (ret < 0 || bdi.cluster_size == 0) { 718a5b8dd2cSEric Blake return bs->bl.request_alignment; 71961007b31SStefan Hajnoczi } else { 72061007b31SStefan Hajnoczi return bdi.cluster_size; 72161007b31SStefan Hajnoczi } 72261007b31SStefan Hajnoczi } 72361007b31SStefan Hajnoczi 72461007b31SStefan Hajnoczi static bool tracked_request_overlaps(BdrvTrackedRequest *req, 72522931a15SFam Zheng int64_t offset, uint64_t bytes) 72661007b31SStefan Hajnoczi { 72761007b31SStefan Hajnoczi /* aaaa bbbb */ 72861007b31SStefan Hajnoczi if (offset >= req->overlap_offset + req->overlap_bytes) { 72961007b31SStefan Hajnoczi return false; 73061007b31SStefan Hajnoczi } 73161007b31SStefan Hajnoczi /* bbbb aaaa */ 73261007b31SStefan Hajnoczi if (req->overlap_offset >= offset + bytes) { 73361007b31SStefan Hajnoczi return false; 73461007b31SStefan Hajnoczi } 73561007b31SStefan Hajnoczi return true; 73661007b31SStefan Hajnoczi } 73761007b31SStefan Hajnoczi 73899723548SPaolo Bonzini void bdrv_inc_in_flight(BlockDriverState *bs) 73999723548SPaolo Bonzini { 74099723548SPaolo Bonzini atomic_inc(&bs->in_flight); 74199723548SPaolo Bonzini } 74299723548SPaolo Bonzini 743c9d1a561SPaolo Bonzini void bdrv_wakeup(BlockDriverState *bs) 744c9d1a561SPaolo Bonzini { 745cfe29d82SKevin Wolf aio_wait_kick(); 746c9d1a561SPaolo Bonzini } 747c9d1a561SPaolo Bonzini 74899723548SPaolo Bonzini void bdrv_dec_in_flight(BlockDriverState *bs) 74999723548SPaolo Bonzini { 75099723548SPaolo Bonzini atomic_dec(&bs->in_flight); 751c9d1a561SPaolo Bonzini bdrv_wakeup(bs); 75299723548SPaolo Bonzini } 75399723548SPaolo Bonzini 75461007b31SStefan Hajnoczi static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self) 75561007b31SStefan Hajnoczi { 75661007b31SStefan Hajnoczi BlockDriverState *bs = self->bs; 75761007b31SStefan Hajnoczi BdrvTrackedRequest *req; 75861007b31SStefan Hajnoczi bool retry; 75961007b31SStefan Hajnoczi bool waited = false; 76061007b31SStefan Hajnoczi 76120fc71b2SPaolo Bonzini if (!atomic_read(&bs->serialising_in_flight)) { 76261007b31SStefan Hajnoczi return false; 76361007b31SStefan Hajnoczi } 76461007b31SStefan Hajnoczi 76561007b31SStefan Hajnoczi do { 76661007b31SStefan Hajnoczi retry = false; 7673783fa3dSPaolo Bonzini qemu_co_mutex_lock(&bs->reqs_lock); 76861007b31SStefan Hajnoczi QLIST_FOREACH(req, &bs->tracked_requests, list) { 76961007b31SStefan Hajnoczi if (req == self || (!req->serialising && !self->serialising)) { 77061007b31SStefan Hajnoczi continue; 77161007b31SStefan Hajnoczi } 77261007b31SStefan Hajnoczi if (tracked_request_overlaps(req, self->overlap_offset, 77361007b31SStefan Hajnoczi self->overlap_bytes)) 77461007b31SStefan Hajnoczi { 77561007b31SStefan Hajnoczi /* Hitting this means there was a reentrant request, for 77661007b31SStefan Hajnoczi * example, a block driver issuing nested requests. This must 77761007b31SStefan Hajnoczi * never happen since it means deadlock. 77861007b31SStefan Hajnoczi */ 77961007b31SStefan Hajnoczi assert(qemu_coroutine_self() != req->co); 78061007b31SStefan Hajnoczi 78161007b31SStefan Hajnoczi /* If the request is already (indirectly) waiting for us, or 78261007b31SStefan Hajnoczi * will wait for us as soon as it wakes up, then just go on 78361007b31SStefan Hajnoczi * (instead of producing a deadlock in the former case). */ 78461007b31SStefan Hajnoczi if (!req->waiting_for) { 78561007b31SStefan Hajnoczi self->waiting_for = req; 7863783fa3dSPaolo Bonzini qemu_co_queue_wait(&req->wait_queue, &bs->reqs_lock); 78761007b31SStefan Hajnoczi self->waiting_for = NULL; 78861007b31SStefan Hajnoczi retry = true; 78961007b31SStefan Hajnoczi waited = true; 79061007b31SStefan Hajnoczi break; 79161007b31SStefan Hajnoczi } 79261007b31SStefan Hajnoczi } 79361007b31SStefan Hajnoczi } 7943783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&bs->reqs_lock); 79561007b31SStefan Hajnoczi } while (retry); 79661007b31SStefan Hajnoczi 79761007b31SStefan Hajnoczi return waited; 79861007b31SStefan Hajnoczi } 79961007b31SStefan Hajnoczi 80061007b31SStefan Hajnoczi static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset, 80161007b31SStefan Hajnoczi size_t size) 80261007b31SStefan Hajnoczi { 80341ae31e3SAlberto Garcia if (size > BDRV_REQUEST_MAX_BYTES) { 80461007b31SStefan Hajnoczi return -EIO; 80561007b31SStefan Hajnoczi } 80661007b31SStefan Hajnoczi 80761007b31SStefan Hajnoczi if (!bdrv_is_inserted(bs)) { 80861007b31SStefan Hajnoczi return -ENOMEDIUM; 80961007b31SStefan Hajnoczi } 81061007b31SStefan Hajnoczi 81161007b31SStefan Hajnoczi if (offset < 0) { 81261007b31SStefan Hajnoczi return -EIO; 81361007b31SStefan Hajnoczi } 81461007b31SStefan Hajnoczi 81561007b31SStefan Hajnoczi return 0; 81661007b31SStefan Hajnoczi } 81761007b31SStefan Hajnoczi 81861007b31SStefan Hajnoczi typedef struct RwCo { 819e293b7a3SKevin Wolf BdrvChild *child; 82061007b31SStefan Hajnoczi int64_t offset; 82161007b31SStefan Hajnoczi QEMUIOVector *qiov; 82261007b31SStefan Hajnoczi bool is_write; 82361007b31SStefan Hajnoczi int ret; 82461007b31SStefan Hajnoczi BdrvRequestFlags flags; 82561007b31SStefan Hajnoczi } RwCo; 82661007b31SStefan Hajnoczi 82761007b31SStefan Hajnoczi static void coroutine_fn bdrv_rw_co_entry(void *opaque) 82861007b31SStefan Hajnoczi { 82961007b31SStefan Hajnoczi RwCo *rwco = opaque; 83061007b31SStefan Hajnoczi 83161007b31SStefan Hajnoczi if (!rwco->is_write) { 832a03ef88fSKevin Wolf rwco->ret = bdrv_co_preadv(rwco->child, rwco->offset, 83361007b31SStefan Hajnoczi rwco->qiov->size, rwco->qiov, 83461007b31SStefan Hajnoczi rwco->flags); 83561007b31SStefan Hajnoczi } else { 836a03ef88fSKevin Wolf rwco->ret = bdrv_co_pwritev(rwco->child, rwco->offset, 83761007b31SStefan Hajnoczi rwco->qiov->size, rwco->qiov, 83861007b31SStefan Hajnoczi rwco->flags); 83961007b31SStefan Hajnoczi } 8404720cbeeSKevin Wolf aio_wait_kick(); 84161007b31SStefan Hajnoczi } 84261007b31SStefan Hajnoczi 84361007b31SStefan Hajnoczi /* 84461007b31SStefan Hajnoczi * Process a vectored synchronous request using coroutines 84561007b31SStefan Hajnoczi */ 846e293b7a3SKevin Wolf static int bdrv_prwv_co(BdrvChild *child, int64_t offset, 84761007b31SStefan Hajnoczi QEMUIOVector *qiov, bool is_write, 84861007b31SStefan Hajnoczi BdrvRequestFlags flags) 84961007b31SStefan Hajnoczi { 85061007b31SStefan Hajnoczi Coroutine *co; 85161007b31SStefan Hajnoczi RwCo rwco = { 852e293b7a3SKevin Wolf .child = child, 85361007b31SStefan Hajnoczi .offset = offset, 85461007b31SStefan Hajnoczi .qiov = qiov, 85561007b31SStefan Hajnoczi .is_write = is_write, 85661007b31SStefan Hajnoczi .ret = NOT_DONE, 85761007b31SStefan Hajnoczi .flags = flags, 85861007b31SStefan Hajnoczi }; 85961007b31SStefan Hajnoczi 86061007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 86161007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 86261007b31SStefan Hajnoczi bdrv_rw_co_entry(&rwco); 86361007b31SStefan Hajnoczi } else { 8640b8b8753SPaolo Bonzini co = qemu_coroutine_create(bdrv_rw_co_entry, &rwco); 865e92f0e19SFam Zheng bdrv_coroutine_enter(child->bs, co); 86688b062c2SPaolo Bonzini BDRV_POLL_WHILE(child->bs, rwco.ret == NOT_DONE); 86761007b31SStefan Hajnoczi } 86861007b31SStefan Hajnoczi return rwco.ret; 86961007b31SStefan Hajnoczi } 87061007b31SStefan Hajnoczi 871720ff280SKevin Wolf int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset, 872f5a5ca79SManos Pitsidianakis int bytes, BdrvRequestFlags flags) 87361007b31SStefan Hajnoczi { 8740d93ed08SVladimir Sementsov-Ogievskiy QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, NULL, bytes); 87574021bc4SEric Blake 876e293b7a3SKevin Wolf return bdrv_prwv_co(child, offset, &qiov, true, 87761007b31SStefan Hajnoczi BDRV_REQ_ZERO_WRITE | flags); 87861007b31SStefan Hajnoczi } 87961007b31SStefan Hajnoczi 88061007b31SStefan Hajnoczi /* 88174021bc4SEric Blake * Completely zero out a block device with the help of bdrv_pwrite_zeroes. 88261007b31SStefan Hajnoczi * The operation is sped up by checking the block status and only writing 88361007b31SStefan Hajnoczi * zeroes to the device if they currently do not return zeroes. Optional 88474021bc4SEric Blake * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP, 885465fe887SEric Blake * BDRV_REQ_FUA). 88661007b31SStefan Hajnoczi * 88761007b31SStefan Hajnoczi * Returns < 0 on error, 0 on success. For error codes see bdrv_write(). 88861007b31SStefan Hajnoczi */ 889720ff280SKevin Wolf int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags) 89061007b31SStefan Hajnoczi { 891237d78f8SEric Blake int ret; 892237d78f8SEric Blake int64_t target_size, bytes, offset = 0; 893720ff280SKevin Wolf BlockDriverState *bs = child->bs; 89461007b31SStefan Hajnoczi 8957286d610SEric Blake target_size = bdrv_getlength(bs); 8967286d610SEric Blake if (target_size < 0) { 8977286d610SEric Blake return target_size; 89861007b31SStefan Hajnoczi } 89961007b31SStefan Hajnoczi 90061007b31SStefan Hajnoczi for (;;) { 9017286d610SEric Blake bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES); 9027286d610SEric Blake if (bytes <= 0) { 90361007b31SStefan Hajnoczi return 0; 90461007b31SStefan Hajnoczi } 905237d78f8SEric Blake ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL); 90661007b31SStefan Hajnoczi if (ret < 0) { 90761007b31SStefan Hajnoczi return ret; 90861007b31SStefan Hajnoczi } 90961007b31SStefan Hajnoczi if (ret & BDRV_BLOCK_ZERO) { 910237d78f8SEric Blake offset += bytes; 91161007b31SStefan Hajnoczi continue; 91261007b31SStefan Hajnoczi } 913237d78f8SEric Blake ret = bdrv_pwrite_zeroes(child, offset, bytes, flags); 91461007b31SStefan Hajnoczi if (ret < 0) { 91561007b31SStefan Hajnoczi return ret; 91661007b31SStefan Hajnoczi } 917237d78f8SEric Blake offset += bytes; 91861007b31SStefan Hajnoczi } 91961007b31SStefan Hajnoczi } 92061007b31SStefan Hajnoczi 921cf2ab8fcSKevin Wolf int bdrv_preadv(BdrvChild *child, int64_t offset, QEMUIOVector *qiov) 922f1e84741SKevin Wolf { 923f1e84741SKevin Wolf int ret; 924f1e84741SKevin Wolf 925e293b7a3SKevin Wolf ret = bdrv_prwv_co(child, offset, qiov, false, 0); 926f1e84741SKevin Wolf if (ret < 0) { 927f1e84741SKevin Wolf return ret; 928f1e84741SKevin Wolf } 929f1e84741SKevin Wolf 930f1e84741SKevin Wolf return qiov->size; 931f1e84741SKevin Wolf } 932f1e84741SKevin Wolf 9332e11d756SAlberto Garcia /* See bdrv_pwrite() for the return codes */ 934cf2ab8fcSKevin Wolf int bdrv_pread(BdrvChild *child, int64_t offset, void *buf, int bytes) 93561007b31SStefan Hajnoczi { 9360d93ed08SVladimir Sementsov-Ogievskiy QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes); 93761007b31SStefan Hajnoczi 93861007b31SStefan Hajnoczi if (bytes < 0) { 93961007b31SStefan Hajnoczi return -EINVAL; 94061007b31SStefan Hajnoczi } 94161007b31SStefan Hajnoczi 942cf2ab8fcSKevin Wolf return bdrv_preadv(child, offset, &qiov); 94361007b31SStefan Hajnoczi } 94461007b31SStefan Hajnoczi 945d9ca2ea2SKevin Wolf int bdrv_pwritev(BdrvChild *child, int64_t offset, QEMUIOVector *qiov) 94661007b31SStefan Hajnoczi { 94761007b31SStefan Hajnoczi int ret; 94861007b31SStefan Hajnoczi 949e293b7a3SKevin Wolf ret = bdrv_prwv_co(child, offset, qiov, true, 0); 95061007b31SStefan Hajnoczi if (ret < 0) { 95161007b31SStefan Hajnoczi return ret; 95261007b31SStefan Hajnoczi } 95361007b31SStefan Hajnoczi 95461007b31SStefan Hajnoczi return qiov->size; 95561007b31SStefan Hajnoczi } 95661007b31SStefan Hajnoczi 9572e11d756SAlberto Garcia /* Return no. of bytes on success or < 0 on error. Important errors are: 9582e11d756SAlberto Garcia -EIO generic I/O error (may happen for all errors) 9592e11d756SAlberto Garcia -ENOMEDIUM No media inserted. 9602e11d756SAlberto Garcia -EINVAL Invalid offset or number of bytes 9612e11d756SAlberto Garcia -EACCES Trying to write a read-only device 9622e11d756SAlberto Garcia */ 963d9ca2ea2SKevin Wolf int bdrv_pwrite(BdrvChild *child, int64_t offset, const void *buf, int bytes) 96461007b31SStefan Hajnoczi { 9650d93ed08SVladimir Sementsov-Ogievskiy QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes); 96661007b31SStefan Hajnoczi 96761007b31SStefan Hajnoczi if (bytes < 0) { 96861007b31SStefan Hajnoczi return -EINVAL; 96961007b31SStefan Hajnoczi } 97061007b31SStefan Hajnoczi 971d9ca2ea2SKevin Wolf return bdrv_pwritev(child, offset, &qiov); 97261007b31SStefan Hajnoczi } 97361007b31SStefan Hajnoczi 97461007b31SStefan Hajnoczi /* 97561007b31SStefan Hajnoczi * Writes to the file and ensures that no writes are reordered across this 97661007b31SStefan Hajnoczi * request (acts as a barrier) 97761007b31SStefan Hajnoczi * 97861007b31SStefan Hajnoczi * Returns 0 on success, -errno in error cases. 97961007b31SStefan Hajnoczi */ 980d9ca2ea2SKevin Wolf int bdrv_pwrite_sync(BdrvChild *child, int64_t offset, 98161007b31SStefan Hajnoczi const void *buf, int count) 98261007b31SStefan Hajnoczi { 98361007b31SStefan Hajnoczi int ret; 98461007b31SStefan Hajnoczi 985d9ca2ea2SKevin Wolf ret = bdrv_pwrite(child, offset, buf, count); 98661007b31SStefan Hajnoczi if (ret < 0) { 98761007b31SStefan Hajnoczi return ret; 98861007b31SStefan Hajnoczi } 98961007b31SStefan Hajnoczi 990d9ca2ea2SKevin Wolf ret = bdrv_flush(child->bs); 991855a6a93SKevin Wolf if (ret < 0) { 992855a6a93SKevin Wolf return ret; 99361007b31SStefan Hajnoczi } 99461007b31SStefan Hajnoczi 99561007b31SStefan Hajnoczi return 0; 99661007b31SStefan Hajnoczi } 99761007b31SStefan Hajnoczi 99808844473SKevin Wolf typedef struct CoroutineIOCompletion { 99908844473SKevin Wolf Coroutine *coroutine; 100008844473SKevin Wolf int ret; 100108844473SKevin Wolf } CoroutineIOCompletion; 100208844473SKevin Wolf 100308844473SKevin Wolf static void bdrv_co_io_em_complete(void *opaque, int ret) 100408844473SKevin Wolf { 100508844473SKevin Wolf CoroutineIOCompletion *co = opaque; 100608844473SKevin Wolf 100708844473SKevin Wolf co->ret = ret; 1008b9e413ddSPaolo Bonzini aio_co_wake(co->coroutine); 100908844473SKevin Wolf } 101008844473SKevin Wolf 1011166fe960SKevin Wolf static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs, 1012166fe960SKevin Wolf uint64_t offset, uint64_t bytes, 1013166fe960SKevin Wolf QEMUIOVector *qiov, int flags) 1014166fe960SKevin Wolf { 1015166fe960SKevin Wolf BlockDriver *drv = bs->drv; 10163fb06697SKevin Wolf int64_t sector_num; 10173fb06697SKevin Wolf unsigned int nb_sectors; 10183fb06697SKevin Wolf 1019fa166538SEric Blake assert(!(flags & ~BDRV_REQ_MASK)); 1020fe0480d6SKevin Wolf assert(!(flags & BDRV_REQ_NO_FALLBACK)); 1021fa166538SEric Blake 1022d470ad42SMax Reitz if (!drv) { 1023d470ad42SMax Reitz return -ENOMEDIUM; 1024d470ad42SMax Reitz } 1025d470ad42SMax Reitz 10263fb06697SKevin Wolf if (drv->bdrv_co_preadv) { 10273fb06697SKevin Wolf return drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags); 10283fb06697SKevin Wolf } 10293fb06697SKevin Wolf 1030edfab6a0SEric Blake if (drv->bdrv_aio_preadv) { 103108844473SKevin Wolf BlockAIOCB *acb; 103208844473SKevin Wolf CoroutineIOCompletion co = { 103308844473SKevin Wolf .coroutine = qemu_coroutine_self(), 103408844473SKevin Wolf }; 103508844473SKevin Wolf 1036e31f6864SEric Blake acb = drv->bdrv_aio_preadv(bs, offset, bytes, qiov, flags, 103708844473SKevin Wolf bdrv_co_io_em_complete, &co); 103808844473SKevin Wolf if (acb == NULL) { 103908844473SKevin Wolf return -EIO; 104008844473SKevin Wolf } else { 104108844473SKevin Wolf qemu_coroutine_yield(); 104208844473SKevin Wolf return co.ret; 104308844473SKevin Wolf } 104408844473SKevin Wolf } 1045edfab6a0SEric Blake 1046edfab6a0SEric Blake sector_num = offset >> BDRV_SECTOR_BITS; 1047edfab6a0SEric Blake nb_sectors = bytes >> BDRV_SECTOR_BITS; 1048edfab6a0SEric Blake 1049edfab6a0SEric Blake assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 1050edfab6a0SEric Blake assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 105141ae31e3SAlberto Garcia assert(bytes <= BDRV_REQUEST_MAX_BYTES); 1052edfab6a0SEric Blake assert(drv->bdrv_co_readv); 1053edfab6a0SEric Blake 1054edfab6a0SEric Blake return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov); 1055166fe960SKevin Wolf } 1056166fe960SKevin Wolf 105778a07294SKevin Wolf static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs, 105878a07294SKevin Wolf uint64_t offset, uint64_t bytes, 105978a07294SKevin Wolf QEMUIOVector *qiov, int flags) 106078a07294SKevin Wolf { 106178a07294SKevin Wolf BlockDriver *drv = bs->drv; 10623fb06697SKevin Wolf int64_t sector_num; 10633fb06697SKevin Wolf unsigned int nb_sectors; 106478a07294SKevin Wolf int ret; 106578a07294SKevin Wolf 1066fa166538SEric Blake assert(!(flags & ~BDRV_REQ_MASK)); 1067fe0480d6SKevin Wolf assert(!(flags & BDRV_REQ_NO_FALLBACK)); 1068fa166538SEric Blake 1069d470ad42SMax Reitz if (!drv) { 1070d470ad42SMax Reitz return -ENOMEDIUM; 1071d470ad42SMax Reitz } 1072d470ad42SMax Reitz 10733fb06697SKevin Wolf if (drv->bdrv_co_pwritev) { 1074515c2f43SKevin Wolf ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, 1075515c2f43SKevin Wolf flags & bs->supported_write_flags); 1076515c2f43SKevin Wolf flags &= ~bs->supported_write_flags; 10773fb06697SKevin Wolf goto emulate_flags; 10783fb06697SKevin Wolf } 10793fb06697SKevin Wolf 1080edfab6a0SEric Blake if (drv->bdrv_aio_pwritev) { 108108844473SKevin Wolf BlockAIOCB *acb; 108208844473SKevin Wolf CoroutineIOCompletion co = { 108308844473SKevin Wolf .coroutine = qemu_coroutine_self(), 108408844473SKevin Wolf }; 108508844473SKevin Wolf 1086e31f6864SEric Blake acb = drv->bdrv_aio_pwritev(bs, offset, bytes, qiov, 1087e31f6864SEric Blake flags & bs->supported_write_flags, 108808844473SKevin Wolf bdrv_co_io_em_complete, &co); 1089e31f6864SEric Blake flags &= ~bs->supported_write_flags; 109008844473SKevin Wolf if (acb == NULL) { 10913fb06697SKevin Wolf ret = -EIO; 109208844473SKevin Wolf } else { 109308844473SKevin Wolf qemu_coroutine_yield(); 10943fb06697SKevin Wolf ret = co.ret; 109508844473SKevin Wolf } 1096edfab6a0SEric Blake goto emulate_flags; 1097edfab6a0SEric Blake } 1098edfab6a0SEric Blake 1099edfab6a0SEric Blake sector_num = offset >> BDRV_SECTOR_BITS; 1100edfab6a0SEric Blake nb_sectors = bytes >> BDRV_SECTOR_BITS; 1101edfab6a0SEric Blake 1102edfab6a0SEric Blake assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 1103edfab6a0SEric Blake assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 110441ae31e3SAlberto Garcia assert(bytes <= BDRV_REQUEST_MAX_BYTES); 1105edfab6a0SEric Blake 1106e18a58b4SEric Blake assert(drv->bdrv_co_writev); 1107e18a58b4SEric Blake ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov, 1108edfab6a0SEric Blake flags & bs->supported_write_flags); 1109edfab6a0SEric Blake flags &= ~bs->supported_write_flags; 111078a07294SKevin Wolf 11113fb06697SKevin Wolf emulate_flags: 11124df863f3SEric Blake if (ret == 0 && (flags & BDRV_REQ_FUA)) { 111378a07294SKevin Wolf ret = bdrv_co_flush(bs); 111478a07294SKevin Wolf } 111578a07294SKevin Wolf 111678a07294SKevin Wolf return ret; 111778a07294SKevin Wolf } 111878a07294SKevin Wolf 111929a298afSPavel Butsykin static int coroutine_fn 112029a298afSPavel Butsykin bdrv_driver_pwritev_compressed(BlockDriverState *bs, uint64_t offset, 112129a298afSPavel Butsykin uint64_t bytes, QEMUIOVector *qiov) 112229a298afSPavel Butsykin { 112329a298afSPavel Butsykin BlockDriver *drv = bs->drv; 112429a298afSPavel Butsykin 1125d470ad42SMax Reitz if (!drv) { 1126d470ad42SMax Reitz return -ENOMEDIUM; 1127d470ad42SMax Reitz } 1128d470ad42SMax Reitz 112929a298afSPavel Butsykin if (!drv->bdrv_co_pwritev_compressed) { 113029a298afSPavel Butsykin return -ENOTSUP; 113129a298afSPavel Butsykin } 113229a298afSPavel Butsykin 113329a298afSPavel Butsykin return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov); 113429a298afSPavel Butsykin } 113529a298afSPavel Butsykin 113685c97ca7SKevin Wolf static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child, 1137244483e6SKevin Wolf int64_t offset, unsigned int bytes, QEMUIOVector *qiov) 113861007b31SStefan Hajnoczi { 113985c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 114085c97ca7SKevin Wolf 114161007b31SStefan Hajnoczi /* Perform I/O through a temporary buffer so that users who scribble over 114261007b31SStefan Hajnoczi * their read buffer while the operation is in progress do not end up 114361007b31SStefan Hajnoczi * modifying the image file. This is critical for zero-copy guest I/O 114461007b31SStefan Hajnoczi * where anything might happen inside guest memory. 114561007b31SStefan Hajnoczi */ 114661007b31SStefan Hajnoczi void *bounce_buffer; 114761007b31SStefan Hajnoczi 114861007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 1149cb2e2878SEric Blake QEMUIOVector local_qiov; 1150244483e6SKevin Wolf int64_t cluster_offset; 11517cfd5275SEric Blake int64_t cluster_bytes; 115261007b31SStefan Hajnoczi size_t skip_bytes; 115361007b31SStefan Hajnoczi int ret; 1154cb2e2878SEric Blake int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, 1155cb2e2878SEric Blake BDRV_REQUEST_MAX_BYTES); 1156cb2e2878SEric Blake unsigned int progress = 0; 115761007b31SStefan Hajnoczi 1158d470ad42SMax Reitz if (!drv) { 1159d470ad42SMax Reitz return -ENOMEDIUM; 1160d470ad42SMax Reitz } 1161d470ad42SMax Reitz 11621bf03e66SKevin Wolf /* FIXME We cannot require callers to have write permissions when all they 11631bf03e66SKevin Wolf * are doing is a read request. If we did things right, write permissions 11641bf03e66SKevin Wolf * would be obtained anyway, but internally by the copy-on-read code. As 1165765d9df9SEric Blake * long as it is implemented here rather than in a separate filter driver, 11661bf03e66SKevin Wolf * the copy-on-read code doesn't have its own BdrvChild, however, for which 11671bf03e66SKevin Wolf * it could request permissions. Therefore we have to bypass the permission 11681bf03e66SKevin Wolf * system for the moment. */ 11691bf03e66SKevin Wolf // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE)); 1170afa4b293SKevin Wolf 117161007b31SStefan Hajnoczi /* Cover entire cluster so no additional backing file I/O is required when 1172cb2e2878SEric Blake * allocating cluster in the image file. Note that this value may exceed 1173cb2e2878SEric Blake * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which 1174cb2e2878SEric Blake * is one reason we loop rather than doing it all at once. 117561007b31SStefan Hajnoczi */ 1176244483e6SKevin Wolf bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes); 1177cb2e2878SEric Blake skip_bytes = offset - cluster_offset; 117861007b31SStefan Hajnoczi 1179244483e6SKevin Wolf trace_bdrv_co_do_copy_on_readv(bs, offset, bytes, 1180244483e6SKevin Wolf cluster_offset, cluster_bytes); 118161007b31SStefan Hajnoczi 1182cb2e2878SEric Blake bounce_buffer = qemu_try_blockalign(bs, 1183cb2e2878SEric Blake MIN(MIN(max_transfer, cluster_bytes), 1184cb2e2878SEric Blake MAX_BOUNCE_BUFFER)); 118561007b31SStefan Hajnoczi if (bounce_buffer == NULL) { 118661007b31SStefan Hajnoczi ret = -ENOMEM; 118761007b31SStefan Hajnoczi goto err; 118861007b31SStefan Hajnoczi } 118961007b31SStefan Hajnoczi 1190cb2e2878SEric Blake while (cluster_bytes) { 1191cb2e2878SEric Blake int64_t pnum; 119261007b31SStefan Hajnoczi 1193cb2e2878SEric Blake ret = bdrv_is_allocated(bs, cluster_offset, 1194cb2e2878SEric Blake MIN(cluster_bytes, max_transfer), &pnum); 1195cb2e2878SEric Blake if (ret < 0) { 1196cb2e2878SEric Blake /* Safe to treat errors in querying allocation as if 1197cb2e2878SEric Blake * unallocated; we'll probably fail again soon on the 1198cb2e2878SEric Blake * read, but at least that will set a decent errno. 1199cb2e2878SEric Blake */ 1200cb2e2878SEric Blake pnum = MIN(cluster_bytes, max_transfer); 1201cb2e2878SEric Blake } 1202cb2e2878SEric Blake 1203b0ddcbbbSKevin Wolf /* Stop at EOF if the image ends in the middle of the cluster */ 1204b0ddcbbbSKevin Wolf if (ret == 0 && pnum == 0) { 1205b0ddcbbbSKevin Wolf assert(progress >= bytes); 1206b0ddcbbbSKevin Wolf break; 1207b0ddcbbbSKevin Wolf } 1208b0ddcbbbSKevin Wolf 1209cb2e2878SEric Blake assert(skip_bytes < pnum); 1210cb2e2878SEric Blake 1211cb2e2878SEric Blake if (ret <= 0) { 1212cb2e2878SEric Blake /* Must copy-on-read; use the bounce buffer */ 12130d93ed08SVladimir Sementsov-Ogievskiy pnum = MIN(pnum, MAX_BOUNCE_BUFFER); 12140d93ed08SVladimir Sementsov-Ogievskiy qemu_iovec_init_buf(&local_qiov, bounce_buffer, pnum); 1215cb2e2878SEric Blake 1216cb2e2878SEric Blake ret = bdrv_driver_preadv(bs, cluster_offset, pnum, 1217cb2e2878SEric Blake &local_qiov, 0); 121861007b31SStefan Hajnoczi if (ret < 0) { 121961007b31SStefan Hajnoczi goto err; 122061007b31SStefan Hajnoczi } 122161007b31SStefan Hajnoczi 1222d855ebcdSEric Blake bdrv_debug_event(bs, BLKDBG_COR_WRITE); 1223c1499a5eSEric Blake if (drv->bdrv_co_pwrite_zeroes && 1224cb2e2878SEric Blake buffer_is_zero(bounce_buffer, pnum)) { 1225a604fa2bSEric Blake /* FIXME: Should we (perhaps conditionally) be setting 1226a604fa2bSEric Blake * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy 1227a604fa2bSEric Blake * that still correctly reads as zero? */ 12287adcf59fSMax Reitz ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, pnum, 12297adcf59fSMax Reitz BDRV_REQ_WRITE_UNCHANGED); 123061007b31SStefan Hajnoczi } else { 1231cb2e2878SEric Blake /* This does not change the data on the disk, it is not 1232cb2e2878SEric Blake * necessary to flush even in cache=writethrough mode. 123361007b31SStefan Hajnoczi */ 1234cb2e2878SEric Blake ret = bdrv_driver_pwritev(bs, cluster_offset, pnum, 12357adcf59fSMax Reitz &local_qiov, 12367adcf59fSMax Reitz BDRV_REQ_WRITE_UNCHANGED); 123761007b31SStefan Hajnoczi } 123861007b31SStefan Hajnoczi 123961007b31SStefan Hajnoczi if (ret < 0) { 1240cb2e2878SEric Blake /* It might be okay to ignore write errors for guest 1241cb2e2878SEric Blake * requests. If this is a deliberate copy-on-read 1242cb2e2878SEric Blake * then we don't want to ignore the error. Simply 1243cb2e2878SEric Blake * report it in all cases. 124461007b31SStefan Hajnoczi */ 124561007b31SStefan Hajnoczi goto err; 124661007b31SStefan Hajnoczi } 124761007b31SStefan Hajnoczi 1248cb2e2878SEric Blake qemu_iovec_from_buf(qiov, progress, bounce_buffer + skip_bytes, 1249cb2e2878SEric Blake pnum - skip_bytes); 1250cb2e2878SEric Blake } else { 1251cb2e2878SEric Blake /* Read directly into the destination */ 1252cb2e2878SEric Blake qemu_iovec_init(&local_qiov, qiov->niov); 1253cb2e2878SEric Blake qemu_iovec_concat(&local_qiov, qiov, progress, pnum - skip_bytes); 1254cb2e2878SEric Blake ret = bdrv_driver_preadv(bs, offset + progress, local_qiov.size, 1255cb2e2878SEric Blake &local_qiov, 0); 1256cb2e2878SEric Blake qemu_iovec_destroy(&local_qiov); 1257cb2e2878SEric Blake if (ret < 0) { 1258cb2e2878SEric Blake goto err; 1259cb2e2878SEric Blake } 1260cb2e2878SEric Blake } 1261cb2e2878SEric Blake 1262cb2e2878SEric Blake cluster_offset += pnum; 1263cb2e2878SEric Blake cluster_bytes -= pnum; 1264cb2e2878SEric Blake progress += pnum - skip_bytes; 1265cb2e2878SEric Blake skip_bytes = 0; 1266cb2e2878SEric Blake } 1267cb2e2878SEric Blake ret = 0; 126861007b31SStefan Hajnoczi 126961007b31SStefan Hajnoczi err: 127061007b31SStefan Hajnoczi qemu_vfree(bounce_buffer); 127161007b31SStefan Hajnoczi return ret; 127261007b31SStefan Hajnoczi } 127361007b31SStefan Hajnoczi 127461007b31SStefan Hajnoczi /* 127561007b31SStefan Hajnoczi * Forwards an already correctly aligned request to the BlockDriver. This 12761a62d0acSEric Blake * handles copy on read, zeroing after EOF, and fragmentation of large 12771a62d0acSEric Blake * reads; any other features must be implemented by the caller. 127861007b31SStefan Hajnoczi */ 127985c97ca7SKevin Wolf static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child, 128061007b31SStefan Hajnoczi BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 128161007b31SStefan Hajnoczi int64_t align, QEMUIOVector *qiov, int flags) 128261007b31SStefan Hajnoczi { 128385c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 1284c9d20029SKevin Wolf int64_t total_bytes, max_bytes; 12851a62d0acSEric Blake int ret = 0; 12861a62d0acSEric Blake uint64_t bytes_remaining = bytes; 12871a62d0acSEric Blake int max_transfer; 128861007b31SStefan Hajnoczi 128949c07526SKevin Wolf assert(is_power_of_2(align)); 129049c07526SKevin Wolf assert((offset & (align - 1)) == 0); 129149c07526SKevin Wolf assert((bytes & (align - 1)) == 0); 129261007b31SStefan Hajnoczi assert(!qiov || bytes == qiov->size); 1293abb06c5aSDaniel P. Berrange assert((bs->open_flags & BDRV_O_NO_IO) == 0); 12941a62d0acSEric Blake max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 12951a62d0acSEric Blake align); 1296a604fa2bSEric Blake 1297a604fa2bSEric Blake /* TODO: We would need a per-BDS .supported_read_flags and 1298a604fa2bSEric Blake * potential fallback support, if we ever implement any read flags 1299a604fa2bSEric Blake * to pass through to drivers. For now, there aren't any 1300a604fa2bSEric Blake * passthrough flags. */ 1301a604fa2bSEric Blake assert(!(flags & ~(BDRV_REQ_NO_SERIALISING | BDRV_REQ_COPY_ON_READ))); 130261007b31SStefan Hajnoczi 130361007b31SStefan Hajnoczi /* Handle Copy on Read and associated serialisation */ 130461007b31SStefan Hajnoczi if (flags & BDRV_REQ_COPY_ON_READ) { 130561007b31SStefan Hajnoczi /* If we touch the same cluster it counts as an overlap. This 130661007b31SStefan Hajnoczi * guarantees that allocating writes will be serialized and not race 130761007b31SStefan Hajnoczi * with each other for the same cluster. For example, in copy-on-read 130861007b31SStefan Hajnoczi * it ensures that the CoR read and write operations are atomic and 130961007b31SStefan Hajnoczi * guest writes cannot interleave between them. */ 131061007b31SStefan Hajnoczi mark_request_serialising(req, bdrv_get_cluster_size(bs)); 131161007b31SStefan Hajnoczi } 131261007b31SStefan Hajnoczi 131309d2f948SVladimir Sementsov-Ogievskiy /* BDRV_REQ_SERIALISING is only for write operation */ 131409d2f948SVladimir Sementsov-Ogievskiy assert(!(flags & BDRV_REQ_SERIALISING)); 131509d2f948SVladimir Sementsov-Ogievskiy 131661408b25SFam Zheng if (!(flags & BDRV_REQ_NO_SERIALISING)) { 131761007b31SStefan Hajnoczi wait_serialising_requests(req); 131861408b25SFam Zheng } 131961007b31SStefan Hajnoczi 132061007b31SStefan Hajnoczi if (flags & BDRV_REQ_COPY_ON_READ) { 1321d6a644bbSEric Blake int64_t pnum; 132261007b31SStefan Hajnoczi 132388e63df2SEric Blake ret = bdrv_is_allocated(bs, offset, bytes, &pnum); 132461007b31SStefan Hajnoczi if (ret < 0) { 132561007b31SStefan Hajnoczi goto out; 132661007b31SStefan Hajnoczi } 132761007b31SStefan Hajnoczi 132888e63df2SEric Blake if (!ret || pnum != bytes) { 132985c97ca7SKevin Wolf ret = bdrv_co_do_copy_on_readv(child, offset, bytes, qiov); 133061007b31SStefan Hajnoczi goto out; 133161007b31SStefan Hajnoczi } 133261007b31SStefan Hajnoczi } 133361007b31SStefan Hajnoczi 13341a62d0acSEric Blake /* Forward the request to the BlockDriver, possibly fragmenting it */ 133549c07526SKevin Wolf total_bytes = bdrv_getlength(bs); 133649c07526SKevin Wolf if (total_bytes < 0) { 133749c07526SKevin Wolf ret = total_bytes; 133861007b31SStefan Hajnoczi goto out; 133961007b31SStefan Hajnoczi } 134061007b31SStefan Hajnoczi 134149c07526SKevin Wolf max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align); 13421a62d0acSEric Blake if (bytes <= max_bytes && bytes <= max_transfer) { 1343166fe960SKevin Wolf ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0); 13441a62d0acSEric Blake goto out; 134561007b31SStefan Hajnoczi } 134661007b31SStefan Hajnoczi 13471a62d0acSEric Blake while (bytes_remaining) { 13481a62d0acSEric Blake int num; 13491a62d0acSEric Blake 13501a62d0acSEric Blake if (max_bytes) { 13511a62d0acSEric Blake QEMUIOVector local_qiov; 13521a62d0acSEric Blake 13531a62d0acSEric Blake num = MIN(bytes_remaining, MIN(max_bytes, max_transfer)); 13541a62d0acSEric Blake assert(num); 13551a62d0acSEric Blake qemu_iovec_init(&local_qiov, qiov->niov); 13561a62d0acSEric Blake qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num); 13571a62d0acSEric Blake 13581a62d0acSEric Blake ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining, 13591a62d0acSEric Blake num, &local_qiov, 0); 13601a62d0acSEric Blake max_bytes -= num; 13611a62d0acSEric Blake qemu_iovec_destroy(&local_qiov); 13621a62d0acSEric Blake } else { 13631a62d0acSEric Blake num = bytes_remaining; 13641a62d0acSEric Blake ret = qemu_iovec_memset(qiov, bytes - bytes_remaining, 0, 13651a62d0acSEric Blake bytes_remaining); 13661a62d0acSEric Blake } 13671a62d0acSEric Blake if (ret < 0) { 13681a62d0acSEric Blake goto out; 13691a62d0acSEric Blake } 13701a62d0acSEric Blake bytes_remaining -= num; 137161007b31SStefan Hajnoczi } 137261007b31SStefan Hajnoczi 137361007b31SStefan Hajnoczi out: 13741a62d0acSEric Blake return ret < 0 ? ret : 0; 137561007b31SStefan Hajnoczi } 137661007b31SStefan Hajnoczi 137761007b31SStefan Hajnoczi /* 137861007b31SStefan Hajnoczi * Handle a read request in coroutine context 137961007b31SStefan Hajnoczi */ 1380a03ef88fSKevin Wolf int coroutine_fn bdrv_co_preadv(BdrvChild *child, 138161007b31SStefan Hajnoczi int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 138261007b31SStefan Hajnoczi BdrvRequestFlags flags) 138361007b31SStefan Hajnoczi { 1384a03ef88fSKevin Wolf BlockDriverState *bs = child->bs; 138561007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 138661007b31SStefan Hajnoczi BdrvTrackedRequest req; 138761007b31SStefan Hajnoczi 1388a5b8dd2cSEric Blake uint64_t align = bs->bl.request_alignment; 138961007b31SStefan Hajnoczi uint8_t *head_buf = NULL; 139061007b31SStefan Hajnoczi uint8_t *tail_buf = NULL; 139161007b31SStefan Hajnoczi QEMUIOVector local_qiov; 139261007b31SStefan Hajnoczi bool use_local_qiov = false; 139361007b31SStefan Hajnoczi int ret; 139461007b31SStefan Hajnoczi 1395f42cf447SDaniel P. Berrange trace_bdrv_co_preadv(child->bs, offset, bytes, flags); 1396f42cf447SDaniel P. Berrange 139761007b31SStefan Hajnoczi if (!drv) { 139861007b31SStefan Hajnoczi return -ENOMEDIUM; 139961007b31SStefan Hajnoczi } 140061007b31SStefan Hajnoczi 140161007b31SStefan Hajnoczi ret = bdrv_check_byte_request(bs, offset, bytes); 140261007b31SStefan Hajnoczi if (ret < 0) { 140361007b31SStefan Hajnoczi return ret; 140461007b31SStefan Hajnoczi } 140561007b31SStefan Hajnoczi 140699723548SPaolo Bonzini bdrv_inc_in_flight(bs); 140799723548SPaolo Bonzini 14089568b511SWen Congyang /* Don't do copy-on-read if we read data before write operation */ 1409d3faa13eSPaolo Bonzini if (atomic_read(&bs->copy_on_read) && !(flags & BDRV_REQ_NO_SERIALISING)) { 141061007b31SStefan Hajnoczi flags |= BDRV_REQ_COPY_ON_READ; 141161007b31SStefan Hajnoczi } 141261007b31SStefan Hajnoczi 141361007b31SStefan Hajnoczi /* Align read if necessary by padding qiov */ 141461007b31SStefan Hajnoczi if (offset & (align - 1)) { 141561007b31SStefan Hajnoczi head_buf = qemu_blockalign(bs, align); 141661007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 2); 141761007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 141861007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 141961007b31SStefan Hajnoczi use_local_qiov = true; 142061007b31SStefan Hajnoczi 142161007b31SStefan Hajnoczi bytes += offset & (align - 1); 142261007b31SStefan Hajnoczi offset = offset & ~(align - 1); 142361007b31SStefan Hajnoczi } 142461007b31SStefan Hajnoczi 142561007b31SStefan Hajnoczi if ((offset + bytes) & (align - 1)) { 142661007b31SStefan Hajnoczi if (!use_local_qiov) { 142761007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 1); 142861007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 142961007b31SStefan Hajnoczi use_local_qiov = true; 143061007b31SStefan Hajnoczi } 143161007b31SStefan Hajnoczi tail_buf = qemu_blockalign(bs, align); 143261007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, tail_buf, 143361007b31SStefan Hajnoczi align - ((offset + bytes) & (align - 1))); 143461007b31SStefan Hajnoczi 143561007b31SStefan Hajnoczi bytes = ROUND_UP(bytes, align); 143661007b31SStefan Hajnoczi } 143761007b31SStefan Hajnoczi 1438ebde595cSFam Zheng tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ); 143985c97ca7SKevin Wolf ret = bdrv_aligned_preadv(child, &req, offset, bytes, align, 144061007b31SStefan Hajnoczi use_local_qiov ? &local_qiov : qiov, 144161007b31SStefan Hajnoczi flags); 144261007b31SStefan Hajnoczi tracked_request_end(&req); 144399723548SPaolo Bonzini bdrv_dec_in_flight(bs); 144461007b31SStefan Hajnoczi 144561007b31SStefan Hajnoczi if (use_local_qiov) { 144661007b31SStefan Hajnoczi qemu_iovec_destroy(&local_qiov); 144761007b31SStefan Hajnoczi qemu_vfree(head_buf); 144861007b31SStefan Hajnoczi qemu_vfree(tail_buf); 144961007b31SStefan Hajnoczi } 145061007b31SStefan Hajnoczi 145161007b31SStefan Hajnoczi return ret; 145261007b31SStefan Hajnoczi } 145361007b31SStefan Hajnoczi 1454d05aa8bbSEric Blake static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 1455f5a5ca79SManos Pitsidianakis int64_t offset, int bytes, BdrvRequestFlags flags) 145661007b31SStefan Hajnoczi { 145761007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 145861007b31SStefan Hajnoczi QEMUIOVector qiov; 14590d93ed08SVladimir Sementsov-Ogievskiy void *buf = NULL; 146061007b31SStefan Hajnoczi int ret = 0; 1461465fe887SEric Blake bool need_flush = false; 1462443668caSDenis V. Lunev int head = 0; 1463443668caSDenis V. Lunev int tail = 0; 146461007b31SStefan Hajnoczi 1465cf081fcaSEric Blake int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, INT_MAX); 1466a5b8dd2cSEric Blake int alignment = MAX(bs->bl.pwrite_zeroes_alignment, 1467a5b8dd2cSEric Blake bs->bl.request_alignment); 1468cb2e2878SEric Blake int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER); 1469cf081fcaSEric Blake 1470d470ad42SMax Reitz if (!drv) { 1471d470ad42SMax Reitz return -ENOMEDIUM; 1472d470ad42SMax Reitz } 1473d470ad42SMax Reitz 1474fe0480d6SKevin Wolf if ((flags & ~bs->supported_zero_flags) & BDRV_REQ_NO_FALLBACK) { 1475fe0480d6SKevin Wolf return -ENOTSUP; 1476fe0480d6SKevin Wolf } 1477fe0480d6SKevin Wolf 1478b8d0a980SEric Blake assert(alignment % bs->bl.request_alignment == 0); 1479b8d0a980SEric Blake head = offset % alignment; 1480f5a5ca79SManos Pitsidianakis tail = (offset + bytes) % alignment; 1481b8d0a980SEric Blake max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment); 1482b8d0a980SEric Blake assert(max_write_zeroes >= bs->bl.request_alignment); 148361007b31SStefan Hajnoczi 1484f5a5ca79SManos Pitsidianakis while (bytes > 0 && !ret) { 1485f5a5ca79SManos Pitsidianakis int num = bytes; 148661007b31SStefan Hajnoczi 148761007b31SStefan Hajnoczi /* Align request. Block drivers can expect the "bulk" of the request 1488443668caSDenis V. Lunev * to be aligned, and that unaligned requests do not cross cluster 1489443668caSDenis V. Lunev * boundaries. 149061007b31SStefan Hajnoczi */ 1491443668caSDenis V. Lunev if (head) { 1492b2f95feeSEric Blake /* Make a small request up to the first aligned sector. For 1493b2f95feeSEric Blake * convenience, limit this request to max_transfer even if 1494b2f95feeSEric Blake * we don't need to fall back to writes. */ 1495f5a5ca79SManos Pitsidianakis num = MIN(MIN(bytes, max_transfer), alignment - head); 1496b2f95feeSEric Blake head = (head + num) % alignment; 1497b2f95feeSEric Blake assert(num < max_write_zeroes); 1498d05aa8bbSEric Blake } else if (tail && num > alignment) { 1499443668caSDenis V. Lunev /* Shorten the request to the last aligned sector. */ 1500443668caSDenis V. Lunev num -= tail; 150161007b31SStefan Hajnoczi } 150261007b31SStefan Hajnoczi 150361007b31SStefan Hajnoczi /* limit request size */ 150461007b31SStefan Hajnoczi if (num > max_write_zeroes) { 150561007b31SStefan Hajnoczi num = max_write_zeroes; 150661007b31SStefan Hajnoczi } 150761007b31SStefan Hajnoczi 150861007b31SStefan Hajnoczi ret = -ENOTSUP; 150961007b31SStefan Hajnoczi /* First try the efficient write zeroes operation */ 1510d05aa8bbSEric Blake if (drv->bdrv_co_pwrite_zeroes) { 1511d05aa8bbSEric Blake ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num, 1512d05aa8bbSEric Blake flags & bs->supported_zero_flags); 1513d05aa8bbSEric Blake if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) && 1514d05aa8bbSEric Blake !(bs->supported_zero_flags & BDRV_REQ_FUA)) { 1515d05aa8bbSEric Blake need_flush = true; 1516d05aa8bbSEric Blake } 1517465fe887SEric Blake } else { 1518465fe887SEric Blake assert(!bs->supported_zero_flags); 151961007b31SStefan Hajnoczi } 152061007b31SStefan Hajnoczi 1521118f9944SAndrey Shinkevich if (ret < 0 && !(flags & BDRV_REQ_NO_FALLBACK)) { 152261007b31SStefan Hajnoczi /* Fall back to bounce buffer if write zeroes is unsupported */ 1523465fe887SEric Blake BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE; 1524465fe887SEric Blake 1525465fe887SEric Blake if ((flags & BDRV_REQ_FUA) && 1526465fe887SEric Blake !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1527465fe887SEric Blake /* No need for bdrv_driver_pwrite() to do a fallback 1528465fe887SEric Blake * flush on each chunk; use just one at the end */ 1529465fe887SEric Blake write_flags &= ~BDRV_REQ_FUA; 1530465fe887SEric Blake need_flush = true; 1531465fe887SEric Blake } 15325def6b80SEric Blake num = MIN(num, max_transfer); 15330d93ed08SVladimir Sementsov-Ogievskiy if (buf == NULL) { 15340d93ed08SVladimir Sementsov-Ogievskiy buf = qemu_try_blockalign0(bs, num); 15350d93ed08SVladimir Sementsov-Ogievskiy if (buf == NULL) { 153661007b31SStefan Hajnoczi ret = -ENOMEM; 153761007b31SStefan Hajnoczi goto fail; 153861007b31SStefan Hajnoczi } 153961007b31SStefan Hajnoczi } 15400d93ed08SVladimir Sementsov-Ogievskiy qemu_iovec_init_buf(&qiov, buf, num); 154161007b31SStefan Hajnoczi 1542d05aa8bbSEric Blake ret = bdrv_driver_pwritev(bs, offset, num, &qiov, write_flags); 154361007b31SStefan Hajnoczi 154461007b31SStefan Hajnoczi /* Keep bounce buffer around if it is big enough for all 154561007b31SStefan Hajnoczi * all future requests. 154661007b31SStefan Hajnoczi */ 15475def6b80SEric Blake if (num < max_transfer) { 15480d93ed08SVladimir Sementsov-Ogievskiy qemu_vfree(buf); 15490d93ed08SVladimir Sementsov-Ogievskiy buf = NULL; 155061007b31SStefan Hajnoczi } 155161007b31SStefan Hajnoczi } 155261007b31SStefan Hajnoczi 1553d05aa8bbSEric Blake offset += num; 1554f5a5ca79SManos Pitsidianakis bytes -= num; 155561007b31SStefan Hajnoczi } 155661007b31SStefan Hajnoczi 155761007b31SStefan Hajnoczi fail: 1558465fe887SEric Blake if (ret == 0 && need_flush) { 1559465fe887SEric Blake ret = bdrv_co_flush(bs); 1560465fe887SEric Blake } 15610d93ed08SVladimir Sementsov-Ogievskiy qemu_vfree(buf); 156261007b31SStefan Hajnoczi return ret; 156361007b31SStefan Hajnoczi } 156461007b31SStefan Hajnoczi 156585fe2479SFam Zheng static inline int coroutine_fn 156685fe2479SFam Zheng bdrv_co_write_req_prepare(BdrvChild *child, int64_t offset, uint64_t bytes, 156785fe2479SFam Zheng BdrvTrackedRequest *req, int flags) 156885fe2479SFam Zheng { 156985fe2479SFam Zheng BlockDriverState *bs = child->bs; 157085fe2479SFam Zheng bool waited; 157185fe2479SFam Zheng int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 157285fe2479SFam Zheng 157385fe2479SFam Zheng if (bs->read_only) { 157485fe2479SFam Zheng return -EPERM; 157585fe2479SFam Zheng } 157685fe2479SFam Zheng 157785fe2479SFam Zheng /* BDRV_REQ_NO_SERIALISING is only for read operation */ 157885fe2479SFam Zheng assert(!(flags & BDRV_REQ_NO_SERIALISING)); 157985fe2479SFam Zheng assert(!(bs->open_flags & BDRV_O_INACTIVE)); 158085fe2479SFam Zheng assert((bs->open_flags & BDRV_O_NO_IO) == 0); 158185fe2479SFam Zheng assert(!(flags & ~BDRV_REQ_MASK)); 158285fe2479SFam Zheng 158385fe2479SFam Zheng if (flags & BDRV_REQ_SERIALISING) { 158485fe2479SFam Zheng mark_request_serialising(req, bdrv_get_cluster_size(bs)); 158585fe2479SFam Zheng } 158685fe2479SFam Zheng 158785fe2479SFam Zheng waited = wait_serialising_requests(req); 158885fe2479SFam Zheng 158985fe2479SFam Zheng assert(!waited || !req->serialising || 159085fe2479SFam Zheng is_request_serialising_and_aligned(req)); 159185fe2479SFam Zheng assert(req->overlap_offset <= offset); 159285fe2479SFam Zheng assert(offset + bytes <= req->overlap_offset + req->overlap_bytes); 1593cd47d792SFam Zheng assert(end_sector <= bs->total_sectors || child->perm & BLK_PERM_RESIZE); 159485fe2479SFam Zheng 1595cd47d792SFam Zheng switch (req->type) { 1596cd47d792SFam Zheng case BDRV_TRACKED_WRITE: 1597cd47d792SFam Zheng case BDRV_TRACKED_DISCARD: 159885fe2479SFam Zheng if (flags & BDRV_REQ_WRITE_UNCHANGED) { 159985fe2479SFam Zheng assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE)); 160085fe2479SFam Zheng } else { 160185fe2479SFam Zheng assert(child->perm & BLK_PERM_WRITE); 160285fe2479SFam Zheng } 1603cd47d792SFam Zheng return notifier_with_return_list_notify(&bs->before_write_notifiers, 1604cd47d792SFam Zheng req); 1605cd47d792SFam Zheng case BDRV_TRACKED_TRUNCATE: 1606cd47d792SFam Zheng assert(child->perm & BLK_PERM_RESIZE); 1607cd47d792SFam Zheng return 0; 1608cd47d792SFam Zheng default: 1609cd47d792SFam Zheng abort(); 1610cd47d792SFam Zheng } 161185fe2479SFam Zheng } 161285fe2479SFam Zheng 161385fe2479SFam Zheng static inline void coroutine_fn 161485fe2479SFam Zheng bdrv_co_write_req_finish(BdrvChild *child, int64_t offset, uint64_t bytes, 161585fe2479SFam Zheng BdrvTrackedRequest *req, int ret) 161685fe2479SFam Zheng { 161785fe2479SFam Zheng int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 161885fe2479SFam Zheng BlockDriverState *bs = child->bs; 161985fe2479SFam Zheng 162085fe2479SFam Zheng atomic_inc(&bs->write_gen); 162185fe2479SFam Zheng 162200695c27SFam Zheng /* 162300695c27SFam Zheng * Discard cannot extend the image, but in error handling cases, such as 162400695c27SFam Zheng * when reverting a qcow2 cluster allocation, the discarded range can pass 162500695c27SFam Zheng * the end of image file, so we cannot assert about BDRV_TRACKED_DISCARD 162600695c27SFam Zheng * here. Instead, just skip it, since semantically a discard request 162700695c27SFam Zheng * beyond EOF cannot expand the image anyway. 162800695c27SFam Zheng */ 16297f8f03efSFam Zheng if (ret == 0 && 1630cd47d792SFam Zheng (req->type == BDRV_TRACKED_TRUNCATE || 1631cd47d792SFam Zheng end_sector > bs->total_sectors) && 163200695c27SFam Zheng req->type != BDRV_TRACKED_DISCARD) { 16337f8f03efSFam Zheng bs->total_sectors = end_sector; 16347f8f03efSFam Zheng bdrv_parent_cb_resize(bs); 16357f8f03efSFam Zheng bdrv_dirty_bitmap_truncate(bs, end_sector << BDRV_SECTOR_BITS); 163685fe2479SFam Zheng } 163700695c27SFam Zheng if (req->bytes) { 163800695c27SFam Zheng switch (req->type) { 163900695c27SFam Zheng case BDRV_TRACKED_WRITE: 164000695c27SFam Zheng stat64_max(&bs->wr_highest_offset, offset + bytes); 164100695c27SFam Zheng /* fall through, to set dirty bits */ 164200695c27SFam Zheng case BDRV_TRACKED_DISCARD: 16437f8f03efSFam Zheng bdrv_set_dirty(bs, offset, bytes); 164400695c27SFam Zheng break; 164500695c27SFam Zheng default: 164600695c27SFam Zheng break; 164700695c27SFam Zheng } 164800695c27SFam Zheng } 164985fe2479SFam Zheng } 165085fe2479SFam Zheng 165161007b31SStefan Hajnoczi /* 165204ed95f4SEric Blake * Forwards an already correctly aligned write request to the BlockDriver, 165304ed95f4SEric Blake * after possibly fragmenting it. 165461007b31SStefan Hajnoczi */ 165585c97ca7SKevin Wolf static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child, 165661007b31SStefan Hajnoczi BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 1657cff86b38SEric Blake int64_t align, QEMUIOVector *qiov, int flags) 165861007b31SStefan Hajnoczi { 165985c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 166061007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 166161007b31SStefan Hajnoczi int ret; 166261007b31SStefan Hajnoczi 166304ed95f4SEric Blake uint64_t bytes_remaining = bytes; 166404ed95f4SEric Blake int max_transfer; 166561007b31SStefan Hajnoczi 1666d470ad42SMax Reitz if (!drv) { 1667d470ad42SMax Reitz return -ENOMEDIUM; 1668d470ad42SMax Reitz } 1669d470ad42SMax Reitz 1670d6883bc9SVladimir Sementsov-Ogievskiy if (bdrv_has_readonly_bitmaps(bs)) { 1671d6883bc9SVladimir Sementsov-Ogievskiy return -EPERM; 1672d6883bc9SVladimir Sementsov-Ogievskiy } 1673d6883bc9SVladimir Sementsov-Ogievskiy 1674cff86b38SEric Blake assert(is_power_of_2(align)); 1675cff86b38SEric Blake assert((offset & (align - 1)) == 0); 1676cff86b38SEric Blake assert((bytes & (align - 1)) == 0); 167761007b31SStefan Hajnoczi assert(!qiov || bytes == qiov->size); 167804ed95f4SEric Blake max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 167904ed95f4SEric Blake align); 168061007b31SStefan Hajnoczi 168185fe2479SFam Zheng ret = bdrv_co_write_req_prepare(child, offset, bytes, req, flags); 168261007b31SStefan Hajnoczi 168361007b31SStefan Hajnoczi if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF && 1684c1499a5eSEric Blake !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes && 168561007b31SStefan Hajnoczi qemu_iovec_is_zero(qiov)) { 168661007b31SStefan Hajnoczi flags |= BDRV_REQ_ZERO_WRITE; 168761007b31SStefan Hajnoczi if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) { 168861007b31SStefan Hajnoczi flags |= BDRV_REQ_MAY_UNMAP; 168961007b31SStefan Hajnoczi } 169061007b31SStefan Hajnoczi } 169161007b31SStefan Hajnoczi 169261007b31SStefan Hajnoczi if (ret < 0) { 169361007b31SStefan Hajnoczi /* Do nothing, write notifier decided to fail this request */ 169461007b31SStefan Hajnoczi } else if (flags & BDRV_REQ_ZERO_WRITE) { 16959a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO); 16969896c876SKevin Wolf ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags); 16973ea1a091SPavel Butsykin } else if (flags & BDRV_REQ_WRITE_COMPRESSED) { 16983ea1a091SPavel Butsykin ret = bdrv_driver_pwritev_compressed(bs, offset, bytes, qiov); 169904ed95f4SEric Blake } else if (bytes <= max_transfer) { 17009a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV); 170178a07294SKevin Wolf ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, flags); 170204ed95f4SEric Blake } else { 170304ed95f4SEric Blake bdrv_debug_event(bs, BLKDBG_PWRITEV); 170404ed95f4SEric Blake while (bytes_remaining) { 170504ed95f4SEric Blake int num = MIN(bytes_remaining, max_transfer); 170604ed95f4SEric Blake QEMUIOVector local_qiov; 170704ed95f4SEric Blake int local_flags = flags; 170804ed95f4SEric Blake 170904ed95f4SEric Blake assert(num); 171004ed95f4SEric Blake if (num < bytes_remaining && (flags & BDRV_REQ_FUA) && 171104ed95f4SEric Blake !(bs->supported_write_flags & BDRV_REQ_FUA)) { 171204ed95f4SEric Blake /* If FUA is going to be emulated by flush, we only 171304ed95f4SEric Blake * need to flush on the last iteration */ 171404ed95f4SEric Blake local_flags &= ~BDRV_REQ_FUA; 171504ed95f4SEric Blake } 171604ed95f4SEric Blake qemu_iovec_init(&local_qiov, qiov->niov); 171704ed95f4SEric Blake qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num); 171804ed95f4SEric Blake 171904ed95f4SEric Blake ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining, 172004ed95f4SEric Blake num, &local_qiov, local_flags); 172104ed95f4SEric Blake qemu_iovec_destroy(&local_qiov); 172204ed95f4SEric Blake if (ret < 0) { 172304ed95f4SEric Blake break; 172404ed95f4SEric Blake } 172504ed95f4SEric Blake bytes_remaining -= num; 172604ed95f4SEric Blake } 172761007b31SStefan Hajnoczi } 17289a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE); 172961007b31SStefan Hajnoczi 173061007b31SStefan Hajnoczi if (ret >= 0) { 173104ed95f4SEric Blake ret = 0; 173261007b31SStefan Hajnoczi } 173385fe2479SFam Zheng bdrv_co_write_req_finish(child, offset, bytes, req, ret); 173461007b31SStefan Hajnoczi 173561007b31SStefan Hajnoczi return ret; 173661007b31SStefan Hajnoczi } 173761007b31SStefan Hajnoczi 173885c97ca7SKevin Wolf static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child, 17399eeb6dd1SFam Zheng int64_t offset, 17409eeb6dd1SFam Zheng unsigned int bytes, 17419eeb6dd1SFam Zheng BdrvRequestFlags flags, 17429eeb6dd1SFam Zheng BdrvTrackedRequest *req) 17439eeb6dd1SFam Zheng { 174485c97ca7SKevin Wolf BlockDriverState *bs = child->bs; 17459eeb6dd1SFam Zheng uint8_t *buf = NULL; 17469eeb6dd1SFam Zheng QEMUIOVector local_qiov; 1747a5b8dd2cSEric Blake uint64_t align = bs->bl.request_alignment; 17489eeb6dd1SFam Zheng unsigned int head_padding_bytes, tail_padding_bytes; 17499eeb6dd1SFam Zheng int ret = 0; 17509eeb6dd1SFam Zheng 17519eeb6dd1SFam Zheng head_padding_bytes = offset & (align - 1); 1752f13ce1beSDenis V. Lunev tail_padding_bytes = (align - (offset + bytes)) & (align - 1); 17539eeb6dd1SFam Zheng 17549eeb6dd1SFam Zheng 17559eeb6dd1SFam Zheng assert(flags & BDRV_REQ_ZERO_WRITE); 17569eeb6dd1SFam Zheng if (head_padding_bytes || tail_padding_bytes) { 17579eeb6dd1SFam Zheng buf = qemu_blockalign(bs, align); 17580d93ed08SVladimir Sementsov-Ogievskiy qemu_iovec_init_buf(&local_qiov, buf, align); 17599eeb6dd1SFam Zheng } 17609eeb6dd1SFam Zheng if (head_padding_bytes) { 17619eeb6dd1SFam Zheng uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes); 17629eeb6dd1SFam Zheng 17639eeb6dd1SFam Zheng /* RMW the unaligned part before head. */ 17649eeb6dd1SFam Zheng mark_request_serialising(req, align); 17659eeb6dd1SFam Zheng wait_serialising_requests(req); 17669a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 176785c97ca7SKevin Wolf ret = bdrv_aligned_preadv(child, req, offset & ~(align - 1), align, 17689eeb6dd1SFam Zheng align, &local_qiov, 0); 17699eeb6dd1SFam Zheng if (ret < 0) { 17709eeb6dd1SFam Zheng goto fail; 17719eeb6dd1SFam Zheng } 17729a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 17739eeb6dd1SFam Zheng 17749eeb6dd1SFam Zheng memset(buf + head_padding_bytes, 0, zero_bytes); 177585c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, req, offset & ~(align - 1), align, 1776cff86b38SEric Blake align, &local_qiov, 17779eeb6dd1SFam Zheng flags & ~BDRV_REQ_ZERO_WRITE); 17789eeb6dd1SFam Zheng if (ret < 0) { 17799eeb6dd1SFam Zheng goto fail; 17809eeb6dd1SFam Zheng } 17819eeb6dd1SFam Zheng offset += zero_bytes; 17829eeb6dd1SFam Zheng bytes -= zero_bytes; 17839eeb6dd1SFam Zheng } 17849eeb6dd1SFam Zheng 17859eeb6dd1SFam Zheng assert(!bytes || (offset & (align - 1)) == 0); 17869eeb6dd1SFam Zheng if (bytes >= align) { 17879eeb6dd1SFam Zheng /* Write the aligned part in the middle. */ 17889eeb6dd1SFam Zheng uint64_t aligned_bytes = bytes & ~(align - 1); 178985c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align, 17909eeb6dd1SFam Zheng NULL, flags); 17919eeb6dd1SFam Zheng if (ret < 0) { 17929eeb6dd1SFam Zheng goto fail; 17939eeb6dd1SFam Zheng } 17949eeb6dd1SFam Zheng bytes -= aligned_bytes; 17959eeb6dd1SFam Zheng offset += aligned_bytes; 17969eeb6dd1SFam Zheng } 17979eeb6dd1SFam Zheng 17989eeb6dd1SFam Zheng assert(!bytes || (offset & (align - 1)) == 0); 17999eeb6dd1SFam Zheng if (bytes) { 18009eeb6dd1SFam Zheng assert(align == tail_padding_bytes + bytes); 18019eeb6dd1SFam Zheng /* RMW the unaligned part after tail. */ 18029eeb6dd1SFam Zheng mark_request_serialising(req, align); 18039eeb6dd1SFam Zheng wait_serialising_requests(req); 18049a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 180585c97ca7SKevin Wolf ret = bdrv_aligned_preadv(child, req, offset, align, 18069eeb6dd1SFam Zheng align, &local_qiov, 0); 18079eeb6dd1SFam Zheng if (ret < 0) { 18089eeb6dd1SFam Zheng goto fail; 18099eeb6dd1SFam Zheng } 18109a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 18119eeb6dd1SFam Zheng 18129eeb6dd1SFam Zheng memset(buf, 0, bytes); 181385c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, req, offset, align, align, 18149eeb6dd1SFam Zheng &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE); 18159eeb6dd1SFam Zheng } 18169eeb6dd1SFam Zheng fail: 18179eeb6dd1SFam Zheng qemu_vfree(buf); 18189eeb6dd1SFam Zheng return ret; 18199eeb6dd1SFam Zheng 18209eeb6dd1SFam Zheng } 18219eeb6dd1SFam Zheng 182261007b31SStefan Hajnoczi /* 182361007b31SStefan Hajnoczi * Handle a write request in coroutine context 182461007b31SStefan Hajnoczi */ 1825a03ef88fSKevin Wolf int coroutine_fn bdrv_co_pwritev(BdrvChild *child, 182661007b31SStefan Hajnoczi int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 182761007b31SStefan Hajnoczi BdrvRequestFlags flags) 182861007b31SStefan Hajnoczi { 1829a03ef88fSKevin Wolf BlockDriverState *bs = child->bs; 183061007b31SStefan Hajnoczi BdrvTrackedRequest req; 1831a5b8dd2cSEric Blake uint64_t align = bs->bl.request_alignment; 183261007b31SStefan Hajnoczi uint8_t *head_buf = NULL; 183361007b31SStefan Hajnoczi uint8_t *tail_buf = NULL; 183461007b31SStefan Hajnoczi QEMUIOVector local_qiov; 183561007b31SStefan Hajnoczi bool use_local_qiov = false; 183661007b31SStefan Hajnoczi int ret; 183761007b31SStefan Hajnoczi 1838f42cf447SDaniel P. Berrange trace_bdrv_co_pwritev(child->bs, offset, bytes, flags); 1839f42cf447SDaniel P. Berrange 184061007b31SStefan Hajnoczi if (!bs->drv) { 184161007b31SStefan Hajnoczi return -ENOMEDIUM; 184261007b31SStefan Hajnoczi } 184361007b31SStefan Hajnoczi 184461007b31SStefan Hajnoczi ret = bdrv_check_byte_request(bs, offset, bytes); 184561007b31SStefan Hajnoczi if (ret < 0) { 184661007b31SStefan Hajnoczi return ret; 184761007b31SStefan Hajnoczi } 184861007b31SStefan Hajnoczi 184999723548SPaolo Bonzini bdrv_inc_in_flight(bs); 185061007b31SStefan Hajnoczi /* 185161007b31SStefan Hajnoczi * Align write if necessary by performing a read-modify-write cycle. 185261007b31SStefan Hajnoczi * Pad qiov with the read parts and be sure to have a tracked request not 185361007b31SStefan Hajnoczi * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle. 185461007b31SStefan Hajnoczi */ 1855ebde595cSFam Zheng tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE); 185661007b31SStefan Hajnoczi 185718a59f03SAnton Nefedov if (flags & BDRV_REQ_ZERO_WRITE) { 185885c97ca7SKevin Wolf ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req); 18599eeb6dd1SFam Zheng goto out; 18609eeb6dd1SFam Zheng } 18619eeb6dd1SFam Zheng 186261007b31SStefan Hajnoczi if (offset & (align - 1)) { 186361007b31SStefan Hajnoczi QEMUIOVector head_qiov; 186461007b31SStefan Hajnoczi 186561007b31SStefan Hajnoczi mark_request_serialising(&req, align); 186661007b31SStefan Hajnoczi wait_serialising_requests(&req); 186761007b31SStefan Hajnoczi 186861007b31SStefan Hajnoczi head_buf = qemu_blockalign(bs, align); 18690d93ed08SVladimir Sementsov-Ogievskiy qemu_iovec_init_buf(&head_qiov, head_buf, align); 187061007b31SStefan Hajnoczi 18719a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 187285c97ca7SKevin Wolf ret = bdrv_aligned_preadv(child, &req, offset & ~(align - 1), align, 187361007b31SStefan Hajnoczi align, &head_qiov, 0); 187461007b31SStefan Hajnoczi if (ret < 0) { 187561007b31SStefan Hajnoczi goto fail; 187661007b31SStefan Hajnoczi } 18779a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 187861007b31SStefan Hajnoczi 187961007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 2); 188061007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 188161007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 188261007b31SStefan Hajnoczi use_local_qiov = true; 188361007b31SStefan Hajnoczi 188461007b31SStefan Hajnoczi bytes += offset & (align - 1); 188561007b31SStefan Hajnoczi offset = offset & ~(align - 1); 1886117bc3faSPeter Lieven 1887117bc3faSPeter Lieven /* We have read the tail already if the request is smaller 1888117bc3faSPeter Lieven * than one aligned block. 1889117bc3faSPeter Lieven */ 1890117bc3faSPeter Lieven if (bytes < align) { 1891117bc3faSPeter Lieven qemu_iovec_add(&local_qiov, head_buf + bytes, align - bytes); 1892117bc3faSPeter Lieven bytes = align; 1893117bc3faSPeter Lieven } 189461007b31SStefan Hajnoczi } 189561007b31SStefan Hajnoczi 189661007b31SStefan Hajnoczi if ((offset + bytes) & (align - 1)) { 189761007b31SStefan Hajnoczi QEMUIOVector tail_qiov; 189861007b31SStefan Hajnoczi size_t tail_bytes; 189961007b31SStefan Hajnoczi bool waited; 190061007b31SStefan Hajnoczi 190161007b31SStefan Hajnoczi mark_request_serialising(&req, align); 190261007b31SStefan Hajnoczi waited = wait_serialising_requests(&req); 190361007b31SStefan Hajnoczi assert(!waited || !use_local_qiov); 190461007b31SStefan Hajnoczi 190561007b31SStefan Hajnoczi tail_buf = qemu_blockalign(bs, align); 19060d93ed08SVladimir Sementsov-Ogievskiy qemu_iovec_init_buf(&tail_qiov, tail_buf, align); 190761007b31SStefan Hajnoczi 19089a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 190985c97ca7SKevin Wolf ret = bdrv_aligned_preadv(child, &req, (offset + bytes) & ~(align - 1), 191085c97ca7SKevin Wolf align, align, &tail_qiov, 0); 191161007b31SStefan Hajnoczi if (ret < 0) { 191261007b31SStefan Hajnoczi goto fail; 191361007b31SStefan Hajnoczi } 19149a4f4c31SKevin Wolf bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 191561007b31SStefan Hajnoczi 191661007b31SStefan Hajnoczi if (!use_local_qiov) { 191761007b31SStefan Hajnoczi qemu_iovec_init(&local_qiov, qiov->niov + 1); 191861007b31SStefan Hajnoczi qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 191961007b31SStefan Hajnoczi use_local_qiov = true; 192061007b31SStefan Hajnoczi } 192161007b31SStefan Hajnoczi 192261007b31SStefan Hajnoczi tail_bytes = (offset + bytes) & (align - 1); 192361007b31SStefan Hajnoczi qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes); 192461007b31SStefan Hajnoczi 192561007b31SStefan Hajnoczi bytes = ROUND_UP(bytes, align); 192661007b31SStefan Hajnoczi } 192761007b31SStefan Hajnoczi 192885c97ca7SKevin Wolf ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align, 192961007b31SStefan Hajnoczi use_local_qiov ? &local_qiov : qiov, 193061007b31SStefan Hajnoczi flags); 193161007b31SStefan Hajnoczi 193261007b31SStefan Hajnoczi fail: 193361007b31SStefan Hajnoczi 193461007b31SStefan Hajnoczi if (use_local_qiov) { 193561007b31SStefan Hajnoczi qemu_iovec_destroy(&local_qiov); 193661007b31SStefan Hajnoczi } 193761007b31SStefan Hajnoczi qemu_vfree(head_buf); 193861007b31SStefan Hajnoczi qemu_vfree(tail_buf); 19399eeb6dd1SFam Zheng out: 19409eeb6dd1SFam Zheng tracked_request_end(&req); 194199723548SPaolo Bonzini bdrv_dec_in_flight(bs); 194261007b31SStefan Hajnoczi return ret; 194361007b31SStefan Hajnoczi } 194461007b31SStefan Hajnoczi 1945a03ef88fSKevin Wolf int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset, 1946f5a5ca79SManos Pitsidianakis int bytes, BdrvRequestFlags flags) 194761007b31SStefan Hajnoczi { 1948f5a5ca79SManos Pitsidianakis trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags); 194961007b31SStefan Hajnoczi 1950a03ef88fSKevin Wolf if (!(child->bs->open_flags & BDRV_O_UNMAP)) { 195161007b31SStefan Hajnoczi flags &= ~BDRV_REQ_MAY_UNMAP; 195261007b31SStefan Hajnoczi } 195361007b31SStefan Hajnoczi 1954f5a5ca79SManos Pitsidianakis return bdrv_co_pwritev(child, offset, bytes, NULL, 195561007b31SStefan Hajnoczi BDRV_REQ_ZERO_WRITE | flags); 195661007b31SStefan Hajnoczi } 195761007b31SStefan Hajnoczi 19584085f5c7SJohn Snow /* 19594085f5c7SJohn Snow * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not. 19604085f5c7SJohn Snow */ 19614085f5c7SJohn Snow int bdrv_flush_all(void) 19624085f5c7SJohn Snow { 19634085f5c7SJohn Snow BdrvNextIterator it; 19644085f5c7SJohn Snow BlockDriverState *bs = NULL; 19654085f5c7SJohn Snow int result = 0; 19664085f5c7SJohn Snow 19674085f5c7SJohn Snow for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 19684085f5c7SJohn Snow AioContext *aio_context = bdrv_get_aio_context(bs); 19694085f5c7SJohn Snow int ret; 19704085f5c7SJohn Snow 19714085f5c7SJohn Snow aio_context_acquire(aio_context); 19724085f5c7SJohn Snow ret = bdrv_flush(bs); 19734085f5c7SJohn Snow if (ret < 0 && !result) { 19744085f5c7SJohn Snow result = ret; 19754085f5c7SJohn Snow } 19764085f5c7SJohn Snow aio_context_release(aio_context); 19774085f5c7SJohn Snow } 19784085f5c7SJohn Snow 19794085f5c7SJohn Snow return result; 19804085f5c7SJohn Snow } 19814085f5c7SJohn Snow 19824085f5c7SJohn Snow 19834bcd936eSEric Blake typedef struct BdrvCoBlockStatusData { 198461007b31SStefan Hajnoczi BlockDriverState *bs; 198561007b31SStefan Hajnoczi BlockDriverState *base; 1986c9ce8c4dSEric Blake bool want_zero; 19874bcd936eSEric Blake int64_t offset; 19884bcd936eSEric Blake int64_t bytes; 19894bcd936eSEric Blake int64_t *pnum; 19904bcd936eSEric Blake int64_t *map; 1991c9ce8c4dSEric Blake BlockDriverState **file; 19924bcd936eSEric Blake int ret; 199361007b31SStefan Hajnoczi bool done; 19944bcd936eSEric Blake } BdrvCoBlockStatusData; 199561007b31SStefan Hajnoczi 19963e4d0e72SEric Blake int coroutine_fn bdrv_co_block_status_from_file(BlockDriverState *bs, 19973e4d0e72SEric Blake bool want_zero, 19983e4d0e72SEric Blake int64_t offset, 19993e4d0e72SEric Blake int64_t bytes, 20003e4d0e72SEric Blake int64_t *pnum, 20013e4d0e72SEric Blake int64_t *map, 2002f7cc69b3SManos Pitsidianakis BlockDriverState **file) 2003f7cc69b3SManos Pitsidianakis { 2004f7cc69b3SManos Pitsidianakis assert(bs->file && bs->file->bs); 20053e4d0e72SEric Blake *pnum = bytes; 20063e4d0e72SEric Blake *map = offset; 2007f7cc69b3SManos Pitsidianakis *file = bs->file->bs; 20083e4d0e72SEric Blake return BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID; 2009f7cc69b3SManos Pitsidianakis } 2010f7cc69b3SManos Pitsidianakis 20113e4d0e72SEric Blake int coroutine_fn bdrv_co_block_status_from_backing(BlockDriverState *bs, 20123e4d0e72SEric Blake bool want_zero, 20133e4d0e72SEric Blake int64_t offset, 20143e4d0e72SEric Blake int64_t bytes, 20153e4d0e72SEric Blake int64_t *pnum, 20163e4d0e72SEric Blake int64_t *map, 2017f7cc69b3SManos Pitsidianakis BlockDriverState **file) 2018f7cc69b3SManos Pitsidianakis { 2019f7cc69b3SManos Pitsidianakis assert(bs->backing && bs->backing->bs); 20203e4d0e72SEric Blake *pnum = bytes; 20213e4d0e72SEric Blake *map = offset; 2022f7cc69b3SManos Pitsidianakis *file = bs->backing->bs; 20233e4d0e72SEric Blake return BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID; 2024f7cc69b3SManos Pitsidianakis } 2025f7cc69b3SManos Pitsidianakis 202661007b31SStefan Hajnoczi /* 202761007b31SStefan Hajnoczi * Returns the allocation status of the specified sectors. 202861007b31SStefan Hajnoczi * Drivers not implementing the functionality are assumed to not support 202961007b31SStefan Hajnoczi * backing files, hence all their sectors are reported as allocated. 203061007b31SStefan Hajnoczi * 203186a3d5c6SEric Blake * If 'want_zero' is true, the caller is querying for mapping 203286a3d5c6SEric Blake * purposes, with a focus on valid BDRV_BLOCK_OFFSET_VALID, _DATA, and 203386a3d5c6SEric Blake * _ZERO where possible; otherwise, the result favors larger 'pnum', 203486a3d5c6SEric Blake * with a focus on accurate BDRV_BLOCK_ALLOCATED. 2035c9ce8c4dSEric Blake * 20362e8bc787SEric Blake * If 'offset' is beyond the end of the disk image the return value is 2037fb0d8654SEric Blake * BDRV_BLOCK_EOF and 'pnum' is set to 0. 203861007b31SStefan Hajnoczi * 20392e8bc787SEric Blake * 'bytes' is the max value 'pnum' should be set to. If bytes goes 2040fb0d8654SEric Blake * beyond the end of the disk image it will be clamped; if 'pnum' is set to 2041fb0d8654SEric Blake * the end of the image, then the returned value will include BDRV_BLOCK_EOF. 204267a0fd2aSFam Zheng * 20432e8bc787SEric Blake * 'pnum' is set to the number of bytes (including and immediately 20442e8bc787SEric Blake * following the specified offset) that are easily known to be in the 20452e8bc787SEric Blake * same allocated/unallocated state. Note that a second call starting 20462e8bc787SEric Blake * at the original offset plus returned pnum may have the same status. 20472e8bc787SEric Blake * The returned value is non-zero on success except at end-of-file. 20482e8bc787SEric Blake * 20492e8bc787SEric Blake * Returns negative errno on failure. Otherwise, if the 20502e8bc787SEric Blake * BDRV_BLOCK_OFFSET_VALID bit is set, 'map' and 'file' (if non-NULL) are 20512e8bc787SEric Blake * set to the host mapping and BDS corresponding to the guest offset. 205261007b31SStefan Hajnoczi */ 20532e8bc787SEric Blake static int coroutine_fn bdrv_co_block_status(BlockDriverState *bs, 2054c9ce8c4dSEric Blake bool want_zero, 20552e8bc787SEric Blake int64_t offset, int64_t bytes, 20562e8bc787SEric Blake int64_t *pnum, int64_t *map, 205767a0fd2aSFam Zheng BlockDriverState **file) 205861007b31SStefan Hajnoczi { 20592e8bc787SEric Blake int64_t total_size; 20602e8bc787SEric Blake int64_t n; /* bytes */ 2061efa6e2edSEric Blake int ret; 20622e8bc787SEric Blake int64_t local_map = 0; 2063298a1665SEric Blake BlockDriverState *local_file = NULL; 2064efa6e2edSEric Blake int64_t aligned_offset, aligned_bytes; 2065efa6e2edSEric Blake uint32_t align; 206661007b31SStefan Hajnoczi 2067298a1665SEric Blake assert(pnum); 2068298a1665SEric Blake *pnum = 0; 20692e8bc787SEric Blake total_size = bdrv_getlength(bs); 20702e8bc787SEric Blake if (total_size < 0) { 20712e8bc787SEric Blake ret = total_size; 2072298a1665SEric Blake goto early_out; 207361007b31SStefan Hajnoczi } 207461007b31SStefan Hajnoczi 20752e8bc787SEric Blake if (offset >= total_size) { 2076298a1665SEric Blake ret = BDRV_BLOCK_EOF; 2077298a1665SEric Blake goto early_out; 207861007b31SStefan Hajnoczi } 20792e8bc787SEric Blake if (!bytes) { 2080298a1665SEric Blake ret = 0; 2081298a1665SEric Blake goto early_out; 20829cdcfd9fSEric Blake } 208361007b31SStefan Hajnoczi 20842e8bc787SEric Blake n = total_size - offset; 20852e8bc787SEric Blake if (n < bytes) { 20862e8bc787SEric Blake bytes = n; 208761007b31SStefan Hajnoczi } 208861007b31SStefan Hajnoczi 2089d470ad42SMax Reitz /* Must be non-NULL or bdrv_getlength() would have failed */ 2090d470ad42SMax Reitz assert(bs->drv); 2091636cb512SEric Blake if (!bs->drv->bdrv_co_block_status) { 20922e8bc787SEric Blake *pnum = bytes; 209361007b31SStefan Hajnoczi ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED; 20942e8bc787SEric Blake if (offset + bytes == total_size) { 2095fb0d8654SEric Blake ret |= BDRV_BLOCK_EOF; 2096fb0d8654SEric Blake } 209761007b31SStefan Hajnoczi if (bs->drv->protocol_name) { 20982e8bc787SEric Blake ret |= BDRV_BLOCK_OFFSET_VALID; 20992e8bc787SEric Blake local_map = offset; 2100298a1665SEric Blake local_file = bs; 210161007b31SStefan Hajnoczi } 2102298a1665SEric Blake goto early_out; 210361007b31SStefan Hajnoczi } 210461007b31SStefan Hajnoczi 210599723548SPaolo Bonzini bdrv_inc_in_flight(bs); 2106efa6e2edSEric Blake 2107efa6e2edSEric Blake /* Round out to request_alignment boundaries */ 210886a3d5c6SEric Blake align = bs->bl.request_alignment; 2109efa6e2edSEric Blake aligned_offset = QEMU_ALIGN_DOWN(offset, align); 2110efa6e2edSEric Blake aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset; 2111efa6e2edSEric Blake 211286a3d5c6SEric Blake ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset, 211386a3d5c6SEric Blake aligned_bytes, pnum, &local_map, 211486a3d5c6SEric Blake &local_file); 211586a3d5c6SEric Blake if (ret < 0) { 211686a3d5c6SEric Blake *pnum = 0; 211786a3d5c6SEric Blake goto out; 211886a3d5c6SEric Blake } 2119efa6e2edSEric Blake 2120efa6e2edSEric Blake /* 2121636cb512SEric Blake * The driver's result must be a non-zero multiple of request_alignment. 2122efa6e2edSEric Blake * Clamp pnum and adjust map to original request. 2123efa6e2edSEric Blake */ 2124636cb512SEric Blake assert(*pnum && QEMU_IS_ALIGNED(*pnum, align) && 2125636cb512SEric Blake align > offset - aligned_offset); 212669f47505SVladimir Sementsov-Ogievskiy if (ret & BDRV_BLOCK_RECURSE) { 212769f47505SVladimir Sementsov-Ogievskiy assert(ret & BDRV_BLOCK_DATA); 212869f47505SVladimir Sementsov-Ogievskiy assert(ret & BDRV_BLOCK_OFFSET_VALID); 212969f47505SVladimir Sementsov-Ogievskiy assert(!(ret & BDRV_BLOCK_ZERO)); 213069f47505SVladimir Sementsov-Ogievskiy } 213169f47505SVladimir Sementsov-Ogievskiy 2132efa6e2edSEric Blake *pnum -= offset - aligned_offset; 2133efa6e2edSEric Blake if (*pnum > bytes) { 2134efa6e2edSEric Blake *pnum = bytes; 2135efa6e2edSEric Blake } 2136efa6e2edSEric Blake if (ret & BDRV_BLOCK_OFFSET_VALID) { 2137efa6e2edSEric Blake local_map += offset - aligned_offset; 2138efa6e2edSEric Blake } 213961007b31SStefan Hajnoczi 214061007b31SStefan Hajnoczi if (ret & BDRV_BLOCK_RAW) { 2141298a1665SEric Blake assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file); 21422e8bc787SEric Blake ret = bdrv_co_block_status(local_file, want_zero, local_map, 21432e8bc787SEric Blake *pnum, pnum, &local_map, &local_file); 214499723548SPaolo Bonzini goto out; 214561007b31SStefan Hajnoczi } 214661007b31SStefan Hajnoczi 214761007b31SStefan Hajnoczi if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) { 214861007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ALLOCATED; 2149c9ce8c4dSEric Blake } else if (want_zero) { 215061007b31SStefan Hajnoczi if (bdrv_unallocated_blocks_are_zero(bs)) { 215161007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 2152760e0063SKevin Wolf } else if (bs->backing) { 2153760e0063SKevin Wolf BlockDriverState *bs2 = bs->backing->bs; 21542e8bc787SEric Blake int64_t size2 = bdrv_getlength(bs2); 2155c9ce8c4dSEric Blake 21562e8bc787SEric Blake if (size2 >= 0 && offset >= size2) { 215761007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 215861007b31SStefan Hajnoczi } 215961007b31SStefan Hajnoczi } 216061007b31SStefan Hajnoczi } 216161007b31SStefan Hajnoczi 216269f47505SVladimir Sementsov-Ogievskiy if (want_zero && ret & BDRV_BLOCK_RECURSE && 216369f47505SVladimir Sementsov-Ogievskiy local_file && local_file != bs && 216461007b31SStefan Hajnoczi (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) && 216561007b31SStefan Hajnoczi (ret & BDRV_BLOCK_OFFSET_VALID)) { 21662e8bc787SEric Blake int64_t file_pnum; 21672e8bc787SEric Blake int ret2; 216861007b31SStefan Hajnoczi 21692e8bc787SEric Blake ret2 = bdrv_co_block_status(local_file, want_zero, local_map, 21702e8bc787SEric Blake *pnum, &file_pnum, NULL, NULL); 217161007b31SStefan Hajnoczi if (ret2 >= 0) { 217261007b31SStefan Hajnoczi /* Ignore errors. This is just providing extra information, it 217361007b31SStefan Hajnoczi * is useful but not necessary. 217461007b31SStefan Hajnoczi */ 2175c61e684eSEric Blake if (ret2 & BDRV_BLOCK_EOF && 2176c61e684eSEric Blake (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) { 2177c61e684eSEric Blake /* 2178c61e684eSEric Blake * It is valid for the format block driver to read 2179c61e684eSEric Blake * beyond the end of the underlying file's current 2180c61e684eSEric Blake * size; such areas read as zero. 2181c61e684eSEric Blake */ 218261007b31SStefan Hajnoczi ret |= BDRV_BLOCK_ZERO; 218361007b31SStefan Hajnoczi } else { 218461007b31SStefan Hajnoczi /* Limit request to the range reported by the protocol driver */ 218561007b31SStefan Hajnoczi *pnum = file_pnum; 218661007b31SStefan Hajnoczi ret |= (ret2 & BDRV_BLOCK_ZERO); 218761007b31SStefan Hajnoczi } 218861007b31SStefan Hajnoczi } 218961007b31SStefan Hajnoczi } 219061007b31SStefan Hajnoczi 219199723548SPaolo Bonzini out: 219299723548SPaolo Bonzini bdrv_dec_in_flight(bs); 21932e8bc787SEric Blake if (ret >= 0 && offset + *pnum == total_size) { 2194fb0d8654SEric Blake ret |= BDRV_BLOCK_EOF; 2195fb0d8654SEric Blake } 2196298a1665SEric Blake early_out: 2197298a1665SEric Blake if (file) { 2198298a1665SEric Blake *file = local_file; 2199298a1665SEric Blake } 22002e8bc787SEric Blake if (map) { 22012e8bc787SEric Blake *map = local_map; 22022e8bc787SEric Blake } 220361007b31SStefan Hajnoczi return ret; 220461007b31SStefan Hajnoczi } 220561007b31SStefan Hajnoczi 22065b648c67SEric Blake static int coroutine_fn bdrv_co_block_status_above(BlockDriverState *bs, 2207ba3f0e25SFam Zheng BlockDriverState *base, 2208c9ce8c4dSEric Blake bool want_zero, 22095b648c67SEric Blake int64_t offset, 22105b648c67SEric Blake int64_t bytes, 22115b648c67SEric Blake int64_t *pnum, 22125b648c67SEric Blake int64_t *map, 221367a0fd2aSFam Zheng BlockDriverState **file) 2214ba3f0e25SFam Zheng { 2215ba3f0e25SFam Zheng BlockDriverState *p; 22165b648c67SEric Blake int ret = 0; 2217c61e684eSEric Blake bool first = true; 2218ba3f0e25SFam Zheng 2219ba3f0e25SFam Zheng assert(bs != base); 2220760e0063SKevin Wolf for (p = bs; p != base; p = backing_bs(p)) { 22215b648c67SEric Blake ret = bdrv_co_block_status(p, want_zero, offset, bytes, pnum, map, 22225b648c67SEric Blake file); 2223c61e684eSEric Blake if (ret < 0) { 2224c61e684eSEric Blake break; 2225c61e684eSEric Blake } 2226c61e684eSEric Blake if (ret & BDRV_BLOCK_ZERO && ret & BDRV_BLOCK_EOF && !first) { 2227c61e684eSEric Blake /* 2228c61e684eSEric Blake * Reading beyond the end of the file continues to read 2229c61e684eSEric Blake * zeroes, but we can only widen the result to the 2230c61e684eSEric Blake * unallocated length we learned from an earlier 2231c61e684eSEric Blake * iteration. 2232c61e684eSEric Blake */ 22335b648c67SEric Blake *pnum = bytes; 2234c61e684eSEric Blake } 2235c61e684eSEric Blake if (ret & (BDRV_BLOCK_ZERO | BDRV_BLOCK_DATA)) { 2236ba3f0e25SFam Zheng break; 2237ba3f0e25SFam Zheng } 22385b648c67SEric Blake /* [offset, pnum] unallocated on this layer, which could be only 22395b648c67SEric Blake * the first part of [offset, bytes]. */ 22405b648c67SEric Blake bytes = MIN(bytes, *pnum); 2241c61e684eSEric Blake first = false; 2242ba3f0e25SFam Zheng } 2243ba3f0e25SFam Zheng return ret; 2244ba3f0e25SFam Zheng } 2245ba3f0e25SFam Zheng 224631826642SEric Blake /* Coroutine wrapper for bdrv_block_status_above() */ 22475b648c67SEric Blake static void coroutine_fn bdrv_block_status_above_co_entry(void *opaque) 224861007b31SStefan Hajnoczi { 22494bcd936eSEric Blake BdrvCoBlockStatusData *data = opaque; 225061007b31SStefan Hajnoczi 22515b648c67SEric Blake data->ret = bdrv_co_block_status_above(data->bs, data->base, 2252c9ce8c4dSEric Blake data->want_zero, 22535b648c67SEric Blake data->offset, data->bytes, 22545b648c67SEric Blake data->pnum, data->map, data->file); 225561007b31SStefan Hajnoczi data->done = true; 22564720cbeeSKevin Wolf aio_wait_kick(); 225761007b31SStefan Hajnoczi } 225861007b31SStefan Hajnoczi 225961007b31SStefan Hajnoczi /* 22605b648c67SEric Blake * Synchronous wrapper around bdrv_co_block_status_above(). 226161007b31SStefan Hajnoczi * 22625b648c67SEric Blake * See bdrv_co_block_status_above() for details. 226361007b31SStefan Hajnoczi */ 22647ddb99b9SEric Blake static int bdrv_common_block_status_above(BlockDriverState *bs, 2265ba3f0e25SFam Zheng BlockDriverState *base, 22667ddb99b9SEric Blake bool want_zero, int64_t offset, 22677ddb99b9SEric Blake int64_t bytes, int64_t *pnum, 22687ddb99b9SEric Blake int64_t *map, 226967a0fd2aSFam Zheng BlockDriverState **file) 227061007b31SStefan Hajnoczi { 227161007b31SStefan Hajnoczi Coroutine *co; 22724bcd936eSEric Blake BdrvCoBlockStatusData data = { 227361007b31SStefan Hajnoczi .bs = bs, 2274ba3f0e25SFam Zheng .base = base, 2275c9ce8c4dSEric Blake .want_zero = want_zero, 22767ddb99b9SEric Blake .offset = offset, 22777ddb99b9SEric Blake .bytes = bytes, 22787ddb99b9SEric Blake .pnum = pnum, 22797ddb99b9SEric Blake .map = map, 2280c9ce8c4dSEric Blake .file = file, 228161007b31SStefan Hajnoczi .done = false, 228261007b31SStefan Hajnoczi }; 228361007b31SStefan Hajnoczi 228461007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 228561007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 22865b648c67SEric Blake bdrv_block_status_above_co_entry(&data); 228761007b31SStefan Hajnoczi } else { 22885b648c67SEric Blake co = qemu_coroutine_create(bdrv_block_status_above_co_entry, &data); 2289e92f0e19SFam Zheng bdrv_coroutine_enter(bs, co); 229088b062c2SPaolo Bonzini BDRV_POLL_WHILE(bs, !data.done); 229161007b31SStefan Hajnoczi } 229261007b31SStefan Hajnoczi return data.ret; 229361007b31SStefan Hajnoczi } 229461007b31SStefan Hajnoczi 229531826642SEric Blake int bdrv_block_status_above(BlockDriverState *bs, BlockDriverState *base, 229631826642SEric Blake int64_t offset, int64_t bytes, int64_t *pnum, 229731826642SEric Blake int64_t *map, BlockDriverState **file) 2298c9ce8c4dSEric Blake { 229931826642SEric Blake return bdrv_common_block_status_above(bs, base, true, offset, bytes, 230031826642SEric Blake pnum, map, file); 2301c9ce8c4dSEric Blake } 2302c9ce8c4dSEric Blake 2303237d78f8SEric Blake int bdrv_block_status(BlockDriverState *bs, int64_t offset, int64_t bytes, 2304237d78f8SEric Blake int64_t *pnum, int64_t *map, BlockDriverState **file) 2305ba3f0e25SFam Zheng { 230631826642SEric Blake return bdrv_block_status_above(bs, backing_bs(bs), 230731826642SEric Blake offset, bytes, pnum, map, file); 2308ba3f0e25SFam Zheng } 2309ba3f0e25SFam Zheng 2310d6a644bbSEric Blake int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t offset, 2311d6a644bbSEric Blake int64_t bytes, int64_t *pnum) 231261007b31SStefan Hajnoczi { 23137ddb99b9SEric Blake int ret; 23147ddb99b9SEric Blake int64_t dummy; 2315d6a644bbSEric Blake 23167ddb99b9SEric Blake ret = bdrv_common_block_status_above(bs, backing_bs(bs), false, offset, 23177ddb99b9SEric Blake bytes, pnum ? pnum : &dummy, NULL, 2318298a1665SEric Blake NULL); 231961007b31SStefan Hajnoczi if (ret < 0) { 232061007b31SStefan Hajnoczi return ret; 232161007b31SStefan Hajnoczi } 232261007b31SStefan Hajnoczi return !!(ret & BDRV_BLOCK_ALLOCATED); 232361007b31SStefan Hajnoczi } 232461007b31SStefan Hajnoczi 232561007b31SStefan Hajnoczi /* 232661007b31SStefan Hajnoczi * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP] 232761007b31SStefan Hajnoczi * 2328170d3bd3SAndrey Shinkevich * Return 1 if (a prefix of) the given range is allocated in any image 2329170d3bd3SAndrey Shinkevich * between BASE and TOP (BASE is only included if include_base is set). 2330170d3bd3SAndrey Shinkevich * BASE can be NULL to check if the given offset is allocated in any 2331170d3bd3SAndrey Shinkevich * image of the chain. Return 0 otherwise, or negative errno on 2332170d3bd3SAndrey Shinkevich * failure. 233361007b31SStefan Hajnoczi * 233451b0a488SEric Blake * 'pnum' is set to the number of bytes (including and immediately 233551b0a488SEric Blake * following the specified offset) that are known to be in the same 233651b0a488SEric Blake * allocated/unallocated state. Note that a subsequent call starting 233751b0a488SEric Blake * at 'offset + *pnum' may return the same allocation status (in other 233851b0a488SEric Blake * words, the result is not necessarily the maximum possible range); 233951b0a488SEric Blake * but 'pnum' will only be 0 when end of file is reached. 234061007b31SStefan Hajnoczi * 234161007b31SStefan Hajnoczi */ 234261007b31SStefan Hajnoczi int bdrv_is_allocated_above(BlockDriverState *top, 234361007b31SStefan Hajnoczi BlockDriverState *base, 2344170d3bd3SAndrey Shinkevich bool include_base, int64_t offset, 2345170d3bd3SAndrey Shinkevich int64_t bytes, int64_t *pnum) 234661007b31SStefan Hajnoczi { 234761007b31SStefan Hajnoczi BlockDriverState *intermediate; 234851b0a488SEric Blake int ret; 234951b0a488SEric Blake int64_t n = bytes; 235061007b31SStefan Hajnoczi 2351170d3bd3SAndrey Shinkevich assert(base || !include_base); 2352170d3bd3SAndrey Shinkevich 235361007b31SStefan Hajnoczi intermediate = top; 2354170d3bd3SAndrey Shinkevich while (include_base || intermediate != base) { 2355d6a644bbSEric Blake int64_t pnum_inter; 2356c00716beSEric Blake int64_t size_inter; 2357d6a644bbSEric Blake 2358170d3bd3SAndrey Shinkevich assert(intermediate); 235951b0a488SEric Blake ret = bdrv_is_allocated(intermediate, offset, bytes, &pnum_inter); 236061007b31SStefan Hajnoczi if (ret < 0) { 236161007b31SStefan Hajnoczi return ret; 2362d6a644bbSEric Blake } 2363d6a644bbSEric Blake if (ret) { 236451b0a488SEric Blake *pnum = pnum_inter; 236561007b31SStefan Hajnoczi return 1; 236661007b31SStefan Hajnoczi } 236761007b31SStefan Hajnoczi 236851b0a488SEric Blake size_inter = bdrv_getlength(intermediate); 2369c00716beSEric Blake if (size_inter < 0) { 2370c00716beSEric Blake return size_inter; 2371c00716beSEric Blake } 237251b0a488SEric Blake if (n > pnum_inter && 237351b0a488SEric Blake (intermediate == top || offset + pnum_inter < size_inter)) { 237451b0a488SEric Blake n = pnum_inter; 237561007b31SStefan Hajnoczi } 237661007b31SStefan Hajnoczi 2377170d3bd3SAndrey Shinkevich if (intermediate == base) { 2378170d3bd3SAndrey Shinkevich break; 2379170d3bd3SAndrey Shinkevich } 2380170d3bd3SAndrey Shinkevich 2381760e0063SKevin Wolf intermediate = backing_bs(intermediate); 238261007b31SStefan Hajnoczi } 238361007b31SStefan Hajnoczi 238461007b31SStefan Hajnoczi *pnum = n; 238561007b31SStefan Hajnoczi return 0; 238661007b31SStefan Hajnoczi } 238761007b31SStefan Hajnoczi 23881a8ae822SKevin Wolf typedef struct BdrvVmstateCo { 23891a8ae822SKevin Wolf BlockDriverState *bs; 23901a8ae822SKevin Wolf QEMUIOVector *qiov; 23911a8ae822SKevin Wolf int64_t pos; 23921a8ae822SKevin Wolf bool is_read; 23931a8ae822SKevin Wolf int ret; 23941a8ae822SKevin Wolf } BdrvVmstateCo; 23951a8ae822SKevin Wolf 23961a8ae822SKevin Wolf static int coroutine_fn 23971a8ae822SKevin Wolf bdrv_co_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos, 23981a8ae822SKevin Wolf bool is_read) 23991a8ae822SKevin Wolf { 24001a8ae822SKevin Wolf BlockDriver *drv = bs->drv; 2401dc88a467SStefan Hajnoczi int ret = -ENOTSUP; 2402dc88a467SStefan Hajnoczi 2403dc88a467SStefan Hajnoczi bdrv_inc_in_flight(bs); 24041a8ae822SKevin Wolf 24051a8ae822SKevin Wolf if (!drv) { 2406dc88a467SStefan Hajnoczi ret = -ENOMEDIUM; 24071a8ae822SKevin Wolf } else if (drv->bdrv_load_vmstate) { 2408dc88a467SStefan Hajnoczi if (is_read) { 2409dc88a467SStefan Hajnoczi ret = drv->bdrv_load_vmstate(bs, qiov, pos); 2410dc88a467SStefan Hajnoczi } else { 2411dc88a467SStefan Hajnoczi ret = drv->bdrv_save_vmstate(bs, qiov, pos); 2412dc88a467SStefan Hajnoczi } 24131a8ae822SKevin Wolf } else if (bs->file) { 2414dc88a467SStefan Hajnoczi ret = bdrv_co_rw_vmstate(bs->file->bs, qiov, pos, is_read); 24151a8ae822SKevin Wolf } 24161a8ae822SKevin Wolf 2417dc88a467SStefan Hajnoczi bdrv_dec_in_flight(bs); 2418dc88a467SStefan Hajnoczi return ret; 24191a8ae822SKevin Wolf } 24201a8ae822SKevin Wolf 24211a8ae822SKevin Wolf static void coroutine_fn bdrv_co_rw_vmstate_entry(void *opaque) 24221a8ae822SKevin Wolf { 24231a8ae822SKevin Wolf BdrvVmstateCo *co = opaque; 24241a8ae822SKevin Wolf co->ret = bdrv_co_rw_vmstate(co->bs, co->qiov, co->pos, co->is_read); 24254720cbeeSKevin Wolf aio_wait_kick(); 24261a8ae822SKevin Wolf } 24271a8ae822SKevin Wolf 24281a8ae822SKevin Wolf static inline int 24291a8ae822SKevin Wolf bdrv_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos, 24301a8ae822SKevin Wolf bool is_read) 24311a8ae822SKevin Wolf { 24321a8ae822SKevin Wolf if (qemu_in_coroutine()) { 24331a8ae822SKevin Wolf return bdrv_co_rw_vmstate(bs, qiov, pos, is_read); 24341a8ae822SKevin Wolf } else { 24351a8ae822SKevin Wolf BdrvVmstateCo data = { 24361a8ae822SKevin Wolf .bs = bs, 24371a8ae822SKevin Wolf .qiov = qiov, 24381a8ae822SKevin Wolf .pos = pos, 24391a8ae822SKevin Wolf .is_read = is_read, 24401a8ae822SKevin Wolf .ret = -EINPROGRESS, 24411a8ae822SKevin Wolf }; 24420b8b8753SPaolo Bonzini Coroutine *co = qemu_coroutine_create(bdrv_co_rw_vmstate_entry, &data); 24431a8ae822SKevin Wolf 2444e92f0e19SFam Zheng bdrv_coroutine_enter(bs, co); 2445ea17c9d2SStefan Hajnoczi BDRV_POLL_WHILE(bs, data.ret == -EINPROGRESS); 24461a8ae822SKevin Wolf return data.ret; 24471a8ae822SKevin Wolf } 24481a8ae822SKevin Wolf } 24491a8ae822SKevin Wolf 245061007b31SStefan Hajnoczi int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf, 245161007b31SStefan Hajnoczi int64_t pos, int size) 245261007b31SStefan Hajnoczi { 24530d93ed08SVladimir Sementsov-Ogievskiy QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size); 2454b433d942SKevin Wolf int ret; 245561007b31SStefan Hajnoczi 2456b433d942SKevin Wolf ret = bdrv_writev_vmstate(bs, &qiov, pos); 2457b433d942SKevin Wolf if (ret < 0) { 2458b433d942SKevin Wolf return ret; 2459b433d942SKevin Wolf } 2460b433d942SKevin Wolf 2461b433d942SKevin Wolf return size; 246261007b31SStefan Hajnoczi } 246361007b31SStefan Hajnoczi 246461007b31SStefan Hajnoczi int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 246561007b31SStefan Hajnoczi { 24661a8ae822SKevin Wolf return bdrv_rw_vmstate(bs, qiov, pos, false); 246761007b31SStefan Hajnoczi } 246861007b31SStefan Hajnoczi 246961007b31SStefan Hajnoczi int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf, 247061007b31SStefan Hajnoczi int64_t pos, int size) 247161007b31SStefan Hajnoczi { 24720d93ed08SVladimir Sementsov-Ogievskiy QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size); 2473b433d942SKevin Wolf int ret; 24745ddda0b8SKevin Wolf 2475b433d942SKevin Wolf ret = bdrv_readv_vmstate(bs, &qiov, pos); 2476b433d942SKevin Wolf if (ret < 0) { 2477b433d942SKevin Wolf return ret; 2478b433d942SKevin Wolf } 2479b433d942SKevin Wolf 2480b433d942SKevin Wolf return size; 24815ddda0b8SKevin Wolf } 24825ddda0b8SKevin Wolf 24835ddda0b8SKevin Wolf int bdrv_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 24845ddda0b8SKevin Wolf { 24851a8ae822SKevin Wolf return bdrv_rw_vmstate(bs, qiov, pos, true); 248661007b31SStefan Hajnoczi } 248761007b31SStefan Hajnoczi 248861007b31SStefan Hajnoczi /**************************************************************/ 248961007b31SStefan Hajnoczi /* async I/Os */ 249061007b31SStefan Hajnoczi 249161007b31SStefan Hajnoczi void bdrv_aio_cancel(BlockAIOCB *acb) 249261007b31SStefan Hajnoczi { 249361007b31SStefan Hajnoczi qemu_aio_ref(acb); 249461007b31SStefan Hajnoczi bdrv_aio_cancel_async(acb); 249561007b31SStefan Hajnoczi while (acb->refcnt > 1) { 249661007b31SStefan Hajnoczi if (acb->aiocb_info->get_aio_context) { 249761007b31SStefan Hajnoczi aio_poll(acb->aiocb_info->get_aio_context(acb), true); 249861007b31SStefan Hajnoczi } else if (acb->bs) { 24992f47da5fSPaolo Bonzini /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so 25002f47da5fSPaolo Bonzini * assert that we're not using an I/O thread. Thread-safe 25012f47da5fSPaolo Bonzini * code should use bdrv_aio_cancel_async exclusively. 25022f47da5fSPaolo Bonzini */ 25032f47da5fSPaolo Bonzini assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context()); 250461007b31SStefan Hajnoczi aio_poll(bdrv_get_aio_context(acb->bs), true); 250561007b31SStefan Hajnoczi } else { 250661007b31SStefan Hajnoczi abort(); 250761007b31SStefan Hajnoczi } 250861007b31SStefan Hajnoczi } 250961007b31SStefan Hajnoczi qemu_aio_unref(acb); 251061007b31SStefan Hajnoczi } 251161007b31SStefan Hajnoczi 251261007b31SStefan Hajnoczi /* Async version of aio cancel. The caller is not blocked if the acb implements 251361007b31SStefan Hajnoczi * cancel_async, otherwise we do nothing and let the request normally complete. 251461007b31SStefan Hajnoczi * In either case the completion callback must be called. */ 251561007b31SStefan Hajnoczi void bdrv_aio_cancel_async(BlockAIOCB *acb) 251661007b31SStefan Hajnoczi { 251761007b31SStefan Hajnoczi if (acb->aiocb_info->cancel_async) { 251861007b31SStefan Hajnoczi acb->aiocb_info->cancel_async(acb); 251961007b31SStefan Hajnoczi } 252061007b31SStefan Hajnoczi } 252161007b31SStefan Hajnoczi 252261007b31SStefan Hajnoczi /**************************************************************/ 252361007b31SStefan Hajnoczi /* Coroutine block device emulation */ 252461007b31SStefan Hajnoczi 2525e293b7a3SKevin Wolf typedef struct FlushCo { 2526e293b7a3SKevin Wolf BlockDriverState *bs; 2527e293b7a3SKevin Wolf int ret; 2528e293b7a3SKevin Wolf } FlushCo; 2529e293b7a3SKevin Wolf 2530e293b7a3SKevin Wolf 253161007b31SStefan Hajnoczi static void coroutine_fn bdrv_flush_co_entry(void *opaque) 253261007b31SStefan Hajnoczi { 2533e293b7a3SKevin Wolf FlushCo *rwco = opaque; 253461007b31SStefan Hajnoczi 253561007b31SStefan Hajnoczi rwco->ret = bdrv_co_flush(rwco->bs); 25364720cbeeSKevin Wolf aio_wait_kick(); 253761007b31SStefan Hajnoczi } 253861007b31SStefan Hajnoczi 253961007b31SStefan Hajnoczi int coroutine_fn bdrv_co_flush(BlockDriverState *bs) 254061007b31SStefan Hajnoczi { 254149ca6259SFam Zheng int current_gen; 254249ca6259SFam Zheng int ret = 0; 254361007b31SStefan Hajnoczi 254499723548SPaolo Bonzini bdrv_inc_in_flight(bs); 2545c32b82afSPavel Dovgalyuk 2546e914404eSFam Zheng if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs) || 254749ca6259SFam Zheng bdrv_is_sg(bs)) { 254849ca6259SFam Zheng goto early_exit; 254949ca6259SFam Zheng } 255049ca6259SFam Zheng 25513783fa3dSPaolo Bonzini qemu_co_mutex_lock(&bs->reqs_lock); 255247fec599SPaolo Bonzini current_gen = atomic_read(&bs->write_gen); 25533ff2f67aSEvgeny Yakovlev 25543ff2f67aSEvgeny Yakovlev /* Wait until any previous flushes are completed */ 255599723548SPaolo Bonzini while (bs->active_flush_req) { 25563783fa3dSPaolo Bonzini qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock); 25573ff2f67aSEvgeny Yakovlev } 25583ff2f67aSEvgeny Yakovlev 25593783fa3dSPaolo Bonzini /* Flushes reach this point in nondecreasing current_gen order. */ 256099723548SPaolo Bonzini bs->active_flush_req = true; 25613783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&bs->reqs_lock); 25623ff2f67aSEvgeny Yakovlev 2563c32b82afSPavel Dovgalyuk /* Write back all layers by calling one driver function */ 2564c32b82afSPavel Dovgalyuk if (bs->drv->bdrv_co_flush) { 2565c32b82afSPavel Dovgalyuk ret = bs->drv->bdrv_co_flush(bs); 2566c32b82afSPavel Dovgalyuk goto out; 2567c32b82afSPavel Dovgalyuk } 2568c32b82afSPavel Dovgalyuk 256961007b31SStefan Hajnoczi /* Write back cached data to the OS even with cache=unsafe */ 257061007b31SStefan Hajnoczi BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS); 257161007b31SStefan Hajnoczi if (bs->drv->bdrv_co_flush_to_os) { 257261007b31SStefan Hajnoczi ret = bs->drv->bdrv_co_flush_to_os(bs); 257361007b31SStefan Hajnoczi if (ret < 0) { 2574cdb5e315SFam Zheng goto out; 257561007b31SStefan Hajnoczi } 257661007b31SStefan Hajnoczi } 257761007b31SStefan Hajnoczi 257861007b31SStefan Hajnoczi /* But don't actually force it to the disk with cache=unsafe */ 257961007b31SStefan Hajnoczi if (bs->open_flags & BDRV_O_NO_FLUSH) { 258061007b31SStefan Hajnoczi goto flush_parent; 258161007b31SStefan Hajnoczi } 258261007b31SStefan Hajnoczi 25833ff2f67aSEvgeny Yakovlev /* Check if we really need to flush anything */ 25843ff2f67aSEvgeny Yakovlev if (bs->flushed_gen == current_gen) { 25853ff2f67aSEvgeny Yakovlev goto flush_parent; 25863ff2f67aSEvgeny Yakovlev } 25873ff2f67aSEvgeny Yakovlev 258861007b31SStefan Hajnoczi BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK); 2589d470ad42SMax Reitz if (!bs->drv) { 2590d470ad42SMax Reitz /* bs->drv->bdrv_co_flush() might have ejected the BDS 2591d470ad42SMax Reitz * (even in case of apparent success) */ 2592d470ad42SMax Reitz ret = -ENOMEDIUM; 2593d470ad42SMax Reitz goto out; 2594d470ad42SMax Reitz } 259561007b31SStefan Hajnoczi if (bs->drv->bdrv_co_flush_to_disk) { 259661007b31SStefan Hajnoczi ret = bs->drv->bdrv_co_flush_to_disk(bs); 259761007b31SStefan Hajnoczi } else if (bs->drv->bdrv_aio_flush) { 259861007b31SStefan Hajnoczi BlockAIOCB *acb; 259961007b31SStefan Hajnoczi CoroutineIOCompletion co = { 260061007b31SStefan Hajnoczi .coroutine = qemu_coroutine_self(), 260161007b31SStefan Hajnoczi }; 260261007b31SStefan Hajnoczi 260361007b31SStefan Hajnoczi acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co); 260461007b31SStefan Hajnoczi if (acb == NULL) { 260561007b31SStefan Hajnoczi ret = -EIO; 260661007b31SStefan Hajnoczi } else { 260761007b31SStefan Hajnoczi qemu_coroutine_yield(); 260861007b31SStefan Hajnoczi ret = co.ret; 260961007b31SStefan Hajnoczi } 261061007b31SStefan Hajnoczi } else { 261161007b31SStefan Hajnoczi /* 261261007b31SStefan Hajnoczi * Some block drivers always operate in either writethrough or unsafe 261361007b31SStefan Hajnoczi * mode and don't support bdrv_flush therefore. Usually qemu doesn't 261461007b31SStefan Hajnoczi * know how the server works (because the behaviour is hardcoded or 261561007b31SStefan Hajnoczi * depends on server-side configuration), so we can't ensure that 261661007b31SStefan Hajnoczi * everything is safe on disk. Returning an error doesn't work because 261761007b31SStefan Hajnoczi * that would break guests even if the server operates in writethrough 261861007b31SStefan Hajnoczi * mode. 261961007b31SStefan Hajnoczi * 262061007b31SStefan Hajnoczi * Let's hope the user knows what he's doing. 262161007b31SStefan Hajnoczi */ 262261007b31SStefan Hajnoczi ret = 0; 262361007b31SStefan Hajnoczi } 26243ff2f67aSEvgeny Yakovlev 262561007b31SStefan Hajnoczi if (ret < 0) { 2626cdb5e315SFam Zheng goto out; 262761007b31SStefan Hajnoczi } 262861007b31SStefan Hajnoczi 262961007b31SStefan Hajnoczi /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH 263061007b31SStefan Hajnoczi * in the case of cache=unsafe, so there are no useless flushes. 263161007b31SStefan Hajnoczi */ 263261007b31SStefan Hajnoczi flush_parent: 2633cdb5e315SFam Zheng ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0; 2634cdb5e315SFam Zheng out: 26353ff2f67aSEvgeny Yakovlev /* Notify any pending flushes that we have completed */ 2636e6af1e08SKevin Wolf if (ret == 0) { 26373ff2f67aSEvgeny Yakovlev bs->flushed_gen = current_gen; 2638e6af1e08SKevin Wolf } 26393783fa3dSPaolo Bonzini 26403783fa3dSPaolo Bonzini qemu_co_mutex_lock(&bs->reqs_lock); 264199723548SPaolo Bonzini bs->active_flush_req = false; 2642156af3acSDenis V. Lunev /* Return value is ignored - it's ok if wait queue is empty */ 2643156af3acSDenis V. Lunev qemu_co_queue_next(&bs->flush_queue); 26443783fa3dSPaolo Bonzini qemu_co_mutex_unlock(&bs->reqs_lock); 26453ff2f67aSEvgeny Yakovlev 264649ca6259SFam Zheng early_exit: 264799723548SPaolo Bonzini bdrv_dec_in_flight(bs); 2648cdb5e315SFam Zheng return ret; 264961007b31SStefan Hajnoczi } 265061007b31SStefan Hajnoczi 265161007b31SStefan Hajnoczi int bdrv_flush(BlockDriverState *bs) 265261007b31SStefan Hajnoczi { 265361007b31SStefan Hajnoczi Coroutine *co; 2654e293b7a3SKevin Wolf FlushCo flush_co = { 265561007b31SStefan Hajnoczi .bs = bs, 265661007b31SStefan Hajnoczi .ret = NOT_DONE, 265761007b31SStefan Hajnoczi }; 265861007b31SStefan Hajnoczi 265961007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 266061007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 2661e293b7a3SKevin Wolf bdrv_flush_co_entry(&flush_co); 266261007b31SStefan Hajnoczi } else { 26630b8b8753SPaolo Bonzini co = qemu_coroutine_create(bdrv_flush_co_entry, &flush_co); 2664e92f0e19SFam Zheng bdrv_coroutine_enter(bs, co); 266588b062c2SPaolo Bonzini BDRV_POLL_WHILE(bs, flush_co.ret == NOT_DONE); 266661007b31SStefan Hajnoczi } 266761007b31SStefan Hajnoczi 2668e293b7a3SKevin Wolf return flush_co.ret; 266961007b31SStefan Hajnoczi } 267061007b31SStefan Hajnoczi 267161007b31SStefan Hajnoczi typedef struct DiscardCo { 26720b9fd3f4SFam Zheng BdrvChild *child; 26730c51a893SEric Blake int64_t offset; 2674d93e5726SVladimir Sementsov-Ogievskiy int64_t bytes; 267561007b31SStefan Hajnoczi int ret; 267661007b31SStefan Hajnoczi } DiscardCo; 26770c51a893SEric Blake static void coroutine_fn bdrv_pdiscard_co_entry(void *opaque) 267861007b31SStefan Hajnoczi { 267961007b31SStefan Hajnoczi DiscardCo *rwco = opaque; 268061007b31SStefan Hajnoczi 26810b9fd3f4SFam Zheng rwco->ret = bdrv_co_pdiscard(rwco->child, rwco->offset, rwco->bytes); 26824720cbeeSKevin Wolf aio_wait_kick(); 268361007b31SStefan Hajnoczi } 268461007b31SStefan Hajnoczi 2685d93e5726SVladimir Sementsov-Ogievskiy int coroutine_fn bdrv_co_pdiscard(BdrvChild *child, int64_t offset, 2686d93e5726SVladimir Sementsov-Ogievskiy int64_t bytes) 268761007b31SStefan Hajnoczi { 2688b1066c87SFam Zheng BdrvTrackedRequest req; 26899f1963b3SEric Blake int max_pdiscard, ret; 26903482b9bcSEric Blake int head, tail, align; 26910b9fd3f4SFam Zheng BlockDriverState *bs = child->bs; 269261007b31SStefan Hajnoczi 2693d93e5726SVladimir Sementsov-Ogievskiy if (!bs || !bs->drv || !bdrv_is_inserted(bs)) { 269461007b31SStefan Hajnoczi return -ENOMEDIUM; 269561007b31SStefan Hajnoczi } 269661007b31SStefan Hajnoczi 2697d6883bc9SVladimir Sementsov-Ogievskiy if (bdrv_has_readonly_bitmaps(bs)) { 2698d6883bc9SVladimir Sementsov-Ogievskiy return -EPERM; 2699d6883bc9SVladimir Sementsov-Ogievskiy } 2700d6883bc9SVladimir Sementsov-Ogievskiy 2701d93e5726SVladimir Sementsov-Ogievskiy if (offset < 0 || bytes < 0 || bytes > INT64_MAX - offset) { 2702d93e5726SVladimir Sementsov-Ogievskiy return -EIO; 270361007b31SStefan Hajnoczi } 270461007b31SStefan Hajnoczi 270561007b31SStefan Hajnoczi /* Do nothing if disabled. */ 270661007b31SStefan Hajnoczi if (!(bs->open_flags & BDRV_O_UNMAP)) { 270761007b31SStefan Hajnoczi return 0; 270861007b31SStefan Hajnoczi } 270961007b31SStefan Hajnoczi 271002aefe43SEric Blake if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) { 271161007b31SStefan Hajnoczi return 0; 271261007b31SStefan Hajnoczi } 271361007b31SStefan Hajnoczi 27143482b9bcSEric Blake /* Discard is advisory, but some devices track and coalesce 27153482b9bcSEric Blake * unaligned requests, so we must pass everything down rather than 27163482b9bcSEric Blake * round here. Still, most devices will just silently ignore 27173482b9bcSEric Blake * unaligned requests (by returning -ENOTSUP), so we must fragment 27183482b9bcSEric Blake * the request accordingly. */ 271902aefe43SEric Blake align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment); 2720b8d0a980SEric Blake assert(align % bs->bl.request_alignment == 0); 2721b8d0a980SEric Blake head = offset % align; 2722f5a5ca79SManos Pitsidianakis tail = (offset + bytes) % align; 27239f1963b3SEric Blake 272499723548SPaolo Bonzini bdrv_inc_in_flight(bs); 2725f5a5ca79SManos Pitsidianakis tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD); 272650824995SFam Zheng 272700695c27SFam Zheng ret = bdrv_co_write_req_prepare(child, offset, bytes, &req, 0); 2728ec050f77SDenis V. Lunev if (ret < 0) { 2729ec050f77SDenis V. Lunev goto out; 2730ec050f77SDenis V. Lunev } 2731ec050f77SDenis V. Lunev 27329f1963b3SEric Blake max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT_MAX), 27339f1963b3SEric Blake align); 27343482b9bcSEric Blake assert(max_pdiscard >= bs->bl.request_alignment); 27359f1963b3SEric Blake 2736f5a5ca79SManos Pitsidianakis while (bytes > 0) { 2737d93e5726SVladimir Sementsov-Ogievskiy int64_t num = bytes; 27383482b9bcSEric Blake 27393482b9bcSEric Blake if (head) { 27403482b9bcSEric Blake /* Make small requests to get to alignment boundaries. */ 2741f5a5ca79SManos Pitsidianakis num = MIN(bytes, align - head); 27423482b9bcSEric Blake if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) { 27433482b9bcSEric Blake num %= bs->bl.request_alignment; 27443482b9bcSEric Blake } 27453482b9bcSEric Blake head = (head + num) % align; 27463482b9bcSEric Blake assert(num < max_pdiscard); 27473482b9bcSEric Blake } else if (tail) { 27483482b9bcSEric Blake if (num > align) { 27493482b9bcSEric Blake /* Shorten the request to the last aligned cluster. */ 27503482b9bcSEric Blake num -= tail; 27513482b9bcSEric Blake } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) && 27523482b9bcSEric Blake tail > bs->bl.request_alignment) { 27533482b9bcSEric Blake tail %= bs->bl.request_alignment; 27543482b9bcSEric Blake num -= tail; 27553482b9bcSEric Blake } 27563482b9bcSEric Blake } 27573482b9bcSEric Blake /* limit request size */ 27583482b9bcSEric Blake if (num > max_pdiscard) { 27593482b9bcSEric Blake num = max_pdiscard; 27603482b9bcSEric Blake } 276161007b31SStefan Hajnoczi 2762d470ad42SMax Reitz if (!bs->drv) { 2763d470ad42SMax Reitz ret = -ENOMEDIUM; 2764d470ad42SMax Reitz goto out; 2765d470ad42SMax Reitz } 276647a5486dSEric Blake if (bs->drv->bdrv_co_pdiscard) { 276747a5486dSEric Blake ret = bs->drv->bdrv_co_pdiscard(bs, offset, num); 276861007b31SStefan Hajnoczi } else { 276961007b31SStefan Hajnoczi BlockAIOCB *acb; 277061007b31SStefan Hajnoczi CoroutineIOCompletion co = { 277161007b31SStefan Hajnoczi .coroutine = qemu_coroutine_self(), 277261007b31SStefan Hajnoczi }; 277361007b31SStefan Hajnoczi 27744da444a0SEric Blake acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num, 277561007b31SStefan Hajnoczi bdrv_co_io_em_complete, &co); 277661007b31SStefan Hajnoczi if (acb == NULL) { 2777b1066c87SFam Zheng ret = -EIO; 2778b1066c87SFam Zheng goto out; 277961007b31SStefan Hajnoczi } else { 278061007b31SStefan Hajnoczi qemu_coroutine_yield(); 278161007b31SStefan Hajnoczi ret = co.ret; 278261007b31SStefan Hajnoczi } 278361007b31SStefan Hajnoczi } 278461007b31SStefan Hajnoczi if (ret && ret != -ENOTSUP) { 2785b1066c87SFam Zheng goto out; 278661007b31SStefan Hajnoczi } 278761007b31SStefan Hajnoczi 27889f1963b3SEric Blake offset += num; 2789f5a5ca79SManos Pitsidianakis bytes -= num; 279061007b31SStefan Hajnoczi } 2791b1066c87SFam Zheng ret = 0; 2792b1066c87SFam Zheng out: 279300695c27SFam Zheng bdrv_co_write_req_finish(child, req.offset, req.bytes, &req, ret); 2794b1066c87SFam Zheng tracked_request_end(&req); 279599723548SPaolo Bonzini bdrv_dec_in_flight(bs); 2796b1066c87SFam Zheng return ret; 279761007b31SStefan Hajnoczi } 279861007b31SStefan Hajnoczi 2799d93e5726SVladimir Sementsov-Ogievskiy int bdrv_pdiscard(BdrvChild *child, int64_t offset, int64_t bytes) 280061007b31SStefan Hajnoczi { 280161007b31SStefan Hajnoczi Coroutine *co; 280261007b31SStefan Hajnoczi DiscardCo rwco = { 28030b9fd3f4SFam Zheng .child = child, 28040c51a893SEric Blake .offset = offset, 2805f5a5ca79SManos Pitsidianakis .bytes = bytes, 280661007b31SStefan Hajnoczi .ret = NOT_DONE, 280761007b31SStefan Hajnoczi }; 280861007b31SStefan Hajnoczi 280961007b31SStefan Hajnoczi if (qemu_in_coroutine()) { 281061007b31SStefan Hajnoczi /* Fast-path if already in coroutine context */ 28110c51a893SEric Blake bdrv_pdiscard_co_entry(&rwco); 281261007b31SStefan Hajnoczi } else { 28130c51a893SEric Blake co = qemu_coroutine_create(bdrv_pdiscard_co_entry, &rwco); 28140b9fd3f4SFam Zheng bdrv_coroutine_enter(child->bs, co); 28150b9fd3f4SFam Zheng BDRV_POLL_WHILE(child->bs, rwco.ret == NOT_DONE); 281661007b31SStefan Hajnoczi } 281761007b31SStefan Hajnoczi 281861007b31SStefan Hajnoczi return rwco.ret; 281961007b31SStefan Hajnoczi } 282061007b31SStefan Hajnoczi 282148af776aSKevin Wolf int bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf) 282261007b31SStefan Hajnoczi { 282361007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 28245c5ae76aSFam Zheng CoroutineIOCompletion co = { 28255c5ae76aSFam Zheng .coroutine = qemu_coroutine_self(), 28265c5ae76aSFam Zheng }; 28275c5ae76aSFam Zheng BlockAIOCB *acb; 282861007b31SStefan Hajnoczi 282999723548SPaolo Bonzini bdrv_inc_in_flight(bs); 283016a389dcSKevin Wolf if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) { 28315c5ae76aSFam Zheng co.ret = -ENOTSUP; 28325c5ae76aSFam Zheng goto out; 28335c5ae76aSFam Zheng } 28345c5ae76aSFam Zheng 283516a389dcSKevin Wolf if (drv->bdrv_co_ioctl) { 283616a389dcSKevin Wolf co.ret = drv->bdrv_co_ioctl(bs, req, buf); 283716a389dcSKevin Wolf } else { 28385c5ae76aSFam Zheng acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co); 28395c5ae76aSFam Zheng if (!acb) { 2840c8a9fd80SFam Zheng co.ret = -ENOTSUP; 2841c8a9fd80SFam Zheng goto out; 28425c5ae76aSFam Zheng } 28435c5ae76aSFam Zheng qemu_coroutine_yield(); 284416a389dcSKevin Wolf } 28455c5ae76aSFam Zheng out: 284699723548SPaolo Bonzini bdrv_dec_in_flight(bs); 28475c5ae76aSFam Zheng return co.ret; 28485c5ae76aSFam Zheng } 28495c5ae76aSFam Zheng 285061007b31SStefan Hajnoczi void *qemu_blockalign(BlockDriverState *bs, size_t size) 285161007b31SStefan Hajnoczi { 285261007b31SStefan Hajnoczi return qemu_memalign(bdrv_opt_mem_align(bs), size); 285361007b31SStefan Hajnoczi } 285461007b31SStefan Hajnoczi 285561007b31SStefan Hajnoczi void *qemu_blockalign0(BlockDriverState *bs, size_t size) 285661007b31SStefan Hajnoczi { 285761007b31SStefan Hajnoczi return memset(qemu_blockalign(bs, size), 0, size); 285861007b31SStefan Hajnoczi } 285961007b31SStefan Hajnoczi 286061007b31SStefan Hajnoczi void *qemu_try_blockalign(BlockDriverState *bs, size_t size) 286161007b31SStefan Hajnoczi { 286261007b31SStefan Hajnoczi size_t align = bdrv_opt_mem_align(bs); 286361007b31SStefan Hajnoczi 286461007b31SStefan Hajnoczi /* Ensure that NULL is never returned on success */ 286561007b31SStefan Hajnoczi assert(align > 0); 286661007b31SStefan Hajnoczi if (size == 0) { 286761007b31SStefan Hajnoczi size = align; 286861007b31SStefan Hajnoczi } 286961007b31SStefan Hajnoczi 287061007b31SStefan Hajnoczi return qemu_try_memalign(align, size); 287161007b31SStefan Hajnoczi } 287261007b31SStefan Hajnoczi 287361007b31SStefan Hajnoczi void *qemu_try_blockalign0(BlockDriverState *bs, size_t size) 287461007b31SStefan Hajnoczi { 287561007b31SStefan Hajnoczi void *mem = qemu_try_blockalign(bs, size); 287661007b31SStefan Hajnoczi 287761007b31SStefan Hajnoczi if (mem) { 287861007b31SStefan Hajnoczi memset(mem, 0, size); 287961007b31SStefan Hajnoczi } 288061007b31SStefan Hajnoczi 288161007b31SStefan Hajnoczi return mem; 288261007b31SStefan Hajnoczi } 288361007b31SStefan Hajnoczi 288461007b31SStefan Hajnoczi /* 288561007b31SStefan Hajnoczi * Check if all memory in this vector is sector aligned. 288661007b31SStefan Hajnoczi */ 288761007b31SStefan Hajnoczi bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov) 288861007b31SStefan Hajnoczi { 288961007b31SStefan Hajnoczi int i; 28904196d2f0SDenis V. Lunev size_t alignment = bdrv_min_mem_align(bs); 289161007b31SStefan Hajnoczi 289261007b31SStefan Hajnoczi for (i = 0; i < qiov->niov; i++) { 289361007b31SStefan Hajnoczi if ((uintptr_t) qiov->iov[i].iov_base % alignment) { 289461007b31SStefan Hajnoczi return false; 289561007b31SStefan Hajnoczi } 289661007b31SStefan Hajnoczi if (qiov->iov[i].iov_len % alignment) { 289761007b31SStefan Hajnoczi return false; 289861007b31SStefan Hajnoczi } 289961007b31SStefan Hajnoczi } 290061007b31SStefan Hajnoczi 290161007b31SStefan Hajnoczi return true; 290261007b31SStefan Hajnoczi } 290361007b31SStefan Hajnoczi 290461007b31SStefan Hajnoczi void bdrv_add_before_write_notifier(BlockDriverState *bs, 290561007b31SStefan Hajnoczi NotifierWithReturn *notifier) 290661007b31SStefan Hajnoczi { 290761007b31SStefan Hajnoczi notifier_with_return_list_add(&bs->before_write_notifiers, notifier); 290861007b31SStefan Hajnoczi } 290961007b31SStefan Hajnoczi 291061007b31SStefan Hajnoczi void bdrv_io_plug(BlockDriverState *bs) 291161007b31SStefan Hajnoczi { 29126b98bd64SPaolo Bonzini BdrvChild *child; 29136b98bd64SPaolo Bonzini 29146b98bd64SPaolo Bonzini QLIST_FOREACH(child, &bs->children, next) { 29156b98bd64SPaolo Bonzini bdrv_io_plug(child->bs); 29166b98bd64SPaolo Bonzini } 29176b98bd64SPaolo Bonzini 2918850d54a2SPaolo Bonzini if (atomic_fetch_inc(&bs->io_plugged) == 0) { 291961007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 292061007b31SStefan Hajnoczi if (drv && drv->bdrv_io_plug) { 292161007b31SStefan Hajnoczi drv->bdrv_io_plug(bs); 29226b98bd64SPaolo Bonzini } 292361007b31SStefan Hajnoczi } 292461007b31SStefan Hajnoczi } 292561007b31SStefan Hajnoczi 292661007b31SStefan Hajnoczi void bdrv_io_unplug(BlockDriverState *bs) 292761007b31SStefan Hajnoczi { 29286b98bd64SPaolo Bonzini BdrvChild *child; 29296b98bd64SPaolo Bonzini 29306b98bd64SPaolo Bonzini assert(bs->io_plugged); 2931850d54a2SPaolo Bonzini if (atomic_fetch_dec(&bs->io_plugged) == 1) { 293261007b31SStefan Hajnoczi BlockDriver *drv = bs->drv; 293361007b31SStefan Hajnoczi if (drv && drv->bdrv_io_unplug) { 293461007b31SStefan Hajnoczi drv->bdrv_io_unplug(bs); 293561007b31SStefan Hajnoczi } 293661007b31SStefan Hajnoczi } 293761007b31SStefan Hajnoczi 29386b98bd64SPaolo Bonzini QLIST_FOREACH(child, &bs->children, next) { 29396b98bd64SPaolo Bonzini bdrv_io_unplug(child->bs); 29406b98bd64SPaolo Bonzini } 29416b98bd64SPaolo Bonzini } 294223d0ba93SFam Zheng 294323d0ba93SFam Zheng void bdrv_register_buf(BlockDriverState *bs, void *host, size_t size) 294423d0ba93SFam Zheng { 294523d0ba93SFam Zheng BdrvChild *child; 294623d0ba93SFam Zheng 294723d0ba93SFam Zheng if (bs->drv && bs->drv->bdrv_register_buf) { 294823d0ba93SFam Zheng bs->drv->bdrv_register_buf(bs, host, size); 294923d0ba93SFam Zheng } 295023d0ba93SFam Zheng QLIST_FOREACH(child, &bs->children, next) { 295123d0ba93SFam Zheng bdrv_register_buf(child->bs, host, size); 295223d0ba93SFam Zheng } 295323d0ba93SFam Zheng } 295423d0ba93SFam Zheng 295523d0ba93SFam Zheng void bdrv_unregister_buf(BlockDriverState *bs, void *host) 295623d0ba93SFam Zheng { 295723d0ba93SFam Zheng BdrvChild *child; 295823d0ba93SFam Zheng 295923d0ba93SFam Zheng if (bs->drv && bs->drv->bdrv_unregister_buf) { 296023d0ba93SFam Zheng bs->drv->bdrv_unregister_buf(bs, host); 296123d0ba93SFam Zheng } 296223d0ba93SFam Zheng QLIST_FOREACH(child, &bs->children, next) { 296323d0ba93SFam Zheng bdrv_unregister_buf(child->bs, host); 296423d0ba93SFam Zheng } 296523d0ba93SFam Zheng } 2966fcc67678SFam Zheng 296767b51fb9SVladimir Sementsov-Ogievskiy static int coroutine_fn bdrv_co_copy_range_internal( 296867b51fb9SVladimir Sementsov-Ogievskiy BdrvChild *src, uint64_t src_offset, BdrvChild *dst, 296967b51fb9SVladimir Sementsov-Ogievskiy uint64_t dst_offset, uint64_t bytes, 297067b51fb9SVladimir Sementsov-Ogievskiy BdrvRequestFlags read_flags, BdrvRequestFlags write_flags, 2971fcc67678SFam Zheng bool recurse_src) 2972fcc67678SFam Zheng { 2973999658a0SVladimir Sementsov-Ogievskiy BdrvTrackedRequest req; 2974fcc67678SFam Zheng int ret; 2975fcc67678SFam Zheng 2976fe0480d6SKevin Wolf /* TODO We can support BDRV_REQ_NO_FALLBACK here */ 2977fe0480d6SKevin Wolf assert(!(read_flags & BDRV_REQ_NO_FALLBACK)); 2978fe0480d6SKevin Wolf assert(!(write_flags & BDRV_REQ_NO_FALLBACK)); 2979fe0480d6SKevin Wolf 2980d4d3e5a0SFam Zheng if (!dst || !dst->bs) { 2981fcc67678SFam Zheng return -ENOMEDIUM; 2982fcc67678SFam Zheng } 2983fcc67678SFam Zheng ret = bdrv_check_byte_request(dst->bs, dst_offset, bytes); 2984fcc67678SFam Zheng if (ret) { 2985fcc67678SFam Zheng return ret; 2986fcc67678SFam Zheng } 298767b51fb9SVladimir Sementsov-Ogievskiy if (write_flags & BDRV_REQ_ZERO_WRITE) { 298867b51fb9SVladimir Sementsov-Ogievskiy return bdrv_co_pwrite_zeroes(dst, dst_offset, bytes, write_flags); 2989fcc67678SFam Zheng } 2990fcc67678SFam Zheng 2991d4d3e5a0SFam Zheng if (!src || !src->bs) { 2992d4d3e5a0SFam Zheng return -ENOMEDIUM; 2993d4d3e5a0SFam Zheng } 2994d4d3e5a0SFam Zheng ret = bdrv_check_byte_request(src->bs, src_offset, bytes); 2995d4d3e5a0SFam Zheng if (ret) { 2996d4d3e5a0SFam Zheng return ret; 2997d4d3e5a0SFam Zheng } 2998d4d3e5a0SFam Zheng 2999fcc67678SFam Zheng if (!src->bs->drv->bdrv_co_copy_range_from 3000fcc67678SFam Zheng || !dst->bs->drv->bdrv_co_copy_range_to 3001fcc67678SFam Zheng || src->bs->encrypted || dst->bs->encrypted) { 3002fcc67678SFam Zheng return -ENOTSUP; 3003fcc67678SFam Zheng } 3004999658a0SVladimir Sementsov-Ogievskiy 3005999658a0SVladimir Sementsov-Ogievskiy if (recurse_src) { 3006d4d3e5a0SFam Zheng bdrv_inc_in_flight(src->bs); 3007999658a0SVladimir Sementsov-Ogievskiy tracked_request_begin(&req, src->bs, src_offset, bytes, 3008999658a0SVladimir Sementsov-Ogievskiy BDRV_TRACKED_READ); 300937aec7d7SFam Zheng 301009d2f948SVladimir Sementsov-Ogievskiy /* BDRV_REQ_SERIALISING is only for write operation */ 301109d2f948SVladimir Sementsov-Ogievskiy assert(!(read_flags & BDRV_REQ_SERIALISING)); 301267b51fb9SVladimir Sementsov-Ogievskiy if (!(read_flags & BDRV_REQ_NO_SERIALISING)) { 3013999658a0SVladimir Sementsov-Ogievskiy wait_serialising_requests(&req); 3014dee12de8SFam Zheng } 3015999658a0SVladimir Sementsov-Ogievskiy 301637aec7d7SFam Zheng ret = src->bs->drv->bdrv_co_copy_range_from(src->bs, 3017fcc67678SFam Zheng src, src_offset, 3018fcc67678SFam Zheng dst, dst_offset, 301967b51fb9SVladimir Sementsov-Ogievskiy bytes, 302067b51fb9SVladimir Sementsov-Ogievskiy read_flags, write_flags); 3021999658a0SVladimir Sementsov-Ogievskiy 3022999658a0SVladimir Sementsov-Ogievskiy tracked_request_end(&req); 3023999658a0SVladimir Sementsov-Ogievskiy bdrv_dec_in_flight(src->bs); 3024fcc67678SFam Zheng } else { 3025999658a0SVladimir Sementsov-Ogievskiy bdrv_inc_in_flight(dst->bs); 3026999658a0SVladimir Sementsov-Ogievskiy tracked_request_begin(&req, dst->bs, dst_offset, bytes, 3027999658a0SVladimir Sementsov-Ogievskiy BDRV_TRACKED_WRITE); 30280eb1e891SFam Zheng ret = bdrv_co_write_req_prepare(dst, dst_offset, bytes, &req, 30290eb1e891SFam Zheng write_flags); 30300eb1e891SFam Zheng if (!ret) { 303137aec7d7SFam Zheng ret = dst->bs->drv->bdrv_co_copy_range_to(dst->bs, 3032fcc67678SFam Zheng src, src_offset, 3033fcc67678SFam Zheng dst, dst_offset, 303467b51fb9SVladimir Sementsov-Ogievskiy bytes, 303567b51fb9SVladimir Sementsov-Ogievskiy read_flags, write_flags); 30360eb1e891SFam Zheng } 30370eb1e891SFam Zheng bdrv_co_write_req_finish(dst, dst_offset, bytes, &req, ret); 3038999658a0SVladimir Sementsov-Ogievskiy tracked_request_end(&req); 3039d4d3e5a0SFam Zheng bdrv_dec_in_flight(dst->bs); 3040999658a0SVladimir Sementsov-Ogievskiy } 3041999658a0SVladimir Sementsov-Ogievskiy 304237aec7d7SFam Zheng return ret; 3043fcc67678SFam Zheng } 3044fcc67678SFam Zheng 3045fcc67678SFam Zheng /* Copy range from @src to @dst. 3046fcc67678SFam Zheng * 3047fcc67678SFam Zheng * See the comment of bdrv_co_copy_range for the parameter and return value 3048fcc67678SFam Zheng * semantics. */ 3049fcc67678SFam Zheng int coroutine_fn bdrv_co_copy_range_from(BdrvChild *src, uint64_t src_offset, 3050fcc67678SFam Zheng BdrvChild *dst, uint64_t dst_offset, 305167b51fb9SVladimir Sementsov-Ogievskiy uint64_t bytes, 305267b51fb9SVladimir Sementsov-Ogievskiy BdrvRequestFlags read_flags, 305367b51fb9SVladimir Sementsov-Ogievskiy BdrvRequestFlags write_flags) 3054fcc67678SFam Zheng { 3055ecc983a5SFam Zheng trace_bdrv_co_copy_range_from(src, src_offset, dst, dst_offset, bytes, 3056ecc983a5SFam Zheng read_flags, write_flags); 3057fcc67678SFam Zheng return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset, 305867b51fb9SVladimir Sementsov-Ogievskiy bytes, read_flags, write_flags, true); 3059fcc67678SFam Zheng } 3060fcc67678SFam Zheng 3061fcc67678SFam Zheng /* Copy range from @src to @dst. 3062fcc67678SFam Zheng * 3063fcc67678SFam Zheng * See the comment of bdrv_co_copy_range for the parameter and return value 3064fcc67678SFam Zheng * semantics. */ 3065fcc67678SFam Zheng int coroutine_fn bdrv_co_copy_range_to(BdrvChild *src, uint64_t src_offset, 3066fcc67678SFam Zheng BdrvChild *dst, uint64_t dst_offset, 306767b51fb9SVladimir Sementsov-Ogievskiy uint64_t bytes, 306867b51fb9SVladimir Sementsov-Ogievskiy BdrvRequestFlags read_flags, 306967b51fb9SVladimir Sementsov-Ogievskiy BdrvRequestFlags write_flags) 3070fcc67678SFam Zheng { 3071ecc983a5SFam Zheng trace_bdrv_co_copy_range_to(src, src_offset, dst, dst_offset, bytes, 3072ecc983a5SFam Zheng read_flags, write_flags); 3073fcc67678SFam Zheng return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset, 307467b51fb9SVladimir Sementsov-Ogievskiy bytes, read_flags, write_flags, false); 3075fcc67678SFam Zheng } 3076fcc67678SFam Zheng 3077fcc67678SFam Zheng int coroutine_fn bdrv_co_copy_range(BdrvChild *src, uint64_t src_offset, 3078fcc67678SFam Zheng BdrvChild *dst, uint64_t dst_offset, 307967b51fb9SVladimir Sementsov-Ogievskiy uint64_t bytes, BdrvRequestFlags read_flags, 308067b51fb9SVladimir Sementsov-Ogievskiy BdrvRequestFlags write_flags) 3081fcc67678SFam Zheng { 308237aec7d7SFam Zheng return bdrv_co_copy_range_from(src, src_offset, 3083fcc67678SFam Zheng dst, dst_offset, 308467b51fb9SVladimir Sementsov-Ogievskiy bytes, read_flags, write_flags); 3085fcc67678SFam Zheng } 30863d9f2d2aSKevin Wolf 30873d9f2d2aSKevin Wolf static void bdrv_parent_cb_resize(BlockDriverState *bs) 30883d9f2d2aSKevin Wolf { 30893d9f2d2aSKevin Wolf BdrvChild *c; 30903d9f2d2aSKevin Wolf QLIST_FOREACH(c, &bs->parents, next_parent) { 30913d9f2d2aSKevin Wolf if (c->role->resize) { 30923d9f2d2aSKevin Wolf c->role->resize(c); 30933d9f2d2aSKevin Wolf } 30943d9f2d2aSKevin Wolf } 30953d9f2d2aSKevin Wolf } 30963d9f2d2aSKevin Wolf 30973d9f2d2aSKevin Wolf /** 30983d9f2d2aSKevin Wolf * Truncate file to 'offset' bytes (needed only for file protocols) 30993d9f2d2aSKevin Wolf */ 31003d9f2d2aSKevin Wolf int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset, 31013d9f2d2aSKevin Wolf PreallocMode prealloc, Error **errp) 31023d9f2d2aSKevin Wolf { 31033d9f2d2aSKevin Wolf BlockDriverState *bs = child->bs; 31043d9f2d2aSKevin Wolf BlockDriver *drv = bs->drv; 31051bc5f09fSKevin Wolf BdrvTrackedRequest req; 31061bc5f09fSKevin Wolf int64_t old_size, new_bytes; 31073d9f2d2aSKevin Wolf int ret; 31083d9f2d2aSKevin Wolf 31093d9f2d2aSKevin Wolf 31103d9f2d2aSKevin Wolf /* if bs->drv == NULL, bs is closed, so there's nothing to do here */ 31113d9f2d2aSKevin Wolf if (!drv) { 31123d9f2d2aSKevin Wolf error_setg(errp, "No medium inserted"); 31133d9f2d2aSKevin Wolf return -ENOMEDIUM; 31143d9f2d2aSKevin Wolf } 31153d9f2d2aSKevin Wolf if (offset < 0) { 31163d9f2d2aSKevin Wolf error_setg(errp, "Image size cannot be negative"); 31173d9f2d2aSKevin Wolf return -EINVAL; 31183d9f2d2aSKevin Wolf } 31193d9f2d2aSKevin Wolf 31201bc5f09fSKevin Wolf old_size = bdrv_getlength(bs); 31211bc5f09fSKevin Wolf if (old_size < 0) { 31221bc5f09fSKevin Wolf error_setg_errno(errp, -old_size, "Failed to get old image size"); 31231bc5f09fSKevin Wolf return old_size; 31241bc5f09fSKevin Wolf } 31251bc5f09fSKevin Wolf 31261bc5f09fSKevin Wolf if (offset > old_size) { 31271bc5f09fSKevin Wolf new_bytes = offset - old_size; 31281bc5f09fSKevin Wolf } else { 31291bc5f09fSKevin Wolf new_bytes = 0; 31301bc5f09fSKevin Wolf } 31311bc5f09fSKevin Wolf 31323d9f2d2aSKevin Wolf bdrv_inc_in_flight(bs); 31335416a11eSFam Zheng tracked_request_begin(&req, bs, offset - new_bytes, new_bytes, 31345416a11eSFam Zheng BDRV_TRACKED_TRUNCATE); 31351bc5f09fSKevin Wolf 31361bc5f09fSKevin Wolf /* If we are growing the image and potentially using preallocation for the 31371bc5f09fSKevin Wolf * new area, we need to make sure that no write requests are made to it 31381bc5f09fSKevin Wolf * concurrently or they might be overwritten by preallocation. */ 31391bc5f09fSKevin Wolf if (new_bytes) { 31401bc5f09fSKevin Wolf mark_request_serialising(&req, 1); 3141cd47d792SFam Zheng } 3142cd47d792SFam Zheng if (bs->read_only) { 3143cd47d792SFam Zheng error_setg(errp, "Image is read-only"); 3144cd47d792SFam Zheng ret = -EACCES; 3145cd47d792SFam Zheng goto out; 3146cd47d792SFam Zheng } 3147cd47d792SFam Zheng ret = bdrv_co_write_req_prepare(child, offset - new_bytes, new_bytes, &req, 3148cd47d792SFam Zheng 0); 3149cd47d792SFam Zheng if (ret < 0) { 3150cd47d792SFam Zheng error_setg_errno(errp, -ret, 3151cd47d792SFam Zheng "Failed to prepare request for truncation"); 3152cd47d792SFam Zheng goto out; 31531bc5f09fSKevin Wolf } 31543d9f2d2aSKevin Wolf 31553d9f2d2aSKevin Wolf if (!drv->bdrv_co_truncate) { 31563d9f2d2aSKevin Wolf if (bs->file && drv->is_filter) { 31573d9f2d2aSKevin Wolf ret = bdrv_co_truncate(bs->file, offset, prealloc, errp); 31583d9f2d2aSKevin Wolf goto out; 31593d9f2d2aSKevin Wolf } 31603d9f2d2aSKevin Wolf error_setg(errp, "Image format driver does not support resize"); 31613d9f2d2aSKevin Wolf ret = -ENOTSUP; 31623d9f2d2aSKevin Wolf goto out; 31633d9f2d2aSKevin Wolf } 31643d9f2d2aSKevin Wolf 31653d9f2d2aSKevin Wolf ret = drv->bdrv_co_truncate(bs, offset, prealloc, errp); 31663d9f2d2aSKevin Wolf if (ret < 0) { 31673d9f2d2aSKevin Wolf goto out; 31683d9f2d2aSKevin Wolf } 31693d9f2d2aSKevin Wolf ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS); 31703d9f2d2aSKevin Wolf if (ret < 0) { 31713d9f2d2aSKevin Wolf error_setg_errno(errp, -ret, "Could not refresh total sector count"); 31723d9f2d2aSKevin Wolf } else { 31733d9f2d2aSKevin Wolf offset = bs->total_sectors * BDRV_SECTOR_SIZE; 31743d9f2d2aSKevin Wolf } 3175cd47d792SFam Zheng /* It's possible that truncation succeeded but refresh_total_sectors 3176cd47d792SFam Zheng * failed, but the latter doesn't affect how we should finish the request. 3177cd47d792SFam Zheng * Pass 0 as the last parameter so that dirty bitmaps etc. are handled. */ 3178cd47d792SFam Zheng bdrv_co_write_req_finish(child, offset - new_bytes, new_bytes, &req, 0); 31793d9f2d2aSKevin Wolf 31803d9f2d2aSKevin Wolf out: 31811bc5f09fSKevin Wolf tracked_request_end(&req); 31823d9f2d2aSKevin Wolf bdrv_dec_in_flight(bs); 31831bc5f09fSKevin Wolf 31843d9f2d2aSKevin Wolf return ret; 31853d9f2d2aSKevin Wolf } 31863d9f2d2aSKevin Wolf 31873d9f2d2aSKevin Wolf typedef struct TruncateCo { 31883d9f2d2aSKevin Wolf BdrvChild *child; 31893d9f2d2aSKevin Wolf int64_t offset; 31903d9f2d2aSKevin Wolf PreallocMode prealloc; 31913d9f2d2aSKevin Wolf Error **errp; 31923d9f2d2aSKevin Wolf int ret; 31933d9f2d2aSKevin Wolf } TruncateCo; 31943d9f2d2aSKevin Wolf 31953d9f2d2aSKevin Wolf static void coroutine_fn bdrv_truncate_co_entry(void *opaque) 31963d9f2d2aSKevin Wolf { 31973d9f2d2aSKevin Wolf TruncateCo *tco = opaque; 31983d9f2d2aSKevin Wolf tco->ret = bdrv_co_truncate(tco->child, tco->offset, tco->prealloc, 31993d9f2d2aSKevin Wolf tco->errp); 32004720cbeeSKevin Wolf aio_wait_kick(); 32013d9f2d2aSKevin Wolf } 32023d9f2d2aSKevin Wolf 32033d9f2d2aSKevin Wolf int bdrv_truncate(BdrvChild *child, int64_t offset, PreallocMode prealloc, 32043d9f2d2aSKevin Wolf Error **errp) 32053d9f2d2aSKevin Wolf { 32063d9f2d2aSKevin Wolf Coroutine *co; 32073d9f2d2aSKevin Wolf TruncateCo tco = { 32083d9f2d2aSKevin Wolf .child = child, 32093d9f2d2aSKevin Wolf .offset = offset, 32103d9f2d2aSKevin Wolf .prealloc = prealloc, 32113d9f2d2aSKevin Wolf .errp = errp, 32123d9f2d2aSKevin Wolf .ret = NOT_DONE, 32133d9f2d2aSKevin Wolf }; 32143d9f2d2aSKevin Wolf 32153d9f2d2aSKevin Wolf if (qemu_in_coroutine()) { 32163d9f2d2aSKevin Wolf /* Fast-path if already in coroutine context */ 32173d9f2d2aSKevin Wolf bdrv_truncate_co_entry(&tco); 32183d9f2d2aSKevin Wolf } else { 32193d9f2d2aSKevin Wolf co = qemu_coroutine_create(bdrv_truncate_co_entry, &tco); 32204720cbeeSKevin Wolf bdrv_coroutine_enter(child->bs, co); 32213d9f2d2aSKevin Wolf BDRV_POLL_WHILE(child->bs, tco.ret == NOT_DONE); 32223d9f2d2aSKevin Wolf } 32233d9f2d2aSKevin Wolf 32243d9f2d2aSKevin Wolf return tco.ret; 32253d9f2d2aSKevin Wolf } 3226