1893f7ebaSPaolo Bonzini /* 2893f7ebaSPaolo Bonzini * Image mirroring 3893f7ebaSPaolo Bonzini * 4893f7ebaSPaolo Bonzini * Copyright Red Hat, Inc. 2012 5893f7ebaSPaolo Bonzini * 6893f7ebaSPaolo Bonzini * Authors: 7893f7ebaSPaolo Bonzini * Paolo Bonzini <pbonzini@redhat.com> 8893f7ebaSPaolo Bonzini * 9893f7ebaSPaolo Bonzini * This work is licensed under the terms of the GNU LGPL, version 2 or later. 10893f7ebaSPaolo Bonzini * See the COPYING.LIB file in the top-level directory. 11893f7ebaSPaolo Bonzini * 12893f7ebaSPaolo Bonzini */ 13893f7ebaSPaolo Bonzini 14893f7ebaSPaolo Bonzini #include "trace.h" 15737e150eSPaolo Bonzini #include "block/blockjob.h" 16737e150eSPaolo Bonzini #include "block/block_int.h" 17893f7ebaSPaolo Bonzini #include "qemu/ratelimit.h" 18b812f671SPaolo Bonzini #include "qemu/bitmap.h" 19893f7ebaSPaolo Bonzini 20893f7ebaSPaolo Bonzini #define SLICE_TIME 100000000ULL /* ns */ 21402a4741SPaolo Bonzini #define MAX_IN_FLIGHT 16 22402a4741SPaolo Bonzini 23402a4741SPaolo Bonzini /* The mirroring buffer is a list of granularity-sized chunks. 24402a4741SPaolo Bonzini * Free chunks are organized in a list. 25402a4741SPaolo Bonzini */ 26402a4741SPaolo Bonzini typedef struct MirrorBuffer { 27402a4741SPaolo Bonzini QSIMPLEQ_ENTRY(MirrorBuffer) next; 28402a4741SPaolo Bonzini } MirrorBuffer; 29893f7ebaSPaolo Bonzini 30893f7ebaSPaolo Bonzini typedef struct MirrorBlockJob { 31893f7ebaSPaolo Bonzini BlockJob common; 32893f7ebaSPaolo Bonzini RateLimit limit; 33893f7ebaSPaolo Bonzini BlockDriverState *target; 345bc361b8SFam Zheng BlockDriverState *base; 3503544a6eSFam Zheng bool is_none_mode; 36b952b558SPaolo Bonzini BlockdevOnError on_source_error, on_target_error; 37d63ffd87SPaolo Bonzini bool synced; 38d63ffd87SPaolo Bonzini bool should_complete; 39893f7ebaSPaolo Bonzini int64_t sector_num; 40eee13dfeSPaolo Bonzini int64_t granularity; 41b812f671SPaolo Bonzini size_t buf_size; 42b812f671SPaolo Bonzini unsigned long *cow_bitmap; 43e4654d2dSFam Zheng BdrvDirtyBitmap *dirty_bitmap; 448f0720ecSPaolo Bonzini HBitmapIter hbi; 45893f7ebaSPaolo Bonzini uint8_t *buf; 46402a4741SPaolo Bonzini QSIMPLEQ_HEAD(, MirrorBuffer) buf_free; 47402a4741SPaolo Bonzini int buf_free_count; 48bd48bde8SPaolo Bonzini 49402a4741SPaolo Bonzini unsigned long *in_flight_bitmap; 50bd48bde8SPaolo Bonzini int in_flight; 51bd48bde8SPaolo Bonzini int ret; 52893f7ebaSPaolo Bonzini } MirrorBlockJob; 53893f7ebaSPaolo Bonzini 54bd48bde8SPaolo Bonzini typedef struct MirrorOp { 55bd48bde8SPaolo Bonzini MirrorBlockJob *s; 56bd48bde8SPaolo Bonzini QEMUIOVector qiov; 57bd48bde8SPaolo Bonzini int64_t sector_num; 58bd48bde8SPaolo Bonzini int nb_sectors; 59bd48bde8SPaolo Bonzini } MirrorOp; 60bd48bde8SPaolo Bonzini 61b952b558SPaolo Bonzini static BlockErrorAction mirror_error_action(MirrorBlockJob *s, bool read, 62b952b558SPaolo Bonzini int error) 63b952b558SPaolo Bonzini { 64b952b558SPaolo Bonzini s->synced = false; 65b952b558SPaolo Bonzini if (read) { 66b952b558SPaolo Bonzini return block_job_error_action(&s->common, s->common.bs, 67b952b558SPaolo Bonzini s->on_source_error, true, error); 68b952b558SPaolo Bonzini } else { 69b952b558SPaolo Bonzini return block_job_error_action(&s->common, s->target, 70b952b558SPaolo Bonzini s->on_target_error, false, error); 71b952b558SPaolo Bonzini } 72b952b558SPaolo Bonzini } 73b952b558SPaolo Bonzini 74bd48bde8SPaolo Bonzini static void mirror_iteration_done(MirrorOp *op, int ret) 75bd48bde8SPaolo Bonzini { 76bd48bde8SPaolo Bonzini MirrorBlockJob *s = op->s; 77402a4741SPaolo Bonzini struct iovec *iov; 78bd48bde8SPaolo Bonzini int64_t chunk_num; 79402a4741SPaolo Bonzini int i, nb_chunks, sectors_per_chunk; 80bd48bde8SPaolo Bonzini 81bd48bde8SPaolo Bonzini trace_mirror_iteration_done(s, op->sector_num, op->nb_sectors, ret); 82bd48bde8SPaolo Bonzini 83bd48bde8SPaolo Bonzini s->in_flight--; 84402a4741SPaolo Bonzini iov = op->qiov.iov; 85402a4741SPaolo Bonzini for (i = 0; i < op->qiov.niov; i++) { 86402a4741SPaolo Bonzini MirrorBuffer *buf = (MirrorBuffer *) iov[i].iov_base; 87402a4741SPaolo Bonzini QSIMPLEQ_INSERT_TAIL(&s->buf_free, buf, next); 88402a4741SPaolo Bonzini s->buf_free_count++; 89402a4741SPaolo Bonzini } 90402a4741SPaolo Bonzini 91bd48bde8SPaolo Bonzini sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS; 92bd48bde8SPaolo Bonzini chunk_num = op->sector_num / sectors_per_chunk; 93bd48bde8SPaolo Bonzini nb_chunks = op->nb_sectors / sectors_per_chunk; 94402a4741SPaolo Bonzini bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks); 95bd48bde8SPaolo Bonzini if (s->cow_bitmap && ret >= 0) { 96bd48bde8SPaolo Bonzini bitmap_set(s->cow_bitmap, chunk_num, nb_chunks); 97bd48bde8SPaolo Bonzini } 98bd48bde8SPaolo Bonzini 996df3bf8eSZhang Min qemu_iovec_destroy(&op->qiov); 100bd48bde8SPaolo Bonzini g_slice_free(MirrorOp, op); 101bd48bde8SPaolo Bonzini qemu_coroutine_enter(s->common.co, NULL); 102bd48bde8SPaolo Bonzini } 103bd48bde8SPaolo Bonzini 104bd48bde8SPaolo Bonzini static void mirror_write_complete(void *opaque, int ret) 105bd48bde8SPaolo Bonzini { 106bd48bde8SPaolo Bonzini MirrorOp *op = opaque; 107bd48bde8SPaolo Bonzini MirrorBlockJob *s = op->s; 108bd48bde8SPaolo Bonzini if (ret < 0) { 109bd48bde8SPaolo Bonzini BlockDriverState *source = s->common.bs; 110bd48bde8SPaolo Bonzini BlockErrorAction action; 111bd48bde8SPaolo Bonzini 112bd48bde8SPaolo Bonzini bdrv_set_dirty(source, op->sector_num, op->nb_sectors); 113bd48bde8SPaolo Bonzini action = mirror_error_action(s, false, -ret); 114bd48bde8SPaolo Bonzini if (action == BDRV_ACTION_REPORT && s->ret >= 0) { 115bd48bde8SPaolo Bonzini s->ret = ret; 116bd48bde8SPaolo Bonzini } 117bd48bde8SPaolo Bonzini } 118bd48bde8SPaolo Bonzini mirror_iteration_done(op, ret); 119bd48bde8SPaolo Bonzini } 120bd48bde8SPaolo Bonzini 121bd48bde8SPaolo Bonzini static void mirror_read_complete(void *opaque, int ret) 122bd48bde8SPaolo Bonzini { 123bd48bde8SPaolo Bonzini MirrorOp *op = opaque; 124bd48bde8SPaolo Bonzini MirrorBlockJob *s = op->s; 125bd48bde8SPaolo Bonzini if (ret < 0) { 126bd48bde8SPaolo Bonzini BlockDriverState *source = s->common.bs; 127bd48bde8SPaolo Bonzini BlockErrorAction action; 128bd48bde8SPaolo Bonzini 129bd48bde8SPaolo Bonzini bdrv_set_dirty(source, op->sector_num, op->nb_sectors); 130bd48bde8SPaolo Bonzini action = mirror_error_action(s, true, -ret); 131bd48bde8SPaolo Bonzini if (action == BDRV_ACTION_REPORT && s->ret >= 0) { 132bd48bde8SPaolo Bonzini s->ret = ret; 133bd48bde8SPaolo Bonzini } 134bd48bde8SPaolo Bonzini 135bd48bde8SPaolo Bonzini mirror_iteration_done(op, ret); 136bd48bde8SPaolo Bonzini return; 137bd48bde8SPaolo Bonzini } 138bd48bde8SPaolo Bonzini bdrv_aio_writev(s->target, op->sector_num, &op->qiov, op->nb_sectors, 139bd48bde8SPaolo Bonzini mirror_write_complete, op); 140bd48bde8SPaolo Bonzini } 141bd48bde8SPaolo Bonzini 142bd48bde8SPaolo Bonzini static void coroutine_fn mirror_iteration(MirrorBlockJob *s) 143893f7ebaSPaolo Bonzini { 144893f7ebaSPaolo Bonzini BlockDriverState *source = s->common.bs; 145402a4741SPaolo Bonzini int nb_sectors, sectors_per_chunk, nb_chunks; 146884fea4eSPaolo Bonzini int64_t end, sector_num, next_chunk, next_sector, hbitmap_next_sector; 147bd48bde8SPaolo Bonzini MirrorOp *op; 148893f7ebaSPaolo Bonzini 1498f0720ecSPaolo Bonzini s->sector_num = hbitmap_iter_next(&s->hbi); 1508f0720ecSPaolo Bonzini if (s->sector_num < 0) { 151e4654d2dSFam Zheng bdrv_dirty_iter_init(source, s->dirty_bitmap, &s->hbi); 1528f0720ecSPaolo Bonzini s->sector_num = hbitmap_iter_next(&s->hbi); 153e4654d2dSFam Zheng trace_mirror_restart_iter(s, 154e4654d2dSFam Zheng bdrv_get_dirty_count(source, s->dirty_bitmap)); 1558f0720ecSPaolo Bonzini assert(s->sector_num >= 0); 1568f0720ecSPaolo Bonzini } 1578f0720ecSPaolo Bonzini 158402a4741SPaolo Bonzini hbitmap_next_sector = s->sector_num; 159b812f671SPaolo Bonzini sector_num = s->sector_num; 160884fea4eSPaolo Bonzini sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS; 161884fea4eSPaolo Bonzini end = s->common.len >> BDRV_SECTOR_BITS; 162402a4741SPaolo Bonzini 163884fea4eSPaolo Bonzini /* Extend the QEMUIOVector to include all adjacent blocks that will 164884fea4eSPaolo Bonzini * be copied in this operation. 165884fea4eSPaolo Bonzini * 166884fea4eSPaolo Bonzini * We have to do this if we have no backing file yet in the destination, 167884fea4eSPaolo Bonzini * and the cluster size is very large. Then we need to do COW ourselves. 168884fea4eSPaolo Bonzini * The first time a cluster is copied, copy it entirely. Note that, 169884fea4eSPaolo Bonzini * because both the granularity and the cluster size are powers of two, 170884fea4eSPaolo Bonzini * the number of sectors to copy cannot exceed one cluster. 171884fea4eSPaolo Bonzini * 172884fea4eSPaolo Bonzini * We also want to extend the QEMUIOVector to include more adjacent 173884fea4eSPaolo Bonzini * dirty blocks if possible, to limit the number of I/O operations and 174884fea4eSPaolo Bonzini * run efficiently even with a small granularity. 175402a4741SPaolo Bonzini */ 176884fea4eSPaolo Bonzini nb_chunks = 0; 177884fea4eSPaolo Bonzini nb_sectors = 0; 178884fea4eSPaolo Bonzini next_sector = sector_num; 179884fea4eSPaolo Bonzini next_chunk = sector_num / sectors_per_chunk; 180402a4741SPaolo Bonzini 181402a4741SPaolo Bonzini /* Wait for I/O to this cluster (from a previous iteration) to be done. */ 182884fea4eSPaolo Bonzini while (test_bit(next_chunk, s->in_flight_bitmap)) { 183402a4741SPaolo Bonzini trace_mirror_yield_in_flight(s, sector_num, s->in_flight); 184402a4741SPaolo Bonzini qemu_coroutine_yield(); 185b812f671SPaolo Bonzini } 186b812f671SPaolo Bonzini 187884fea4eSPaolo Bonzini do { 188884fea4eSPaolo Bonzini int added_sectors, added_chunks; 189884fea4eSPaolo Bonzini 190e4654d2dSFam Zheng if (!bdrv_get_dirty(source, s->dirty_bitmap, next_sector) || 191884fea4eSPaolo Bonzini test_bit(next_chunk, s->in_flight_bitmap)) { 192884fea4eSPaolo Bonzini assert(nb_sectors > 0); 193884fea4eSPaolo Bonzini break; 194884fea4eSPaolo Bonzini } 195884fea4eSPaolo Bonzini 196884fea4eSPaolo Bonzini added_sectors = sectors_per_chunk; 197884fea4eSPaolo Bonzini if (s->cow_bitmap && !test_bit(next_chunk, s->cow_bitmap)) { 198884fea4eSPaolo Bonzini bdrv_round_to_clusters(s->target, 199884fea4eSPaolo Bonzini next_sector, added_sectors, 200884fea4eSPaolo Bonzini &next_sector, &added_sectors); 201884fea4eSPaolo Bonzini 202884fea4eSPaolo Bonzini /* On the first iteration, the rounding may make us copy 203884fea4eSPaolo Bonzini * sectors before the first dirty one. 204884fea4eSPaolo Bonzini */ 205884fea4eSPaolo Bonzini if (next_sector < sector_num) { 206884fea4eSPaolo Bonzini assert(nb_sectors == 0); 207884fea4eSPaolo Bonzini sector_num = next_sector; 208884fea4eSPaolo Bonzini next_chunk = next_sector / sectors_per_chunk; 209884fea4eSPaolo Bonzini } 210884fea4eSPaolo Bonzini } 211884fea4eSPaolo Bonzini 212884fea4eSPaolo Bonzini added_sectors = MIN(added_sectors, end - (sector_num + nb_sectors)); 213884fea4eSPaolo Bonzini added_chunks = (added_sectors + sectors_per_chunk - 1) / sectors_per_chunk; 214884fea4eSPaolo Bonzini 215884fea4eSPaolo Bonzini /* When doing COW, it may happen that there is not enough space for 216884fea4eSPaolo Bonzini * a full cluster. Wait if that is the case. 217884fea4eSPaolo Bonzini */ 218884fea4eSPaolo Bonzini while (nb_chunks == 0 && s->buf_free_count < added_chunks) { 219402a4741SPaolo Bonzini trace_mirror_yield_buf_busy(s, nb_chunks, s->in_flight); 220402a4741SPaolo Bonzini qemu_coroutine_yield(); 221402a4741SPaolo Bonzini } 222884fea4eSPaolo Bonzini if (s->buf_free_count < nb_chunks + added_chunks) { 223884fea4eSPaolo Bonzini trace_mirror_break_buf_busy(s, nb_chunks, s->in_flight); 224884fea4eSPaolo Bonzini break; 225884fea4eSPaolo Bonzini } 226402a4741SPaolo Bonzini 227402a4741SPaolo Bonzini /* We have enough free space to copy these sectors. */ 228884fea4eSPaolo Bonzini bitmap_set(s->in_flight_bitmap, next_chunk, added_chunks); 229884fea4eSPaolo Bonzini 230884fea4eSPaolo Bonzini nb_sectors += added_sectors; 231884fea4eSPaolo Bonzini nb_chunks += added_chunks; 232884fea4eSPaolo Bonzini next_sector += added_sectors; 233884fea4eSPaolo Bonzini next_chunk += added_chunks; 234884fea4eSPaolo Bonzini } while (next_sector < end); 235bd48bde8SPaolo Bonzini 236bd48bde8SPaolo Bonzini /* Allocate a MirrorOp that is used as an AIO callback. */ 237bd48bde8SPaolo Bonzini op = g_slice_new(MirrorOp); 238bd48bde8SPaolo Bonzini op->s = s; 239bd48bde8SPaolo Bonzini op->sector_num = sector_num; 240bd48bde8SPaolo Bonzini op->nb_sectors = nb_sectors; 241402a4741SPaolo Bonzini 242402a4741SPaolo Bonzini /* Now make a QEMUIOVector taking enough granularity-sized chunks 243402a4741SPaolo Bonzini * from s->buf_free. 244402a4741SPaolo Bonzini */ 245402a4741SPaolo Bonzini qemu_iovec_init(&op->qiov, nb_chunks); 246402a4741SPaolo Bonzini next_sector = sector_num; 247402a4741SPaolo Bonzini while (nb_chunks-- > 0) { 248402a4741SPaolo Bonzini MirrorBuffer *buf = QSIMPLEQ_FIRST(&s->buf_free); 249402a4741SPaolo Bonzini QSIMPLEQ_REMOVE_HEAD(&s->buf_free, next); 250402a4741SPaolo Bonzini s->buf_free_count--; 251402a4741SPaolo Bonzini qemu_iovec_add(&op->qiov, buf, s->granularity); 252402a4741SPaolo Bonzini 253402a4741SPaolo Bonzini /* Advance the HBitmapIter in parallel, so that we do not examine 254402a4741SPaolo Bonzini * the same sector twice. 255402a4741SPaolo Bonzini */ 256e4654d2dSFam Zheng if (next_sector > hbitmap_next_sector 257e4654d2dSFam Zheng && bdrv_get_dirty(source, s->dirty_bitmap, next_sector)) { 258402a4741SPaolo Bonzini hbitmap_next_sector = hbitmap_iter_next(&s->hbi); 259402a4741SPaolo Bonzini } 260402a4741SPaolo Bonzini 261402a4741SPaolo Bonzini next_sector += sectors_per_chunk; 262402a4741SPaolo Bonzini } 263bd48bde8SPaolo Bonzini 264b812f671SPaolo Bonzini bdrv_reset_dirty(source, sector_num, nb_sectors); 265893f7ebaSPaolo Bonzini 266893f7ebaSPaolo Bonzini /* Copy the dirty cluster. */ 267bd48bde8SPaolo Bonzini s->in_flight++; 268b812f671SPaolo Bonzini trace_mirror_one_iteration(s, sector_num, nb_sectors); 269bd48bde8SPaolo Bonzini bdrv_aio_readv(source, sector_num, &op->qiov, nb_sectors, 270bd48bde8SPaolo Bonzini mirror_read_complete, op); 271893f7ebaSPaolo Bonzini } 272b952b558SPaolo Bonzini 273402a4741SPaolo Bonzini static void mirror_free_init(MirrorBlockJob *s) 274402a4741SPaolo Bonzini { 275402a4741SPaolo Bonzini int granularity = s->granularity; 276402a4741SPaolo Bonzini size_t buf_size = s->buf_size; 277402a4741SPaolo Bonzini uint8_t *buf = s->buf; 278402a4741SPaolo Bonzini 279402a4741SPaolo Bonzini assert(s->buf_free_count == 0); 280402a4741SPaolo Bonzini QSIMPLEQ_INIT(&s->buf_free); 281402a4741SPaolo Bonzini while (buf_size != 0) { 282402a4741SPaolo Bonzini MirrorBuffer *cur = (MirrorBuffer *)buf; 283402a4741SPaolo Bonzini QSIMPLEQ_INSERT_TAIL(&s->buf_free, cur, next); 284402a4741SPaolo Bonzini s->buf_free_count++; 285402a4741SPaolo Bonzini buf_size -= granularity; 286402a4741SPaolo Bonzini buf += granularity; 287402a4741SPaolo Bonzini } 288402a4741SPaolo Bonzini } 289402a4741SPaolo Bonzini 290bd48bde8SPaolo Bonzini static void mirror_drain(MirrorBlockJob *s) 291bd48bde8SPaolo Bonzini { 292bd48bde8SPaolo Bonzini while (s->in_flight > 0) { 293bd48bde8SPaolo Bonzini qemu_coroutine_yield(); 294bd48bde8SPaolo Bonzini } 295893f7ebaSPaolo Bonzini } 296893f7ebaSPaolo Bonzini 297893f7ebaSPaolo Bonzini static void coroutine_fn mirror_run(void *opaque) 298893f7ebaSPaolo Bonzini { 299893f7ebaSPaolo Bonzini MirrorBlockJob *s = opaque; 300893f7ebaSPaolo Bonzini BlockDriverState *bs = s->common.bs; 301eee13dfeSPaolo Bonzini int64_t sector_num, end, sectors_per_chunk, length; 302bd48bde8SPaolo Bonzini uint64_t last_pause_ns; 303b812f671SPaolo Bonzini BlockDriverInfo bdi; 304b812f671SPaolo Bonzini char backing_filename[1024]; 305893f7ebaSPaolo Bonzini int ret = 0; 306893f7ebaSPaolo Bonzini int n; 307893f7ebaSPaolo Bonzini 308893f7ebaSPaolo Bonzini if (block_job_is_cancelled(&s->common)) { 309893f7ebaSPaolo Bonzini goto immediate_exit; 310893f7ebaSPaolo Bonzini } 311893f7ebaSPaolo Bonzini 312893f7ebaSPaolo Bonzini s->common.len = bdrv_getlength(bs); 31388ff0e48SPaolo Bonzini if (s->common.len <= 0) { 314893f7ebaSPaolo Bonzini block_job_completed(&s->common, s->common.len); 315893f7ebaSPaolo Bonzini return; 316893f7ebaSPaolo Bonzini } 317893f7ebaSPaolo Bonzini 318402a4741SPaolo Bonzini length = (bdrv_getlength(bs) + s->granularity - 1) / s->granularity; 319402a4741SPaolo Bonzini s->in_flight_bitmap = bitmap_new(length); 320402a4741SPaolo Bonzini 321b812f671SPaolo Bonzini /* If we have no backing file yet in the destination, we cannot let 322b812f671SPaolo Bonzini * the destination do COW. Instead, we copy sectors around the 323b812f671SPaolo Bonzini * dirty data if needed. We need a bitmap to do that. 324b812f671SPaolo Bonzini */ 325b812f671SPaolo Bonzini bdrv_get_backing_filename(s->target, backing_filename, 326b812f671SPaolo Bonzini sizeof(backing_filename)); 327b812f671SPaolo Bonzini if (backing_filename[0] && !s->target->backing_hd) { 328b812f671SPaolo Bonzini bdrv_get_info(s->target, &bdi); 329eee13dfeSPaolo Bonzini if (s->granularity < bdi.cluster_size) { 33008e4ed6cSPaolo Bonzini s->buf_size = MAX(s->buf_size, bdi.cluster_size); 331b812f671SPaolo Bonzini s->cow_bitmap = bitmap_new(length); 332b812f671SPaolo Bonzini } 333b812f671SPaolo Bonzini } 334b812f671SPaolo Bonzini 335893f7ebaSPaolo Bonzini end = s->common.len >> BDRV_SECTOR_BITS; 336b812f671SPaolo Bonzini s->buf = qemu_blockalign(bs, s->buf_size); 337eee13dfeSPaolo Bonzini sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS; 338402a4741SPaolo Bonzini mirror_free_init(s); 339893f7ebaSPaolo Bonzini 34003544a6eSFam Zheng if (!s->is_none_mode) { 341893f7ebaSPaolo Bonzini /* First part, loop on the sectors and initialize the dirty bitmap. */ 3425bc361b8SFam Zheng BlockDriverState *base = s->base; 343893f7ebaSPaolo Bonzini for (sector_num = 0; sector_num < end; ) { 344eee13dfeSPaolo Bonzini int64_t next = (sector_num | (sectors_per_chunk - 1)) + 1; 3454f578637SPaolo Bonzini ret = bdrv_is_allocated_above(bs, base, 346893f7ebaSPaolo Bonzini sector_num, next - sector_num, &n); 347893f7ebaSPaolo Bonzini 348893f7ebaSPaolo Bonzini if (ret < 0) { 349893f7ebaSPaolo Bonzini goto immediate_exit; 350893f7ebaSPaolo Bonzini } 351893f7ebaSPaolo Bonzini 352893f7ebaSPaolo Bonzini assert(n > 0); 353893f7ebaSPaolo Bonzini if (ret == 1) { 354893f7ebaSPaolo Bonzini bdrv_set_dirty(bs, sector_num, n); 355893f7ebaSPaolo Bonzini sector_num = next; 356893f7ebaSPaolo Bonzini } else { 357893f7ebaSPaolo Bonzini sector_num += n; 358893f7ebaSPaolo Bonzini } 359893f7ebaSPaolo Bonzini } 360893f7ebaSPaolo Bonzini } 361893f7ebaSPaolo Bonzini 362e4654d2dSFam Zheng bdrv_dirty_iter_init(bs, s->dirty_bitmap, &s->hbi); 363bc72ad67SAlex Bligh last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); 364893f7ebaSPaolo Bonzini for (;;) { 365893f7ebaSPaolo Bonzini uint64_t delay_ns; 366893f7ebaSPaolo Bonzini int64_t cnt; 367893f7ebaSPaolo Bonzini bool should_complete; 368893f7ebaSPaolo Bonzini 369bd48bde8SPaolo Bonzini if (s->ret < 0) { 370bd48bde8SPaolo Bonzini ret = s->ret; 371893f7ebaSPaolo Bonzini goto immediate_exit; 372893f7ebaSPaolo Bonzini } 373bd48bde8SPaolo Bonzini 374e4654d2dSFam Zheng cnt = bdrv_get_dirty_count(bs, s->dirty_bitmap); 375bd48bde8SPaolo Bonzini 376bd48bde8SPaolo Bonzini /* Note that even when no rate limit is applied we need to yield 377bd48bde8SPaolo Bonzini * periodically with no pending I/O so that qemu_aio_flush() returns. 378bd48bde8SPaolo Bonzini * We do so every SLICE_TIME nanoseconds, or when there is an error, 379bd48bde8SPaolo Bonzini * or when the source is clean, whichever comes first. 380bd48bde8SPaolo Bonzini */ 381bc72ad67SAlex Bligh if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - last_pause_ns < SLICE_TIME && 382bd48bde8SPaolo Bonzini s->common.iostatus == BLOCK_DEVICE_IO_STATUS_OK) { 383402a4741SPaolo Bonzini if (s->in_flight == MAX_IN_FLIGHT || s->buf_free_count == 0 || 384402a4741SPaolo Bonzini (cnt == 0 && s->in_flight > 0)) { 385402a4741SPaolo Bonzini trace_mirror_yield(s, s->in_flight, s->buf_free_count, cnt); 386bd48bde8SPaolo Bonzini qemu_coroutine_yield(); 387bd48bde8SPaolo Bonzini continue; 388bd48bde8SPaolo Bonzini } else if (cnt != 0) { 389bd48bde8SPaolo Bonzini mirror_iteration(s); 390bd48bde8SPaolo Bonzini continue; 391bd48bde8SPaolo Bonzini } 392893f7ebaSPaolo Bonzini } 393893f7ebaSPaolo Bonzini 394893f7ebaSPaolo Bonzini should_complete = false; 395bd48bde8SPaolo Bonzini if (s->in_flight == 0 && cnt == 0) { 396893f7ebaSPaolo Bonzini trace_mirror_before_flush(s); 397893f7ebaSPaolo Bonzini ret = bdrv_flush(s->target); 398893f7ebaSPaolo Bonzini if (ret < 0) { 399b952b558SPaolo Bonzini if (mirror_error_action(s, false, -ret) == BDRV_ACTION_REPORT) { 400893f7ebaSPaolo Bonzini goto immediate_exit; 401893f7ebaSPaolo Bonzini } 402b952b558SPaolo Bonzini } else { 403893f7ebaSPaolo Bonzini /* We're out of the streaming phase. From now on, if the job 404893f7ebaSPaolo Bonzini * is cancelled we will actually complete all pending I/O and 405893f7ebaSPaolo Bonzini * report completion. This way, block-job-cancel will leave 406893f7ebaSPaolo Bonzini * the target in a consistent state. 407893f7ebaSPaolo Bonzini */ 408893f7ebaSPaolo Bonzini s->common.offset = end * BDRV_SECTOR_SIZE; 409d63ffd87SPaolo Bonzini if (!s->synced) { 410d63ffd87SPaolo Bonzini block_job_ready(&s->common); 411d63ffd87SPaolo Bonzini s->synced = true; 412d63ffd87SPaolo Bonzini } 413d63ffd87SPaolo Bonzini 414d63ffd87SPaolo Bonzini should_complete = s->should_complete || 415d63ffd87SPaolo Bonzini block_job_is_cancelled(&s->common); 416e4654d2dSFam Zheng cnt = bdrv_get_dirty_count(bs, s->dirty_bitmap); 417893f7ebaSPaolo Bonzini } 418b952b558SPaolo Bonzini } 419893f7ebaSPaolo Bonzini 420893f7ebaSPaolo Bonzini if (cnt == 0 && should_complete) { 421893f7ebaSPaolo Bonzini /* The dirty bitmap is not updated while operations are pending. 422893f7ebaSPaolo Bonzini * If we're about to exit, wait for pending operations before 423893f7ebaSPaolo Bonzini * calling bdrv_get_dirty_count(bs), or we may exit while the 424893f7ebaSPaolo Bonzini * source has dirty data to copy! 425893f7ebaSPaolo Bonzini * 426893f7ebaSPaolo Bonzini * Note that I/O can be submitted by the guest while 427893f7ebaSPaolo Bonzini * mirror_populate runs. 428893f7ebaSPaolo Bonzini */ 429893f7ebaSPaolo Bonzini trace_mirror_before_drain(s, cnt); 430893f7ebaSPaolo Bonzini bdrv_drain_all(); 431e4654d2dSFam Zheng cnt = bdrv_get_dirty_count(bs, s->dirty_bitmap); 432893f7ebaSPaolo Bonzini } 433893f7ebaSPaolo Bonzini 434893f7ebaSPaolo Bonzini ret = 0; 435d63ffd87SPaolo Bonzini trace_mirror_before_sleep(s, cnt, s->synced); 436d63ffd87SPaolo Bonzini if (!s->synced) { 437893f7ebaSPaolo Bonzini /* Publish progress */ 438acc906c6SPaolo Bonzini s->common.offset = (end - cnt) * BDRV_SECTOR_SIZE; 439893f7ebaSPaolo Bonzini 440893f7ebaSPaolo Bonzini if (s->common.speed) { 441eee13dfeSPaolo Bonzini delay_ns = ratelimit_calculate_delay(&s->limit, sectors_per_chunk); 442893f7ebaSPaolo Bonzini } else { 443893f7ebaSPaolo Bonzini delay_ns = 0; 444893f7ebaSPaolo Bonzini } 445893f7ebaSPaolo Bonzini 4467483d1e5SAlex Bligh block_job_sleep_ns(&s->common, QEMU_CLOCK_REALTIME, delay_ns); 447893f7ebaSPaolo Bonzini if (block_job_is_cancelled(&s->common)) { 448893f7ebaSPaolo Bonzini break; 449893f7ebaSPaolo Bonzini } 450893f7ebaSPaolo Bonzini } else if (!should_complete) { 451bd48bde8SPaolo Bonzini delay_ns = (s->in_flight == 0 && cnt == 0 ? SLICE_TIME : 0); 4527483d1e5SAlex Bligh block_job_sleep_ns(&s->common, QEMU_CLOCK_REALTIME, delay_ns); 453893f7ebaSPaolo Bonzini } else if (cnt == 0) { 454893f7ebaSPaolo Bonzini /* The two disks are in sync. Exit and report successful 455893f7ebaSPaolo Bonzini * completion. 456893f7ebaSPaolo Bonzini */ 457893f7ebaSPaolo Bonzini assert(QLIST_EMPTY(&bs->tracked_requests)); 458893f7ebaSPaolo Bonzini s->common.cancelled = false; 459893f7ebaSPaolo Bonzini break; 460893f7ebaSPaolo Bonzini } 461bc72ad67SAlex Bligh last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); 462893f7ebaSPaolo Bonzini } 463893f7ebaSPaolo Bonzini 464893f7ebaSPaolo Bonzini immediate_exit: 465bd48bde8SPaolo Bonzini if (s->in_flight > 0) { 466bd48bde8SPaolo Bonzini /* We get here only if something went wrong. Either the job failed, 467bd48bde8SPaolo Bonzini * or it was cancelled prematurely so that we do not guarantee that 468bd48bde8SPaolo Bonzini * the target is a copy of the source. 469bd48bde8SPaolo Bonzini */ 470bd48bde8SPaolo Bonzini assert(ret < 0 || (!s->synced && block_job_is_cancelled(&s->common))); 471bd48bde8SPaolo Bonzini mirror_drain(s); 472bd48bde8SPaolo Bonzini } 473bd48bde8SPaolo Bonzini 474bd48bde8SPaolo Bonzini assert(s->in_flight == 0); 4757191bf31SMarkus Armbruster qemu_vfree(s->buf); 476b812f671SPaolo Bonzini g_free(s->cow_bitmap); 477402a4741SPaolo Bonzini g_free(s->in_flight_bitmap); 478e4654d2dSFam Zheng bdrv_release_dirty_bitmap(bs, s->dirty_bitmap); 479b952b558SPaolo Bonzini bdrv_iostatus_disable(s->target); 480d63ffd87SPaolo Bonzini if (s->should_complete && ret == 0) { 481d63ffd87SPaolo Bonzini if (bdrv_get_flags(s->target) != bdrv_get_flags(s->common.bs)) { 482d63ffd87SPaolo Bonzini bdrv_reopen(s->target, bdrv_get_flags(s->common.bs), NULL); 483d63ffd87SPaolo Bonzini } 484d63ffd87SPaolo Bonzini bdrv_swap(s->target, s->common.bs); 48520a63d2cSFam Zheng if (s->common.driver->job_type == BLOCK_JOB_TYPE_COMMIT) { 48620a63d2cSFam Zheng /* drop the bs loop chain formed by the swap: break the loop then 48720a63d2cSFam Zheng * trigger the unref from the top one */ 48820a63d2cSFam Zheng BlockDriverState *p = s->base->backing_hd; 48920a63d2cSFam Zheng s->base->backing_hd = NULL; 49020a63d2cSFam Zheng bdrv_unref(p); 49120a63d2cSFam Zheng } 492d63ffd87SPaolo Bonzini } 4934f6fd349SFam Zheng bdrv_unref(s->target); 494893f7ebaSPaolo Bonzini block_job_completed(&s->common, ret); 495893f7ebaSPaolo Bonzini } 496893f7ebaSPaolo Bonzini 497893f7ebaSPaolo Bonzini static void mirror_set_speed(BlockJob *job, int64_t speed, Error **errp) 498893f7ebaSPaolo Bonzini { 499893f7ebaSPaolo Bonzini MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 500893f7ebaSPaolo Bonzini 501893f7ebaSPaolo Bonzini if (speed < 0) { 502893f7ebaSPaolo Bonzini error_set(errp, QERR_INVALID_PARAMETER, "speed"); 503893f7ebaSPaolo Bonzini return; 504893f7ebaSPaolo Bonzini } 505893f7ebaSPaolo Bonzini ratelimit_set_speed(&s->limit, speed / BDRV_SECTOR_SIZE, SLICE_TIME); 506893f7ebaSPaolo Bonzini } 507893f7ebaSPaolo Bonzini 508b952b558SPaolo Bonzini static void mirror_iostatus_reset(BlockJob *job) 509b952b558SPaolo Bonzini { 510b952b558SPaolo Bonzini MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 511b952b558SPaolo Bonzini 512b952b558SPaolo Bonzini bdrv_iostatus_reset(s->target); 513b952b558SPaolo Bonzini } 514b952b558SPaolo Bonzini 515d63ffd87SPaolo Bonzini static void mirror_complete(BlockJob *job, Error **errp) 516d63ffd87SPaolo Bonzini { 517d63ffd87SPaolo Bonzini MirrorBlockJob *s = container_of(job, MirrorBlockJob, common); 51834b5d2c6SMax Reitz Error *local_err = NULL; 519d63ffd87SPaolo Bonzini int ret; 520d63ffd87SPaolo Bonzini 52134b5d2c6SMax Reitz ret = bdrv_open_backing_file(s->target, NULL, &local_err); 522d63ffd87SPaolo Bonzini if (ret < 0) { 523d63ffd87SPaolo Bonzini char backing_filename[PATH_MAX]; 524d63ffd87SPaolo Bonzini bdrv_get_full_backing_filename(s->target, backing_filename, 525d63ffd87SPaolo Bonzini sizeof(backing_filename)); 52634b5d2c6SMax Reitz error_propagate(errp, local_err); 527d63ffd87SPaolo Bonzini return; 528d63ffd87SPaolo Bonzini } 529d63ffd87SPaolo Bonzini if (!s->synced) { 530d63ffd87SPaolo Bonzini error_set(errp, QERR_BLOCK_JOB_NOT_READY, job->bs->device_name); 531d63ffd87SPaolo Bonzini return; 532d63ffd87SPaolo Bonzini } 533d63ffd87SPaolo Bonzini 534d63ffd87SPaolo Bonzini s->should_complete = true; 535d63ffd87SPaolo Bonzini block_job_resume(job); 536d63ffd87SPaolo Bonzini } 537d63ffd87SPaolo Bonzini 5383fc4b10aSFam Zheng static const BlockJobDriver mirror_job_driver = { 539893f7ebaSPaolo Bonzini .instance_size = sizeof(MirrorBlockJob), 54079e14bf7SFam Zheng .job_type = BLOCK_JOB_TYPE_MIRROR, 541893f7ebaSPaolo Bonzini .set_speed = mirror_set_speed, 542b952b558SPaolo Bonzini .iostatus_reset= mirror_iostatus_reset, 543d63ffd87SPaolo Bonzini .complete = mirror_complete, 544893f7ebaSPaolo Bonzini }; 545893f7ebaSPaolo Bonzini 54603544a6eSFam Zheng static const BlockJobDriver commit_active_job_driver = { 54703544a6eSFam Zheng .instance_size = sizeof(MirrorBlockJob), 54803544a6eSFam Zheng .job_type = BLOCK_JOB_TYPE_COMMIT, 54903544a6eSFam Zheng .set_speed = mirror_set_speed, 55003544a6eSFam Zheng .iostatus_reset 55103544a6eSFam Zheng = mirror_iostatus_reset, 55203544a6eSFam Zheng .complete = mirror_complete, 55303544a6eSFam Zheng }; 55403544a6eSFam Zheng 55503544a6eSFam Zheng static void mirror_start_job(BlockDriverState *bs, BlockDriverState *target, 55603544a6eSFam Zheng int64_t speed, int64_t granularity, 55703544a6eSFam Zheng int64_t buf_size, 55803544a6eSFam Zheng BlockdevOnError on_source_error, 559b952b558SPaolo Bonzini BlockdevOnError on_target_error, 560893f7ebaSPaolo Bonzini BlockDriverCompletionFunc *cb, 56103544a6eSFam Zheng void *opaque, Error **errp, 56203544a6eSFam Zheng const BlockJobDriver *driver, 56303544a6eSFam Zheng bool is_none_mode, BlockDriverState *base) 564893f7ebaSPaolo Bonzini { 565893f7ebaSPaolo Bonzini MirrorBlockJob *s; 566893f7ebaSPaolo Bonzini 567eee13dfeSPaolo Bonzini if (granularity == 0) { 568eee13dfeSPaolo Bonzini /* Choose the default granularity based on the target file's cluster 569eee13dfeSPaolo Bonzini * size, clamped between 4k and 64k. */ 570eee13dfeSPaolo Bonzini BlockDriverInfo bdi; 571eee13dfeSPaolo Bonzini if (bdrv_get_info(target, &bdi) >= 0 && bdi.cluster_size != 0) { 572eee13dfeSPaolo Bonzini granularity = MAX(4096, bdi.cluster_size); 573eee13dfeSPaolo Bonzini granularity = MIN(65536, granularity); 574eee13dfeSPaolo Bonzini } else { 575eee13dfeSPaolo Bonzini granularity = 65536; 576eee13dfeSPaolo Bonzini } 577eee13dfeSPaolo Bonzini } 578eee13dfeSPaolo Bonzini 579eee13dfeSPaolo Bonzini assert ((granularity & (granularity - 1)) == 0); 580eee13dfeSPaolo Bonzini 581b952b558SPaolo Bonzini if ((on_source_error == BLOCKDEV_ON_ERROR_STOP || 582b952b558SPaolo Bonzini on_source_error == BLOCKDEV_ON_ERROR_ENOSPC) && 583b952b558SPaolo Bonzini !bdrv_iostatus_is_enabled(bs)) { 584b952b558SPaolo Bonzini error_set(errp, QERR_INVALID_PARAMETER, "on-source-error"); 585b952b558SPaolo Bonzini return; 586b952b558SPaolo Bonzini } 587b952b558SPaolo Bonzini 5885bc361b8SFam Zheng 58903544a6eSFam Zheng s = block_job_create(driver, bs, speed, cb, opaque, errp); 590893f7ebaSPaolo Bonzini if (!s) { 591893f7ebaSPaolo Bonzini return; 592893f7ebaSPaolo Bonzini } 593893f7ebaSPaolo Bonzini 594b952b558SPaolo Bonzini s->on_source_error = on_source_error; 595b952b558SPaolo Bonzini s->on_target_error = on_target_error; 596893f7ebaSPaolo Bonzini s->target = target; 59703544a6eSFam Zheng s->is_none_mode = is_none_mode; 5985bc361b8SFam Zheng s->base = base; 599eee13dfeSPaolo Bonzini s->granularity = granularity; 60008e4ed6cSPaolo Bonzini s->buf_size = MAX(buf_size, granularity); 601b812f671SPaolo Bonzini 602e4654d2dSFam Zheng s->dirty_bitmap = bdrv_create_dirty_bitmap(bs, granularity); 603893f7ebaSPaolo Bonzini bdrv_set_enable_write_cache(s->target, true); 604b952b558SPaolo Bonzini bdrv_set_on_error(s->target, on_target_error, on_target_error); 605b952b558SPaolo Bonzini bdrv_iostatus_enable(s->target); 606893f7ebaSPaolo Bonzini s->common.co = qemu_coroutine_create(mirror_run); 607893f7ebaSPaolo Bonzini trace_mirror_start(bs, s, s->common.co, opaque); 608893f7ebaSPaolo Bonzini qemu_coroutine_enter(s->common.co, s); 609893f7ebaSPaolo Bonzini } 61003544a6eSFam Zheng 61103544a6eSFam Zheng void mirror_start(BlockDriverState *bs, BlockDriverState *target, 61203544a6eSFam Zheng int64_t speed, int64_t granularity, int64_t buf_size, 61303544a6eSFam Zheng MirrorSyncMode mode, BlockdevOnError on_source_error, 61403544a6eSFam Zheng BlockdevOnError on_target_error, 61503544a6eSFam Zheng BlockDriverCompletionFunc *cb, 61603544a6eSFam Zheng void *opaque, Error **errp) 61703544a6eSFam Zheng { 61803544a6eSFam Zheng bool is_none_mode; 61903544a6eSFam Zheng BlockDriverState *base; 62003544a6eSFam Zheng 62103544a6eSFam Zheng is_none_mode = mode == MIRROR_SYNC_MODE_NONE; 62203544a6eSFam Zheng base = mode == MIRROR_SYNC_MODE_TOP ? bs->backing_hd : NULL; 62303544a6eSFam Zheng mirror_start_job(bs, target, speed, granularity, buf_size, 62403544a6eSFam Zheng on_source_error, on_target_error, cb, opaque, errp, 62503544a6eSFam Zheng &mirror_job_driver, is_none_mode, base); 62603544a6eSFam Zheng } 62703544a6eSFam Zheng 62803544a6eSFam Zheng void commit_active_start(BlockDriverState *bs, BlockDriverState *base, 62903544a6eSFam Zheng int64_t speed, 63003544a6eSFam Zheng BlockdevOnError on_error, 63103544a6eSFam Zheng BlockDriverCompletionFunc *cb, 63203544a6eSFam Zheng void *opaque, Error **errp) 63303544a6eSFam Zheng { 6344da83585SJeff Cody int64_t length, base_length; 6354da83585SJeff Cody int orig_base_flags; 63639a611a3SJeff Cody int ret; 637*cc67f4d1SJeff Cody Error *local_err = NULL; 6384da83585SJeff Cody 6394da83585SJeff Cody orig_base_flags = bdrv_get_flags(base); 6404da83585SJeff Cody 64120a63d2cSFam Zheng if (bdrv_reopen(base, bs->open_flags, errp)) { 64220a63d2cSFam Zheng return; 64320a63d2cSFam Zheng } 6444da83585SJeff Cody 6454da83585SJeff Cody length = bdrv_getlength(bs); 6464da83585SJeff Cody if (length < 0) { 64739a611a3SJeff Cody error_setg_errno(errp, -length, 64839a611a3SJeff Cody "Unable to determine length of %s", bs->filename); 6494da83585SJeff Cody goto error_restore_flags; 6504da83585SJeff Cody } 6514da83585SJeff Cody 6524da83585SJeff Cody base_length = bdrv_getlength(base); 6534da83585SJeff Cody if (base_length < 0) { 65439a611a3SJeff Cody error_setg_errno(errp, -base_length, 65539a611a3SJeff Cody "Unable to determine length of %s", base->filename); 6564da83585SJeff Cody goto error_restore_flags; 6574da83585SJeff Cody } 6584da83585SJeff Cody 6594da83585SJeff Cody if (length > base_length) { 66039a611a3SJeff Cody ret = bdrv_truncate(base, length); 66139a611a3SJeff Cody if (ret < 0) { 66239a611a3SJeff Cody error_setg_errno(errp, -ret, 66339a611a3SJeff Cody "Top image %s is larger than base image %s, and " 6644da83585SJeff Cody "resize of base image failed", 6654da83585SJeff Cody bs->filename, base->filename); 6664da83585SJeff Cody goto error_restore_flags; 6674da83585SJeff Cody } 6684da83585SJeff Cody } 6694da83585SJeff Cody 67020a63d2cSFam Zheng bdrv_ref(base); 67103544a6eSFam Zheng mirror_start_job(bs, base, speed, 0, 0, 672*cc67f4d1SJeff Cody on_error, on_error, cb, opaque, &local_err, 67303544a6eSFam Zheng &commit_active_job_driver, false, base); 674*cc67f4d1SJeff Cody if (error_is_set(&local_err)) { 675*cc67f4d1SJeff Cody error_propagate(errp, local_err); 6764da83585SJeff Cody goto error_restore_flags; 6774da83585SJeff Cody } 6784da83585SJeff Cody 6794da83585SJeff Cody return; 6804da83585SJeff Cody 6814da83585SJeff Cody error_restore_flags: 6824da83585SJeff Cody /* ignore error and errp for bdrv_reopen, because we want to propagate 6834da83585SJeff Cody * the original error */ 6844da83585SJeff Cody bdrv_reopen(base, orig_base_flags, NULL); 6854da83585SJeff Cody return; 68603544a6eSFam Zheng } 687