xref: /qemu/block/mirror.c (revision 1ba793889583cf9b548bf61bd5c8f224ad6c1415)
1893f7ebaSPaolo Bonzini /*
2893f7ebaSPaolo Bonzini  * Image mirroring
3893f7ebaSPaolo Bonzini  *
4893f7ebaSPaolo Bonzini  * Copyright Red Hat, Inc. 2012
5893f7ebaSPaolo Bonzini  *
6893f7ebaSPaolo Bonzini  * Authors:
7893f7ebaSPaolo Bonzini  *  Paolo Bonzini  <pbonzini@redhat.com>
8893f7ebaSPaolo Bonzini  *
9893f7ebaSPaolo Bonzini  * This work is licensed under the terms of the GNU LGPL, version 2 or later.
10893f7ebaSPaolo Bonzini  * See the COPYING.LIB file in the top-level directory.
11893f7ebaSPaolo Bonzini  *
12893f7ebaSPaolo Bonzini  */
13893f7ebaSPaolo Bonzini 
1480c71a24SPeter Maydell #include "qemu/osdep.h"
15fd4a6493SKevin Wolf #include "qemu/cutils.h"
1612aa4082SMax Reitz #include "qemu/coroutine.h"
171181e19aSMax Reitz #include "qemu/range.h"
18893f7ebaSPaolo Bonzini #include "trace.h"
19c87621eaSJohn Snow #include "block/blockjob_int.h"
20737e150eSPaolo Bonzini #include "block/block_int.h"
21373340b2SMax Reitz #include "sysemu/block-backend.h"
22da34e65cSMarkus Armbruster #include "qapi/error.h"
23cc7a8ea7SMarkus Armbruster #include "qapi/qmp/qerror.h"
24893f7ebaSPaolo Bonzini #include "qemu/ratelimit.h"
25b812f671SPaolo Bonzini #include "qemu/bitmap.h"
26893f7ebaSPaolo Bonzini 
27402a4741SPaolo Bonzini #define MAX_IN_FLIGHT 16
28b436982fSEric Blake #define MAX_IO_BYTES (1 << 20) /* 1 Mb */
29b436982fSEric Blake #define DEFAULT_MIRROR_BUF_SIZE (MAX_IN_FLIGHT * MAX_IO_BYTES)
30402a4741SPaolo Bonzini 
31402a4741SPaolo Bonzini /* The mirroring buffer is a list of granularity-sized chunks.
32402a4741SPaolo Bonzini  * Free chunks are organized in a list.
33402a4741SPaolo Bonzini  */
34402a4741SPaolo Bonzini typedef struct MirrorBuffer {
35402a4741SPaolo Bonzini     QSIMPLEQ_ENTRY(MirrorBuffer) next;
36402a4741SPaolo Bonzini } MirrorBuffer;
37893f7ebaSPaolo Bonzini 
3812aa4082SMax Reitz typedef struct MirrorOp MirrorOp;
3912aa4082SMax Reitz 
40893f7ebaSPaolo Bonzini typedef struct MirrorBlockJob {
41893f7ebaSPaolo Bonzini     BlockJob common;
42e253f4b8SKevin Wolf     BlockBackend *target;
434ef85a9cSKevin Wolf     BlockDriverState *mirror_top_bs;
445bc361b8SFam Zheng     BlockDriverState *base;
454ef85a9cSKevin Wolf 
4609158f00SBenoît Canet     /* The name of the graph node to replace */
4709158f00SBenoît Canet     char *replaces;
4809158f00SBenoît Canet     /* The BDS to replace */
4909158f00SBenoît Canet     BlockDriverState *to_replace;
5009158f00SBenoît Canet     /* Used to block operations on the drive-mirror-replace target */
5109158f00SBenoît Canet     Error *replace_blocker;
5203544a6eSFam Zheng     bool is_none_mode;
53274fcceeSMax Reitz     BlockMirrorBackingMode backing_mode;
54d06107adSMax Reitz     MirrorCopyMode copy_mode;
55b952b558SPaolo Bonzini     BlockdevOnError on_source_error, on_target_error;
56d63ffd87SPaolo Bonzini     bool synced;
57d06107adSMax Reitz     /* Set when the target is synced (dirty bitmap is clean, nothing
58d06107adSMax Reitz      * in flight) and the job is running in active mode */
59d06107adSMax Reitz     bool actively_synced;
60d63ffd87SPaolo Bonzini     bool should_complete;
61eee13dfeSPaolo Bonzini     int64_t granularity;
62b812f671SPaolo Bonzini     size_t buf_size;
63b21c7652SMax Reitz     int64_t bdev_length;
64b812f671SPaolo Bonzini     unsigned long *cow_bitmap;
65e4654d2dSFam Zheng     BdrvDirtyBitmap *dirty_bitmap;
66dc162c8eSFam Zheng     BdrvDirtyBitmapIter *dbi;
67893f7ebaSPaolo Bonzini     uint8_t *buf;
68402a4741SPaolo Bonzini     QSIMPLEQ_HEAD(, MirrorBuffer) buf_free;
69402a4741SPaolo Bonzini     int buf_free_count;
70bd48bde8SPaolo Bonzini 
7149efb1f5SDenis V. Lunev     uint64_t last_pause_ns;
72402a4741SPaolo Bonzini     unsigned long *in_flight_bitmap;
73bd48bde8SPaolo Bonzini     int in_flight;
74b436982fSEric Blake     int64_t bytes_in_flight;
7512aa4082SMax Reitz     QTAILQ_HEAD(MirrorOpList, MirrorOp) ops_in_flight;
76bd48bde8SPaolo Bonzini     int ret;
770fc9f8eaSFam Zheng     bool unmap;
78b436982fSEric Blake     int target_cluster_size;
79e5b43573SFam Zheng     int max_iov;
8090ab48ebSAnton Nefedov     bool initial_zeroing_ongoing;
81d06107adSMax Reitz     int in_active_write_counter;
82737efc1eSJohn Snow     bool prepared;
83893f7ebaSPaolo Bonzini } MirrorBlockJob;
84893f7ebaSPaolo Bonzini 
85429076e8SMax Reitz typedef struct MirrorBDSOpaque {
86429076e8SMax Reitz     MirrorBlockJob *job;
87429076e8SMax Reitz } MirrorBDSOpaque;
88429076e8SMax Reitz 
8912aa4082SMax Reitz struct MirrorOp {
90bd48bde8SPaolo Bonzini     MirrorBlockJob *s;
91bd48bde8SPaolo Bonzini     QEMUIOVector qiov;
92b436982fSEric Blake     int64_t offset;
93b436982fSEric Blake     uint64_t bytes;
942e1990b2SMax Reitz 
952e1990b2SMax Reitz     /* The pointee is set by mirror_co_read(), mirror_co_zero(), and
962e1990b2SMax Reitz      * mirror_co_discard() before yielding for the first time */
972e1990b2SMax Reitz     int64_t *bytes_handled;
9812aa4082SMax Reitz 
991181e19aSMax Reitz     bool is_pseudo_op;
100d06107adSMax Reitz     bool is_active_write;
10112aa4082SMax Reitz     CoQueue waiting_requests;
10212aa4082SMax Reitz 
10312aa4082SMax Reitz     QTAILQ_ENTRY(MirrorOp) next;
10412aa4082SMax Reitz };
105bd48bde8SPaolo Bonzini 
1064295c5fcSMax Reitz typedef enum MirrorMethod {
1074295c5fcSMax Reitz     MIRROR_METHOD_COPY,
1084295c5fcSMax Reitz     MIRROR_METHOD_ZERO,
1094295c5fcSMax Reitz     MIRROR_METHOD_DISCARD,
1104295c5fcSMax Reitz } MirrorMethod;
1114295c5fcSMax Reitz 
112b952b558SPaolo Bonzini static BlockErrorAction mirror_error_action(MirrorBlockJob *s, bool read,
113b952b558SPaolo Bonzini                                             int error)
114b952b558SPaolo Bonzini {
115b952b558SPaolo Bonzini     s->synced = false;
116d06107adSMax Reitz     s->actively_synced = false;
117b952b558SPaolo Bonzini     if (read) {
11881e254dcSKevin Wolf         return block_job_error_action(&s->common, s->on_source_error,
11981e254dcSKevin Wolf                                       true, error);
120b952b558SPaolo Bonzini     } else {
12181e254dcSKevin Wolf         return block_job_error_action(&s->common, s->on_target_error,
12281e254dcSKevin Wolf                                       false, error);
123b952b558SPaolo Bonzini     }
124b952b558SPaolo Bonzini }
125b952b558SPaolo Bonzini 
1261181e19aSMax Reitz static void coroutine_fn mirror_wait_on_conflicts(MirrorOp *self,
1271181e19aSMax Reitz                                                   MirrorBlockJob *s,
1281181e19aSMax Reitz                                                   uint64_t offset,
1291181e19aSMax Reitz                                                   uint64_t bytes)
1301181e19aSMax Reitz {
1311181e19aSMax Reitz     uint64_t self_start_chunk = offset / s->granularity;
1321181e19aSMax Reitz     uint64_t self_end_chunk = DIV_ROUND_UP(offset + bytes, s->granularity);
1331181e19aSMax Reitz     uint64_t self_nb_chunks = self_end_chunk - self_start_chunk;
1341181e19aSMax Reitz 
1351181e19aSMax Reitz     while (find_next_bit(s->in_flight_bitmap, self_end_chunk,
1361181e19aSMax Reitz                          self_start_chunk) < self_end_chunk &&
1371181e19aSMax Reitz            s->ret >= 0)
1381181e19aSMax Reitz     {
1391181e19aSMax Reitz         MirrorOp *op;
1401181e19aSMax Reitz 
1411181e19aSMax Reitz         QTAILQ_FOREACH(op, &s->ops_in_flight, next) {
1421181e19aSMax Reitz             uint64_t op_start_chunk = op->offset / s->granularity;
1431181e19aSMax Reitz             uint64_t op_nb_chunks = DIV_ROUND_UP(op->offset + op->bytes,
1441181e19aSMax Reitz                                                  s->granularity) -
1451181e19aSMax Reitz                                     op_start_chunk;
1461181e19aSMax Reitz 
1471181e19aSMax Reitz             if (op == self) {
1481181e19aSMax Reitz                 continue;
1491181e19aSMax Reitz             }
1501181e19aSMax Reitz 
1511181e19aSMax Reitz             if (ranges_overlap(self_start_chunk, self_nb_chunks,
1521181e19aSMax Reitz                                op_start_chunk, op_nb_chunks))
1531181e19aSMax Reitz             {
1541181e19aSMax Reitz                 qemu_co_queue_wait(&op->waiting_requests, NULL);
1551181e19aSMax Reitz                 break;
1561181e19aSMax Reitz             }
1571181e19aSMax Reitz         }
1581181e19aSMax Reitz     }
1591181e19aSMax Reitz }
1601181e19aSMax Reitz 
1612e1990b2SMax Reitz static void coroutine_fn mirror_iteration_done(MirrorOp *op, int ret)
162bd48bde8SPaolo Bonzini {
163bd48bde8SPaolo Bonzini     MirrorBlockJob *s = op->s;
164402a4741SPaolo Bonzini     struct iovec *iov;
165bd48bde8SPaolo Bonzini     int64_t chunk_num;
166b436982fSEric Blake     int i, nb_chunks;
167bd48bde8SPaolo Bonzini 
168b436982fSEric Blake     trace_mirror_iteration_done(s, op->offset, op->bytes, ret);
169bd48bde8SPaolo Bonzini 
170bd48bde8SPaolo Bonzini     s->in_flight--;
171b436982fSEric Blake     s->bytes_in_flight -= op->bytes;
172402a4741SPaolo Bonzini     iov = op->qiov.iov;
173402a4741SPaolo Bonzini     for (i = 0; i < op->qiov.niov; i++) {
174402a4741SPaolo Bonzini         MirrorBuffer *buf = (MirrorBuffer *) iov[i].iov_base;
175402a4741SPaolo Bonzini         QSIMPLEQ_INSERT_TAIL(&s->buf_free, buf, next);
176402a4741SPaolo Bonzini         s->buf_free_count++;
177402a4741SPaolo Bonzini     }
178402a4741SPaolo Bonzini 
179b436982fSEric Blake     chunk_num = op->offset / s->granularity;
180b436982fSEric Blake     nb_chunks = DIV_ROUND_UP(op->bytes, s->granularity);
18112aa4082SMax Reitz 
182402a4741SPaolo Bonzini     bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks);
18312aa4082SMax Reitz     QTAILQ_REMOVE(&s->ops_in_flight, op, next);
184b21c7652SMax Reitz     if (ret >= 0) {
185b21c7652SMax Reitz         if (s->cow_bitmap) {
186bd48bde8SPaolo Bonzini             bitmap_set(s->cow_bitmap, chunk_num, nb_chunks);
187bd48bde8SPaolo Bonzini         }
18890ab48ebSAnton Nefedov         if (!s->initial_zeroing_ongoing) {
18930a5c887SKevin Wolf             job_progress_update(&s->common.job, op->bytes);
190b21c7652SMax Reitz         }
19190ab48ebSAnton Nefedov     }
1926df3bf8eSZhang Min     qemu_iovec_destroy(&op->qiov);
1937b770c72SStefan Hajnoczi 
19412aa4082SMax Reitz     qemu_co_queue_restart_all(&op->waiting_requests);
19512aa4082SMax Reitz     g_free(op);
1967b770c72SStefan Hajnoczi }
197bd48bde8SPaolo Bonzini 
1982e1990b2SMax Reitz static void coroutine_fn mirror_write_complete(MirrorOp *op, int ret)
199bd48bde8SPaolo Bonzini {
200bd48bde8SPaolo Bonzini     MirrorBlockJob *s = op->s;
201b9e413ddSPaolo Bonzini 
202bd48bde8SPaolo Bonzini     if (ret < 0) {
203bd48bde8SPaolo Bonzini         BlockErrorAction action;
204bd48bde8SPaolo Bonzini 
205e0d7f73eSEric Blake         bdrv_set_dirty_bitmap(s->dirty_bitmap, op->offset, op->bytes);
206bd48bde8SPaolo Bonzini         action = mirror_error_action(s, false, -ret);
207a589569fSWenchao Xia         if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) {
208bd48bde8SPaolo Bonzini             s->ret = ret;
209bd48bde8SPaolo Bonzini         }
210bd48bde8SPaolo Bonzini     }
211d12ade57SVladimir Sementsov-Ogievskiy 
212bd48bde8SPaolo Bonzini     mirror_iteration_done(op, ret);
213bd48bde8SPaolo Bonzini }
214bd48bde8SPaolo Bonzini 
2152e1990b2SMax Reitz static void coroutine_fn mirror_read_complete(MirrorOp *op, int ret)
216bd48bde8SPaolo Bonzini {
217bd48bde8SPaolo Bonzini     MirrorBlockJob *s = op->s;
218b9e413ddSPaolo Bonzini 
219bd48bde8SPaolo Bonzini     if (ret < 0) {
220bd48bde8SPaolo Bonzini         BlockErrorAction action;
221bd48bde8SPaolo Bonzini 
222e0d7f73eSEric Blake         bdrv_set_dirty_bitmap(s->dirty_bitmap, op->offset, op->bytes);
223bd48bde8SPaolo Bonzini         action = mirror_error_action(s, true, -ret);
224a589569fSWenchao Xia         if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) {
225bd48bde8SPaolo Bonzini             s->ret = ret;
226bd48bde8SPaolo Bonzini         }
227bd48bde8SPaolo Bonzini 
228bd48bde8SPaolo Bonzini         mirror_iteration_done(op, ret);
229d12ade57SVladimir Sementsov-Ogievskiy         return;
230bd48bde8SPaolo Bonzini     }
231d12ade57SVladimir Sementsov-Ogievskiy 
232d12ade57SVladimir Sementsov-Ogievskiy     ret = blk_co_pwritev(s->target, op->offset, op->qiov.size, &op->qiov, 0);
233d12ade57SVladimir Sementsov-Ogievskiy     mirror_write_complete(op, ret);
234b9e413ddSPaolo Bonzini }
235bd48bde8SPaolo Bonzini 
236782d97efSEric Blake /* Clip bytes relative to offset to not exceed end-of-file */
237782d97efSEric Blake static inline int64_t mirror_clip_bytes(MirrorBlockJob *s,
238782d97efSEric Blake                                         int64_t offset,
239782d97efSEric Blake                                         int64_t bytes)
240782d97efSEric Blake {
241782d97efSEric Blake     return MIN(bytes, s->bdev_length - offset);
242782d97efSEric Blake }
243782d97efSEric Blake 
244782d97efSEric Blake /* Round offset and/or bytes to target cluster if COW is needed, and
245782d97efSEric Blake  * return the offset of the adjusted tail against original. */
246782d97efSEric Blake static int mirror_cow_align(MirrorBlockJob *s, int64_t *offset,
247ae4cc877SEric Blake                             uint64_t *bytes)
248893f7ebaSPaolo Bonzini {
249e5b43573SFam Zheng     bool need_cow;
250e5b43573SFam Zheng     int ret = 0;
251782d97efSEric Blake     int64_t align_offset = *offset;
2527cfd5275SEric Blake     int64_t align_bytes = *bytes;
253782d97efSEric Blake     int max_bytes = s->granularity * s->max_iov;
254893f7ebaSPaolo Bonzini 
255782d97efSEric Blake     need_cow = !test_bit(*offset / s->granularity, s->cow_bitmap);
256782d97efSEric Blake     need_cow |= !test_bit((*offset + *bytes - 1) / s->granularity,
257e5b43573SFam Zheng                           s->cow_bitmap);
258e5b43573SFam Zheng     if (need_cow) {
259782d97efSEric Blake         bdrv_round_to_clusters(blk_bs(s->target), *offset, *bytes,
260782d97efSEric Blake                                &align_offset, &align_bytes);
2618f0720ecSPaolo Bonzini     }
2628f0720ecSPaolo Bonzini 
263782d97efSEric Blake     if (align_bytes > max_bytes) {
264782d97efSEric Blake         align_bytes = max_bytes;
265e5b43573SFam Zheng         if (need_cow) {
266782d97efSEric Blake             align_bytes = QEMU_ALIGN_DOWN(align_bytes, s->target_cluster_size);
267e5b43573SFam Zheng         }
268e5b43573SFam Zheng     }
269782d97efSEric Blake     /* Clipping may result in align_bytes unaligned to chunk boundary, but
2704150ae60SFam Zheng      * that doesn't matter because it's already the end of source image. */
271782d97efSEric Blake     align_bytes = mirror_clip_bytes(s, align_offset, align_bytes);
272402a4741SPaolo Bonzini 
273782d97efSEric Blake     ret = align_offset + align_bytes - (*offset + *bytes);
274782d97efSEric Blake     *offset = align_offset;
275782d97efSEric Blake     *bytes = align_bytes;
276e5b43573SFam Zheng     assert(ret >= 0);
277e5b43573SFam Zheng     return ret;
278e5b43573SFam Zheng }
279e5b43573SFam Zheng 
280d06107adSMax Reitz static inline void mirror_wait_for_any_operation(MirrorBlockJob *s, bool active)
28121cd917fSFam Zheng {
28212aa4082SMax Reitz     MirrorOp *op;
28312aa4082SMax Reitz 
2841181e19aSMax Reitz     QTAILQ_FOREACH(op, &s->ops_in_flight, next) {
2851181e19aSMax Reitz         /* Do not wait on pseudo ops, because it may in turn wait on
2861181e19aSMax Reitz          * some other operation to start, which may in fact be the
2871181e19aSMax Reitz          * caller of this function.  Since there is only one pseudo op
2881181e19aSMax Reitz          * at any given time, we will always find some real operation
2891181e19aSMax Reitz          * to wait on. */
290d06107adSMax Reitz         if (!op->is_pseudo_op && op->is_active_write == active) {
29112aa4082SMax Reitz             qemu_co_queue_wait(&op->waiting_requests, NULL);
2921181e19aSMax Reitz             return;
2931181e19aSMax Reitz         }
2941181e19aSMax Reitz     }
2951181e19aSMax Reitz     abort();
29621cd917fSFam Zheng }
29721cd917fSFam Zheng 
298d06107adSMax Reitz static inline void mirror_wait_for_free_in_flight_slot(MirrorBlockJob *s)
299d06107adSMax Reitz {
300d06107adSMax Reitz     /* Only non-active operations use up in-flight slots */
301d06107adSMax Reitz     mirror_wait_for_any_operation(s, false);
302d06107adSMax Reitz }
303d06107adSMax Reitz 
3042e1990b2SMax Reitz /* Perform a mirror copy operation.
3052e1990b2SMax Reitz  *
3062e1990b2SMax Reitz  * *op->bytes_handled is set to the number of bytes copied after and
3072e1990b2SMax Reitz  * including offset, excluding any bytes copied prior to offset due
3082e1990b2SMax Reitz  * to alignment.  This will be op->bytes if no alignment is necessary,
3092e1990b2SMax Reitz  * or (new_end - op->offset) if the tail is rounded up or down due to
310e5b43573SFam Zheng  * alignment or buffer limit.
311402a4741SPaolo Bonzini  */
3122e1990b2SMax Reitz static void coroutine_fn mirror_co_read(void *opaque)
313e5b43573SFam Zheng {
3142e1990b2SMax Reitz     MirrorOp *op = opaque;
3152e1990b2SMax Reitz     MirrorBlockJob *s = op->s;
316ae4cc877SEric Blake     int nb_chunks;
317ae4cc877SEric Blake     uint64_t ret;
318ae4cc877SEric Blake     uint64_t max_bytes;
319402a4741SPaolo Bonzini 
320ae4cc877SEric Blake     max_bytes = s->granularity * s->max_iov;
321e5b43573SFam Zheng 
322e5b43573SFam Zheng     /* We can only handle as much as buf_size at a time. */
3232e1990b2SMax Reitz     op->bytes = MIN(s->buf_size, MIN(max_bytes, op->bytes));
3242e1990b2SMax Reitz     assert(op->bytes);
3252e1990b2SMax Reitz     assert(op->bytes < BDRV_REQUEST_MAX_BYTES);
3262e1990b2SMax Reitz     *op->bytes_handled = op->bytes;
327e5b43573SFam Zheng 
328e5b43573SFam Zheng     if (s->cow_bitmap) {
3292e1990b2SMax Reitz         *op->bytes_handled += mirror_cow_align(s, &op->offset, &op->bytes);
330e5b43573SFam Zheng     }
3312e1990b2SMax Reitz     /* Cannot exceed BDRV_REQUEST_MAX_BYTES + INT_MAX */
3322e1990b2SMax Reitz     assert(*op->bytes_handled <= UINT_MAX);
3332e1990b2SMax Reitz     assert(op->bytes <= s->buf_size);
334ae4cc877SEric Blake     /* The offset is granularity-aligned because:
335e5b43573SFam Zheng      * 1) Caller passes in aligned values;
336e5b43573SFam Zheng      * 2) mirror_cow_align is used only when target cluster is larger. */
3372e1990b2SMax Reitz     assert(QEMU_IS_ALIGNED(op->offset, s->granularity));
338ae4cc877SEric Blake     /* The range is sector-aligned, since bdrv_getlength() rounds up. */
3392e1990b2SMax Reitz     assert(QEMU_IS_ALIGNED(op->bytes, BDRV_SECTOR_SIZE));
3402e1990b2SMax Reitz     nb_chunks = DIV_ROUND_UP(op->bytes, s->granularity);
341e5b43573SFam Zheng 
342e5b43573SFam Zheng     while (s->buf_free_count < nb_chunks) {
3432e1990b2SMax Reitz         trace_mirror_yield_in_flight(s, op->offset, s->in_flight);
3441181e19aSMax Reitz         mirror_wait_for_free_in_flight_slot(s);
345b812f671SPaolo Bonzini     }
346b812f671SPaolo Bonzini 
347402a4741SPaolo Bonzini     /* Now make a QEMUIOVector taking enough granularity-sized chunks
348402a4741SPaolo Bonzini      * from s->buf_free.
349402a4741SPaolo Bonzini      */
350402a4741SPaolo Bonzini     qemu_iovec_init(&op->qiov, nb_chunks);
351402a4741SPaolo Bonzini     while (nb_chunks-- > 0) {
352402a4741SPaolo Bonzini         MirrorBuffer *buf = QSIMPLEQ_FIRST(&s->buf_free);
3532e1990b2SMax Reitz         size_t remaining = op->bytes - op->qiov.size;
3545a0f6fd5SKevin Wolf 
355402a4741SPaolo Bonzini         QSIMPLEQ_REMOVE_HEAD(&s->buf_free, next);
356402a4741SPaolo Bonzini         s->buf_free_count--;
3575a0f6fd5SKevin Wolf         qemu_iovec_add(&op->qiov, buf, MIN(s->granularity, remaining));
358402a4741SPaolo Bonzini     }
359402a4741SPaolo Bonzini 
360893f7ebaSPaolo Bonzini     /* Copy the dirty cluster.  */
361bd48bde8SPaolo Bonzini     s->in_flight++;
3622e1990b2SMax Reitz     s->bytes_in_flight += op->bytes;
3632e1990b2SMax Reitz     trace_mirror_one_iteration(s, op->offset, op->bytes);
364dcfb3bebSFam Zheng 
365138f9fffSMax Reitz     ret = bdrv_co_preadv(s->mirror_top_bs->backing, op->offset, op->bytes,
366138f9fffSMax Reitz                          &op->qiov, 0);
3672e1990b2SMax Reitz     mirror_read_complete(op, ret);
368e5b43573SFam Zheng }
369e5b43573SFam Zheng 
3702e1990b2SMax Reitz static void coroutine_fn mirror_co_zero(void *opaque)
371e5b43573SFam Zheng {
3722e1990b2SMax Reitz     MirrorOp *op = opaque;
3732e1990b2SMax Reitz     int ret;
374e5b43573SFam Zheng 
3752e1990b2SMax Reitz     op->s->in_flight++;
3762e1990b2SMax Reitz     op->s->bytes_in_flight += op->bytes;
3772e1990b2SMax Reitz     *op->bytes_handled = op->bytes;
378e5b43573SFam Zheng 
3792e1990b2SMax Reitz     ret = blk_co_pwrite_zeroes(op->s->target, op->offset, op->bytes,
3802e1990b2SMax Reitz                                op->s->unmap ? BDRV_REQ_MAY_UNMAP : 0);
3812e1990b2SMax Reitz     mirror_write_complete(op, ret);
382e5b43573SFam Zheng }
3832e1990b2SMax Reitz 
3842e1990b2SMax Reitz static void coroutine_fn mirror_co_discard(void *opaque)
3852e1990b2SMax Reitz {
3862e1990b2SMax Reitz     MirrorOp *op = opaque;
3872e1990b2SMax Reitz     int ret;
3882e1990b2SMax Reitz 
3892e1990b2SMax Reitz     op->s->in_flight++;
3902e1990b2SMax Reitz     op->s->bytes_in_flight += op->bytes;
3912e1990b2SMax Reitz     *op->bytes_handled = op->bytes;
3922e1990b2SMax Reitz 
3932e1990b2SMax Reitz     ret = blk_co_pdiscard(op->s->target, op->offset, op->bytes);
3942e1990b2SMax Reitz     mirror_write_complete(op, ret);
395e5b43573SFam Zheng }
396e5b43573SFam Zheng 
3974295c5fcSMax Reitz static unsigned mirror_perform(MirrorBlockJob *s, int64_t offset,
3984295c5fcSMax Reitz                                unsigned bytes, MirrorMethod mirror_method)
3994295c5fcSMax Reitz {
4002e1990b2SMax Reitz     MirrorOp *op;
4012e1990b2SMax Reitz     Coroutine *co;
4022e1990b2SMax Reitz     int64_t bytes_handled = -1;
4032e1990b2SMax Reitz 
4042e1990b2SMax Reitz     op = g_new(MirrorOp, 1);
4052e1990b2SMax Reitz     *op = (MirrorOp){
4062e1990b2SMax Reitz         .s              = s,
4072e1990b2SMax Reitz         .offset         = offset,
4082e1990b2SMax Reitz         .bytes          = bytes,
4092e1990b2SMax Reitz         .bytes_handled  = &bytes_handled,
4102e1990b2SMax Reitz     };
41112aa4082SMax Reitz     qemu_co_queue_init(&op->waiting_requests);
4122e1990b2SMax Reitz 
4134295c5fcSMax Reitz     switch (mirror_method) {
4144295c5fcSMax Reitz     case MIRROR_METHOD_COPY:
4152e1990b2SMax Reitz         co = qemu_coroutine_create(mirror_co_read, op);
4162e1990b2SMax Reitz         break;
4174295c5fcSMax Reitz     case MIRROR_METHOD_ZERO:
4182e1990b2SMax Reitz         co = qemu_coroutine_create(mirror_co_zero, op);
4192e1990b2SMax Reitz         break;
4204295c5fcSMax Reitz     case MIRROR_METHOD_DISCARD:
4212e1990b2SMax Reitz         co = qemu_coroutine_create(mirror_co_discard, op);
4222e1990b2SMax Reitz         break;
4234295c5fcSMax Reitz     default:
4244295c5fcSMax Reitz         abort();
4254295c5fcSMax Reitz     }
4262e1990b2SMax Reitz 
42712aa4082SMax Reitz     QTAILQ_INSERT_TAIL(&s->ops_in_flight, op, next);
4282e1990b2SMax Reitz     qemu_coroutine_enter(co);
4292e1990b2SMax Reitz     /* At this point, ownership of op has been moved to the coroutine
4302e1990b2SMax Reitz      * and the object may already be freed */
4312e1990b2SMax Reitz 
4322e1990b2SMax Reitz     /* Assert that this value has been set */
4332e1990b2SMax Reitz     assert(bytes_handled >= 0);
4342e1990b2SMax Reitz 
4352e1990b2SMax Reitz     /* Same assertion as in mirror_co_read() (and for mirror_co_read()
4362e1990b2SMax Reitz      * and mirror_co_discard(), bytes_handled == op->bytes, which
4372e1990b2SMax Reitz      * is the @bytes parameter given to this function) */
4382e1990b2SMax Reitz     assert(bytes_handled <= UINT_MAX);
4392e1990b2SMax Reitz     return bytes_handled;
4404295c5fcSMax Reitz }
4414295c5fcSMax Reitz 
442e5b43573SFam Zheng static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s)
443e5b43573SFam Zheng {
444138f9fffSMax Reitz     BlockDriverState *source = s->mirror_top_bs->backing->bs;
4451181e19aSMax Reitz     MirrorOp *pseudo_op;
4461181e19aSMax Reitz     int64_t offset;
4471181e19aSMax Reitz     uint64_t delay_ns = 0, ret = 0;
448e5b43573SFam Zheng     /* At least the first dirty chunk is mirrored in one iteration. */
449e5b43573SFam Zheng     int nb_chunks = 1;
4504b5004d9SDenis V. Lunev     bool write_zeroes_ok = bdrv_can_write_zeroes_with_unmap(blk_bs(s->target));
451b436982fSEric Blake     int max_io_bytes = MAX(s->buf_size / MAX_IN_FLIGHT, MAX_IO_BYTES);
452e5b43573SFam Zheng 
453b64bd51eSPaolo Bonzini     bdrv_dirty_bitmap_lock(s->dirty_bitmap);
454f798184cSEric Blake     offset = bdrv_dirty_iter_next(s->dbi);
455fb2ef791SEric Blake     if (offset < 0) {
456dc162c8eSFam Zheng         bdrv_set_dirty_iter(s->dbi, 0);
457f798184cSEric Blake         offset = bdrv_dirty_iter_next(s->dbi);
4589a46dba7SEric Blake         trace_mirror_restart_iter(s, bdrv_get_dirty_count(s->dirty_bitmap));
459fb2ef791SEric Blake         assert(offset >= 0);
460e5b43573SFam Zheng     }
461b64bd51eSPaolo Bonzini     bdrv_dirty_bitmap_unlock(s->dirty_bitmap);
462e5b43573SFam Zheng 
4631181e19aSMax Reitz     mirror_wait_on_conflicts(NULL, s, offset, 1);
4649c83625bSMax Reitz 
465da01ff7fSKevin Wolf     job_pause_point(&s->common.job);
466565ac01fSStefan Hajnoczi 
467e5b43573SFam Zheng     /* Find the number of consective dirty chunks following the first dirty
468e5b43573SFam Zheng      * one, and wait for in flight requests in them. */
469b64bd51eSPaolo Bonzini     bdrv_dirty_bitmap_lock(s->dirty_bitmap);
470fb2ef791SEric Blake     while (nb_chunks * s->granularity < s->buf_size) {
471dc162c8eSFam Zheng         int64_t next_dirty;
472fb2ef791SEric Blake         int64_t next_offset = offset + nb_chunks * s->granularity;
473fb2ef791SEric Blake         int64_t next_chunk = next_offset / s->granularity;
474fb2ef791SEric Blake         if (next_offset >= s->bdev_length ||
4753b5d4df0SEric Blake             !bdrv_get_dirty_locked(source, s->dirty_bitmap, next_offset)) {
476e5b43573SFam Zheng             break;
477e5b43573SFam Zheng         }
478e5b43573SFam Zheng         if (test_bit(next_chunk, s->in_flight_bitmap)) {
479e5b43573SFam Zheng             break;
480e5b43573SFam Zheng         }
4819c83625bSMax Reitz 
482f798184cSEric Blake         next_dirty = bdrv_dirty_iter_next(s->dbi);
483fb2ef791SEric Blake         if (next_dirty > next_offset || next_dirty < 0) {
484f27a2742SMax Reitz             /* The bitmap iterator's cache is stale, refresh it */
485715a74d8SEric Blake             bdrv_set_dirty_iter(s->dbi, next_offset);
486f798184cSEric Blake             next_dirty = bdrv_dirty_iter_next(s->dbi);
487f27a2742SMax Reitz         }
488fb2ef791SEric Blake         assert(next_dirty == next_offset);
489e5b43573SFam Zheng         nb_chunks++;
490e5b43573SFam Zheng     }
491e5b43573SFam Zheng 
492e5b43573SFam Zheng     /* Clear dirty bits before querying the block status, because
49331826642SEric Blake      * calling bdrv_block_status_above could yield - if some blocks are
494e5b43573SFam Zheng      * marked dirty in this window, we need to know.
495e5b43573SFam Zheng      */
496e0d7f73eSEric Blake     bdrv_reset_dirty_bitmap_locked(s->dirty_bitmap, offset,
497e0d7f73eSEric Blake                                    nb_chunks * s->granularity);
498b64bd51eSPaolo Bonzini     bdrv_dirty_bitmap_unlock(s->dirty_bitmap);
499b64bd51eSPaolo Bonzini 
5001181e19aSMax Reitz     /* Before claiming an area in the in-flight bitmap, we have to
5011181e19aSMax Reitz      * create a MirrorOp for it so that conflicting requests can wait
5021181e19aSMax Reitz      * for it.  mirror_perform() will create the real MirrorOps later,
5031181e19aSMax Reitz      * for now we just create a pseudo operation that will wake up all
5041181e19aSMax Reitz      * conflicting requests once all real operations have been
5051181e19aSMax Reitz      * launched. */
5061181e19aSMax Reitz     pseudo_op = g_new(MirrorOp, 1);
5071181e19aSMax Reitz     *pseudo_op = (MirrorOp){
5081181e19aSMax Reitz         .offset         = offset,
5091181e19aSMax Reitz         .bytes          = nb_chunks * s->granularity,
5101181e19aSMax Reitz         .is_pseudo_op   = true,
5111181e19aSMax Reitz     };
5121181e19aSMax Reitz     qemu_co_queue_init(&pseudo_op->waiting_requests);
5131181e19aSMax Reitz     QTAILQ_INSERT_TAIL(&s->ops_in_flight, pseudo_op, next);
5141181e19aSMax Reitz 
515fb2ef791SEric Blake     bitmap_set(s->in_flight_bitmap, offset / s->granularity, nb_chunks);
516fb2ef791SEric Blake     while (nb_chunks > 0 && offset < s->bdev_length) {
51731826642SEric Blake         int ret;
5187cfd5275SEric Blake         int64_t io_bytes;
519f3e4ce4aSEric Blake         int64_t io_bytes_acct;
5204295c5fcSMax Reitz         MirrorMethod mirror_method = MIRROR_METHOD_COPY;
521e5b43573SFam Zheng 
522fb2ef791SEric Blake         assert(!(offset % s->granularity));
52331826642SEric Blake         ret = bdrv_block_status_above(source, NULL, offset,
52431826642SEric Blake                                       nb_chunks * s->granularity,
52531826642SEric Blake                                       &io_bytes, NULL, NULL);
526e5b43573SFam Zheng         if (ret < 0) {
527fb2ef791SEric Blake             io_bytes = MIN(nb_chunks * s->granularity, max_io_bytes);
5280965a41eSVladimir Sementsov-Ogievskiy         } else if (ret & BDRV_BLOCK_DATA) {
529fb2ef791SEric Blake             io_bytes = MIN(io_bytes, max_io_bytes);
530e5b43573SFam Zheng         }
531e5b43573SFam Zheng 
532fb2ef791SEric Blake         io_bytes -= io_bytes % s->granularity;
533fb2ef791SEric Blake         if (io_bytes < s->granularity) {
534fb2ef791SEric Blake             io_bytes = s->granularity;
535e5b43573SFam Zheng         } else if (ret >= 0 && !(ret & BDRV_BLOCK_DATA)) {
536fb2ef791SEric Blake             int64_t target_offset;
5377cfd5275SEric Blake             int64_t target_bytes;
538fb2ef791SEric Blake             bdrv_round_to_clusters(blk_bs(s->target), offset, io_bytes,
539fb2ef791SEric Blake                                    &target_offset, &target_bytes);
540fb2ef791SEric Blake             if (target_offset == offset &&
541fb2ef791SEric Blake                 target_bytes == io_bytes) {
542e5b43573SFam Zheng                 mirror_method = ret & BDRV_BLOCK_ZERO ?
543e5b43573SFam Zheng                                     MIRROR_METHOD_ZERO :
544e5b43573SFam Zheng                                     MIRROR_METHOD_DISCARD;
545e5b43573SFam Zheng             }
546e5b43573SFam Zheng         }
547e5b43573SFam Zheng 
548cf56a3c6SDenis V. Lunev         while (s->in_flight >= MAX_IN_FLIGHT) {
549fb2ef791SEric Blake             trace_mirror_yield_in_flight(s, offset, s->in_flight);
5501181e19aSMax Reitz             mirror_wait_for_free_in_flight_slot(s);
551cf56a3c6SDenis V. Lunev         }
552cf56a3c6SDenis V. Lunev 
553dbaa7b57SVladimir Sementsov-Ogievskiy         if (s->ret < 0) {
5541181e19aSMax Reitz             ret = 0;
5551181e19aSMax Reitz             goto fail;
556dbaa7b57SVladimir Sementsov-Ogievskiy         }
557dbaa7b57SVladimir Sementsov-Ogievskiy 
558fb2ef791SEric Blake         io_bytes = mirror_clip_bytes(s, offset, io_bytes);
5594295c5fcSMax Reitz         io_bytes = mirror_perform(s, offset, io_bytes, mirror_method);
5604295c5fcSMax Reitz         if (mirror_method != MIRROR_METHOD_COPY && write_zeroes_ok) {
561f3e4ce4aSEric Blake             io_bytes_acct = 0;
5624b5004d9SDenis V. Lunev         } else {
563fb2ef791SEric Blake             io_bytes_acct = io_bytes;
5644b5004d9SDenis V. Lunev         }
565fb2ef791SEric Blake         assert(io_bytes);
566fb2ef791SEric Blake         offset += io_bytes;
567fb2ef791SEric Blake         nb_chunks -= DIV_ROUND_UP(io_bytes, s->granularity);
568dee81d51SKevin Wolf         delay_ns = block_job_ratelimit_get_delay(&s->common, io_bytes_acct);
569dcfb3bebSFam Zheng     }
5701181e19aSMax Reitz 
5711181e19aSMax Reitz     ret = delay_ns;
5721181e19aSMax Reitz fail:
5731181e19aSMax Reitz     QTAILQ_REMOVE(&s->ops_in_flight, pseudo_op, next);
5741181e19aSMax Reitz     qemu_co_queue_restart_all(&pseudo_op->waiting_requests);
5751181e19aSMax Reitz     g_free(pseudo_op);
5761181e19aSMax Reitz 
5771181e19aSMax Reitz     return ret;
578893f7ebaSPaolo Bonzini }
579b952b558SPaolo Bonzini 
580402a4741SPaolo Bonzini static void mirror_free_init(MirrorBlockJob *s)
581402a4741SPaolo Bonzini {
582402a4741SPaolo Bonzini     int granularity = s->granularity;
583402a4741SPaolo Bonzini     size_t buf_size = s->buf_size;
584402a4741SPaolo Bonzini     uint8_t *buf = s->buf;
585402a4741SPaolo Bonzini 
586402a4741SPaolo Bonzini     assert(s->buf_free_count == 0);
587402a4741SPaolo Bonzini     QSIMPLEQ_INIT(&s->buf_free);
588402a4741SPaolo Bonzini     while (buf_size != 0) {
589402a4741SPaolo Bonzini         MirrorBuffer *cur = (MirrorBuffer *)buf;
590402a4741SPaolo Bonzini         QSIMPLEQ_INSERT_TAIL(&s->buf_free, cur, next);
591402a4741SPaolo Bonzini         s->buf_free_count++;
592402a4741SPaolo Bonzini         buf_size -= granularity;
593402a4741SPaolo Bonzini         buf += granularity;
594402a4741SPaolo Bonzini     }
595402a4741SPaolo Bonzini }
596402a4741SPaolo Bonzini 
597bae8196dSPaolo Bonzini /* This is also used for the .pause callback. There is no matching
598bae8196dSPaolo Bonzini  * mirror_resume() because mirror_run() will begin iterating again
599bae8196dSPaolo Bonzini  * when the job is resumed.
600bae8196dSPaolo Bonzini  */
601bae8196dSPaolo Bonzini static void mirror_wait_for_all_io(MirrorBlockJob *s)
602bd48bde8SPaolo Bonzini {
603bd48bde8SPaolo Bonzini     while (s->in_flight > 0) {
6041181e19aSMax Reitz         mirror_wait_for_free_in_flight_slot(s);
605bd48bde8SPaolo Bonzini     }
606893f7ebaSPaolo Bonzini }
607893f7ebaSPaolo Bonzini 
608737efc1eSJohn Snow /**
609737efc1eSJohn Snow  * mirror_exit_common: handle both abort() and prepare() cases.
610737efc1eSJohn Snow  * for .prepare, returns 0 on success and -errno on failure.
611737efc1eSJohn Snow  * for .abort cases, denoted by abort = true, MUST return 0.
612737efc1eSJohn Snow  */
613737efc1eSJohn Snow static int mirror_exit_common(Job *job)
6145a7e7a0bSStefan Hajnoczi {
6151908a559SKevin Wolf     MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
6161908a559SKevin Wolf     BlockJob *bjob = &s->common;
617429076e8SMax Reitz     MirrorBDSOpaque *bs_opaque = s->mirror_top_bs->opaque;
6185a7e7a0bSStefan Hajnoczi     AioContext *replace_aio_context = NULL;
619138f9fffSMax Reitz     BlockDriverState *src = s->mirror_top_bs->backing->bs;
620e253f4b8SKevin Wolf     BlockDriverState *target_bs = blk_bs(s->target);
6214ef85a9cSKevin Wolf     BlockDriverState *mirror_top_bs = s->mirror_top_bs;
62212fa4af6SKevin Wolf     Error *local_err = NULL;
623737efc1eSJohn Snow     bool abort = job->ret < 0;
624737efc1eSJohn Snow     int ret = 0;
625737efc1eSJohn Snow 
626737efc1eSJohn Snow     if (s->prepared) {
627737efc1eSJohn Snow         return 0;
628737efc1eSJohn Snow     }
629737efc1eSJohn Snow     s->prepared = true;
6303f09bfbcSKevin Wolf 
6312119882cSPaolo Bonzini     bdrv_release_dirty_bitmap(src, s->dirty_bitmap);
6322119882cSPaolo Bonzini 
6337b508f6bSJohn Snow     /* Make sure that the source BDS doesn't go away during bdrv_replace_node,
6347b508f6bSJohn Snow      * before we can call bdrv_drained_end */
6353f09bfbcSKevin Wolf     bdrv_ref(src);
6364ef85a9cSKevin Wolf     bdrv_ref(mirror_top_bs);
6377d9fcb39SKevin Wolf     bdrv_ref(target_bs);
6387d9fcb39SKevin Wolf 
6397d9fcb39SKevin Wolf     /* Remove target parent that still uses BLK_PERM_WRITE/RESIZE before
6407d9fcb39SKevin Wolf      * inserting target_bs at s->to_replace, where we might not be able to get
64163c8ef28SKevin Wolf      * these permissions.
64263c8ef28SKevin Wolf      *
64363c8ef28SKevin Wolf      * Note that blk_unref() alone doesn't necessarily drop permissions because
64463c8ef28SKevin Wolf      * we might be running nested inside mirror_drain(), which takes an extra
64563c8ef28SKevin Wolf      * reference, so use an explicit blk_set_perm() first. */
64663c8ef28SKevin Wolf     blk_set_perm(s->target, 0, BLK_PERM_ALL, &error_abort);
6477d9fcb39SKevin Wolf     blk_unref(s->target);
6487d9fcb39SKevin Wolf     s->target = NULL;
6494ef85a9cSKevin Wolf 
6504ef85a9cSKevin Wolf     /* We don't access the source any more. Dropping any WRITE/RESIZE is
6514ef85a9cSKevin Wolf      * required before it could become a backing file of target_bs. */
6524ef85a9cSKevin Wolf     bdrv_child_try_set_perm(mirror_top_bs->backing, 0, BLK_PERM_ALL,
6534ef85a9cSKevin Wolf                             &error_abort);
654737efc1eSJohn Snow     if (!abort && s->backing_mode == MIRROR_SOURCE_BACKING_CHAIN) {
6554ef85a9cSKevin Wolf         BlockDriverState *backing = s->is_none_mode ? src : s->base;
6564ef85a9cSKevin Wolf         if (backing_bs(target_bs) != backing) {
65712fa4af6SKevin Wolf             bdrv_set_backing_hd(target_bs, backing, &local_err);
65812fa4af6SKevin Wolf             if (local_err) {
65912fa4af6SKevin Wolf                 error_report_err(local_err);
6607b508f6bSJohn Snow                 ret = -EPERM;
66112fa4af6SKevin Wolf             }
6624ef85a9cSKevin Wolf         }
6634ef85a9cSKevin Wolf     }
6645a7e7a0bSStefan Hajnoczi 
6655a7e7a0bSStefan Hajnoczi     if (s->to_replace) {
6665a7e7a0bSStefan Hajnoczi         replace_aio_context = bdrv_get_aio_context(s->to_replace);
6675a7e7a0bSStefan Hajnoczi         aio_context_acquire(replace_aio_context);
6685a7e7a0bSStefan Hajnoczi     }
6695a7e7a0bSStefan Hajnoczi 
670737efc1eSJohn Snow     if (s->should_complete && !abort) {
671737efc1eSJohn Snow         BlockDriverState *to_replace = s->to_replace ?: src;
672*1ba79388SAlberto Garcia         bool ro = bdrv_is_read_only(to_replace);
67340365552SKevin Wolf 
674*1ba79388SAlberto Garcia         if (ro != bdrv_is_read_only(target_bs)) {
675*1ba79388SAlberto Garcia             bdrv_reopen_set_read_only(target_bs, ro, NULL);
6765a7e7a0bSStefan Hajnoczi         }
677b8804815SKevin Wolf 
678b8804815SKevin Wolf         /* The mirror job has no requests in flight any more, but we need to
679b8804815SKevin Wolf          * drain potential other users of the BDS before changing the graph. */
680e253f4b8SKevin Wolf         bdrv_drained_begin(target_bs);
6815fe31c25SKevin Wolf         bdrv_replace_node(to_replace, target_bs, &local_err);
682e253f4b8SKevin Wolf         bdrv_drained_end(target_bs);
6835fe31c25SKevin Wolf         if (local_err) {
6845fe31c25SKevin Wolf             error_report_err(local_err);
6857b508f6bSJohn Snow             ret = -EPERM;
6865fe31c25SKevin Wolf         }
6875a7e7a0bSStefan Hajnoczi     }
6885a7e7a0bSStefan Hajnoczi     if (s->to_replace) {
6895a7e7a0bSStefan Hajnoczi         bdrv_op_unblock_all(s->to_replace, s->replace_blocker);
6905a7e7a0bSStefan Hajnoczi         error_free(s->replace_blocker);
6915a7e7a0bSStefan Hajnoczi         bdrv_unref(s->to_replace);
6925a7e7a0bSStefan Hajnoczi     }
6935a7e7a0bSStefan Hajnoczi     if (replace_aio_context) {
6945a7e7a0bSStefan Hajnoczi         aio_context_release(replace_aio_context);
6955a7e7a0bSStefan Hajnoczi     }
6965a7e7a0bSStefan Hajnoczi     g_free(s->replaces);
6977d9fcb39SKevin Wolf     bdrv_unref(target_bs);
6984ef85a9cSKevin Wolf 
6994ef85a9cSKevin Wolf     /* Remove the mirror filter driver from the graph. Before this, get rid of
7004ef85a9cSKevin Wolf      * the blockers on the intermediate nodes so that the resulting state is
7010bf74767SKevin Wolf      * valid. Also give up permissions on mirror_top_bs->backing, which might
7020bf74767SKevin Wolf      * block the removal. */
7031908a559SKevin Wolf     block_job_remove_all_bdrv(bjob);
704c1cef672SFam Zheng     bdrv_child_try_set_perm(mirror_top_bs->backing, 0, BLK_PERM_ALL,
705c1cef672SFam Zheng                             &error_abort);
7065fe31c25SKevin Wolf     bdrv_replace_node(mirror_top_bs, backing_bs(mirror_top_bs), &error_abort);
7074ef85a9cSKevin Wolf 
7084ef85a9cSKevin Wolf     /* We just changed the BDS the job BB refers to (with either or both of the
7095fe31c25SKevin Wolf      * bdrv_replace_node() calls), so switch the BB back so the cleanup does
7105fe31c25SKevin Wolf      * the right thing. We don't need any permissions any more now. */
7111908a559SKevin Wolf     blk_remove_bs(bjob->blk);
7121908a559SKevin Wolf     blk_set_perm(bjob->blk, 0, BLK_PERM_ALL, &error_abort);
7131908a559SKevin Wolf     blk_insert_bs(bjob->blk, mirror_top_bs, &error_abort);
7144ef85a9cSKevin Wolf 
715429076e8SMax Reitz     bs_opaque->job = NULL;
7164ef85a9cSKevin Wolf 
717176c3699SFam Zheng     bdrv_drained_end(src);
7184ef85a9cSKevin Wolf     bdrv_unref(mirror_top_bs);
7193f09bfbcSKevin Wolf     bdrv_unref(src);
7207b508f6bSJohn Snow 
721737efc1eSJohn Snow     return ret;
722737efc1eSJohn Snow }
723737efc1eSJohn Snow 
724737efc1eSJohn Snow static int mirror_prepare(Job *job)
725737efc1eSJohn Snow {
726737efc1eSJohn Snow     return mirror_exit_common(job);
727737efc1eSJohn Snow }
728737efc1eSJohn Snow 
729737efc1eSJohn Snow static void mirror_abort(Job *job)
730737efc1eSJohn Snow {
731737efc1eSJohn Snow     int ret = mirror_exit_common(job);
732737efc1eSJohn Snow     assert(ret == 0);
7335a7e7a0bSStefan Hajnoczi }
7345a7e7a0bSStefan Hajnoczi 
73549efb1f5SDenis V. Lunev static void mirror_throttle(MirrorBlockJob *s)
73649efb1f5SDenis V. Lunev {
73749efb1f5SDenis V. Lunev     int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
73849efb1f5SDenis V. Lunev 
73918bb6928SKevin Wolf     if (now - s->last_pause_ns > BLOCK_JOB_SLICE_TIME) {
74049efb1f5SDenis V. Lunev         s->last_pause_ns = now;
7415d43e86eSKevin Wolf         job_sleep_ns(&s->common.job, 0);
74249efb1f5SDenis V. Lunev     } else {
743da01ff7fSKevin Wolf         job_pause_point(&s->common.job);
74449efb1f5SDenis V. Lunev     }
74549efb1f5SDenis V. Lunev }
74649efb1f5SDenis V. Lunev 
747c0b363adSDenis V. Lunev static int coroutine_fn mirror_dirty_init(MirrorBlockJob *s)
748c0b363adSDenis V. Lunev {
74923ca459aSEric Blake     int64_t offset;
750c0b363adSDenis V. Lunev     BlockDriverState *base = s->base;
751138f9fffSMax Reitz     BlockDriverState *bs = s->mirror_top_bs->backing->bs;
752c0b363adSDenis V. Lunev     BlockDriverState *target_bs = blk_bs(s->target);
75323ca459aSEric Blake     int ret;
75451b0a488SEric Blake     int64_t count;
755c0b363adSDenis V. Lunev 
756b7d5062cSDenis V. Lunev     if (base == NULL && !bdrv_has_zero_init(target_bs)) {
757c7c2769cSDenis V. Lunev         if (!bdrv_can_write_zeroes_with_unmap(target_bs)) {
758e0d7f73eSEric Blake             bdrv_set_dirty_bitmap(s->dirty_bitmap, 0, s->bdev_length);
759b7d5062cSDenis V. Lunev             return 0;
760b7d5062cSDenis V. Lunev         }
761b7d5062cSDenis V. Lunev 
76290ab48ebSAnton Nefedov         s->initial_zeroing_ongoing = true;
76323ca459aSEric Blake         for (offset = 0; offset < s->bdev_length; ) {
76423ca459aSEric Blake             int bytes = MIN(s->bdev_length - offset,
76523ca459aSEric Blake                             QEMU_ALIGN_DOWN(INT_MAX, s->granularity));
766c7c2769cSDenis V. Lunev 
767c7c2769cSDenis V. Lunev             mirror_throttle(s);
768c7c2769cSDenis V. Lunev 
769daa7f2f9SKevin Wolf             if (job_is_cancelled(&s->common.job)) {
77090ab48ebSAnton Nefedov                 s->initial_zeroing_ongoing = false;
771c7c2769cSDenis V. Lunev                 return 0;
772c7c2769cSDenis V. Lunev             }
773c7c2769cSDenis V. Lunev 
774c7c2769cSDenis V. Lunev             if (s->in_flight >= MAX_IN_FLIGHT) {
77567adf4b3SEric Blake                 trace_mirror_yield(s, UINT64_MAX, s->buf_free_count,
77667adf4b3SEric Blake                                    s->in_flight);
7771181e19aSMax Reitz                 mirror_wait_for_free_in_flight_slot(s);
778c7c2769cSDenis V. Lunev                 continue;
779c7c2769cSDenis V. Lunev             }
780c7c2769cSDenis V. Lunev 
7814295c5fcSMax Reitz             mirror_perform(s, offset, bytes, MIRROR_METHOD_ZERO);
78223ca459aSEric Blake             offset += bytes;
783c7c2769cSDenis V. Lunev         }
784c7c2769cSDenis V. Lunev 
785bae8196dSPaolo Bonzini         mirror_wait_for_all_io(s);
78690ab48ebSAnton Nefedov         s->initial_zeroing_ongoing = false;
787c7c2769cSDenis V. Lunev     }
788c7c2769cSDenis V. Lunev 
789c0b363adSDenis V. Lunev     /* First part, loop on the sectors and initialize the dirty bitmap.  */
79023ca459aSEric Blake     for (offset = 0; offset < s->bdev_length; ) {
791c0b363adSDenis V. Lunev         /* Just to make sure we are not exceeding int limit. */
79223ca459aSEric Blake         int bytes = MIN(s->bdev_length - offset,
79323ca459aSEric Blake                         QEMU_ALIGN_DOWN(INT_MAX, s->granularity));
794c0b363adSDenis V. Lunev 
795c0b363adSDenis V. Lunev         mirror_throttle(s);
796c0b363adSDenis V. Lunev 
797daa7f2f9SKevin Wolf         if (job_is_cancelled(&s->common.job)) {
798c0b363adSDenis V. Lunev             return 0;
799c0b363adSDenis V. Lunev         }
800c0b363adSDenis V. Lunev 
80123ca459aSEric Blake         ret = bdrv_is_allocated_above(bs, base, offset, bytes, &count);
802c0b363adSDenis V. Lunev         if (ret < 0) {
803c0b363adSDenis V. Lunev             return ret;
804c0b363adSDenis V. Lunev         }
805c0b363adSDenis V. Lunev 
80623ca459aSEric Blake         assert(count);
807b7d5062cSDenis V. Lunev         if (ret == 1) {
80823ca459aSEric Blake             bdrv_set_dirty_bitmap(s->dirty_bitmap, offset, count);
809c0b363adSDenis V. Lunev         }
81023ca459aSEric Blake         offset += count;
811c0b363adSDenis V. Lunev     }
812c0b363adSDenis V. Lunev     return 0;
813c0b363adSDenis V. Lunev }
814c0b363adSDenis V. Lunev 
815bdffb31dSPaolo Bonzini /* Called when going out of the streaming phase to flush the bulk of the
816bdffb31dSPaolo Bonzini  * data to the medium, or just before completing.
817bdffb31dSPaolo Bonzini  */
818bdffb31dSPaolo Bonzini static int mirror_flush(MirrorBlockJob *s)
819bdffb31dSPaolo Bonzini {
820bdffb31dSPaolo Bonzini     int ret = blk_flush(s->target);
821bdffb31dSPaolo Bonzini     if (ret < 0) {
822bdffb31dSPaolo Bonzini         if (mirror_error_action(s, false, -ret) == BLOCK_ERROR_ACTION_REPORT) {
823bdffb31dSPaolo Bonzini             s->ret = ret;
824bdffb31dSPaolo Bonzini         }
825bdffb31dSPaolo Bonzini     }
826bdffb31dSPaolo Bonzini     return ret;
827bdffb31dSPaolo Bonzini }
828bdffb31dSPaolo Bonzini 
829f67432a2SJohn Snow static int coroutine_fn mirror_run(Job *job, Error **errp)
830893f7ebaSPaolo Bonzini {
831f67432a2SJohn Snow     MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
832138f9fffSMax Reitz     BlockDriverState *bs = s->mirror_top_bs->backing->bs;
833e253f4b8SKevin Wolf     BlockDriverState *target_bs = blk_bs(s->target);
8349a0cec66SPaolo Bonzini     bool need_drain = true;
835c0b363adSDenis V. Lunev     int64_t length;
836b812f671SPaolo Bonzini     BlockDriverInfo bdi;
8371d33936eSJeff Cody     char backing_filename[2]; /* we only need 2 characters because we are only
8381d33936eSJeff Cody                                  checking for a NULL string */
839893f7ebaSPaolo Bonzini     int ret = 0;
840893f7ebaSPaolo Bonzini 
841daa7f2f9SKevin Wolf     if (job_is_cancelled(&s->common.job)) {
842893f7ebaSPaolo Bonzini         goto immediate_exit;
843893f7ebaSPaolo Bonzini     }
844893f7ebaSPaolo Bonzini 
845b21c7652SMax Reitz     s->bdev_length = bdrv_getlength(bs);
846b21c7652SMax Reitz     if (s->bdev_length < 0) {
847b21c7652SMax Reitz         ret = s->bdev_length;
848373df5b1SFam Zheng         goto immediate_exit;
849becc347eSKevin Wolf     }
850becc347eSKevin Wolf 
851becc347eSKevin Wolf     /* Active commit must resize the base image if its size differs from the
852becc347eSKevin Wolf      * active layer. */
853becc347eSKevin Wolf     if (s->base == blk_bs(s->target)) {
854becc347eSKevin Wolf         int64_t base_length;
855becc347eSKevin Wolf 
856becc347eSKevin Wolf         base_length = blk_getlength(s->target);
857becc347eSKevin Wolf         if (base_length < 0) {
858becc347eSKevin Wolf             ret = base_length;
859becc347eSKevin Wolf             goto immediate_exit;
860becc347eSKevin Wolf         }
861becc347eSKevin Wolf 
862becc347eSKevin Wolf         if (s->bdev_length > base_length) {
8633a691c50SMax Reitz             ret = blk_truncate(s->target, s->bdev_length, PREALLOC_MODE_OFF,
8643a691c50SMax Reitz                                NULL);
865becc347eSKevin Wolf             if (ret < 0) {
866becc347eSKevin Wolf                 goto immediate_exit;
867becc347eSKevin Wolf             }
868becc347eSKevin Wolf         }
869becc347eSKevin Wolf     }
870becc347eSKevin Wolf 
871becc347eSKevin Wolf     if (s->bdev_length == 0) {
8722e1795b5SKevin Wolf         /* Transition to the READY state and wait for complete. */
8732e1795b5SKevin Wolf         job_transition_to_ready(&s->common.job);
8749e48b025SFam Zheng         s->synced = true;
875d06107adSMax Reitz         s->actively_synced = true;
876daa7f2f9SKevin Wolf         while (!job_is_cancelled(&s->common.job) && !s->should_complete) {
877198c49ccSKevin Wolf             job_yield(&s->common.job);
8789e48b025SFam Zheng         }
879daa7f2f9SKevin Wolf         s->common.job.cancelled = false;
8809e48b025SFam Zheng         goto immediate_exit;
881893f7ebaSPaolo Bonzini     }
882893f7ebaSPaolo Bonzini 
883b21c7652SMax Reitz     length = DIV_ROUND_UP(s->bdev_length, s->granularity);
884402a4741SPaolo Bonzini     s->in_flight_bitmap = bitmap_new(length);
885402a4741SPaolo Bonzini 
886b812f671SPaolo Bonzini     /* If we have no backing file yet in the destination, we cannot let
887b812f671SPaolo Bonzini      * the destination do COW.  Instead, we copy sectors around the
888b812f671SPaolo Bonzini      * dirty data if needed.  We need a bitmap to do that.
889b812f671SPaolo Bonzini      */
890e253f4b8SKevin Wolf     bdrv_get_backing_filename(target_bs, backing_filename,
891b812f671SPaolo Bonzini                               sizeof(backing_filename));
892e253f4b8SKevin Wolf     if (!bdrv_get_info(target_bs, &bdi) && bdi.cluster_size) {
893b436982fSEric Blake         s->target_cluster_size = bdi.cluster_size;
894b436982fSEric Blake     } else {
895b436982fSEric Blake         s->target_cluster_size = BDRV_SECTOR_SIZE;
896c3cc95bdSFam Zheng     }
897b436982fSEric Blake     if (backing_filename[0] && !target_bs->backing &&
898b436982fSEric Blake         s->granularity < s->target_cluster_size) {
899b436982fSEric Blake         s->buf_size = MAX(s->buf_size, s->target_cluster_size);
900b812f671SPaolo Bonzini         s->cow_bitmap = bitmap_new(length);
901b812f671SPaolo Bonzini     }
902e253f4b8SKevin Wolf     s->max_iov = MIN(bs->bl.max_iov, target_bs->bl.max_iov);
903b812f671SPaolo Bonzini 
9047504edf4SKevin Wolf     s->buf = qemu_try_blockalign(bs, s->buf_size);
9057504edf4SKevin Wolf     if (s->buf == NULL) {
9067504edf4SKevin Wolf         ret = -ENOMEM;
9077504edf4SKevin Wolf         goto immediate_exit;
9087504edf4SKevin Wolf     }
9097504edf4SKevin Wolf 
910402a4741SPaolo Bonzini     mirror_free_init(s);
911893f7ebaSPaolo Bonzini 
91249efb1f5SDenis V. Lunev     s->last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
91303544a6eSFam Zheng     if (!s->is_none_mode) {
914c0b363adSDenis V. Lunev         ret = mirror_dirty_init(s);
915daa7f2f9SKevin Wolf         if (ret < 0 || job_is_cancelled(&s->common.job)) {
9164c0cbd6fSFam Zheng             goto immediate_exit;
9174c0cbd6fSFam Zheng         }
918893f7ebaSPaolo Bonzini     }
919893f7ebaSPaolo Bonzini 
920dc162c8eSFam Zheng     assert(!s->dbi);
921715a74d8SEric Blake     s->dbi = bdrv_dirty_iter_new(s->dirty_bitmap);
922893f7ebaSPaolo Bonzini     for (;;) {
923cc8c9d6cSPaolo Bonzini         uint64_t delay_ns = 0;
92449efb1f5SDenis V. Lunev         int64_t cnt, delta;
925893f7ebaSPaolo Bonzini         bool should_complete;
926893f7ebaSPaolo Bonzini 
927d06107adSMax Reitz         /* Do not start passive operations while there are active
928d06107adSMax Reitz          * writes in progress */
929d06107adSMax Reitz         while (s->in_active_write_counter) {
930d06107adSMax Reitz             mirror_wait_for_any_operation(s, true);
931d06107adSMax Reitz         }
932d06107adSMax Reitz 
933bd48bde8SPaolo Bonzini         if (s->ret < 0) {
934bd48bde8SPaolo Bonzini             ret = s->ret;
935893f7ebaSPaolo Bonzini             goto immediate_exit;
936893f7ebaSPaolo Bonzini         }
937bd48bde8SPaolo Bonzini 
938da01ff7fSKevin Wolf         job_pause_point(&s->common.job);
939565ac01fSStefan Hajnoczi 
94020dca810SJohn Snow         cnt = bdrv_get_dirty_count(s->dirty_bitmap);
94105df8a6aSKevin Wolf         /* cnt is the number of dirty bytes remaining and s->bytes_in_flight is
94205df8a6aSKevin Wolf          * the number of bytes currently being processed; together those are
94305df8a6aSKevin Wolf          * the current remaining operation length */
94430a5c887SKevin Wolf         job_progress_set_remaining(&s->common.job, s->bytes_in_flight + cnt);
945bd48bde8SPaolo Bonzini 
946bd48bde8SPaolo Bonzini         /* Note that even when no rate limit is applied we need to yield
947a7282330SFam Zheng          * periodically with no pending I/O so that bdrv_drain_all() returns.
94818bb6928SKevin Wolf          * We do so every BLKOCK_JOB_SLICE_TIME nanoseconds, or when there is
94918bb6928SKevin Wolf          * an error, or when the source is clean, whichever comes first. */
95049efb1f5SDenis V. Lunev         delta = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - s->last_pause_ns;
95118bb6928SKevin Wolf         if (delta < BLOCK_JOB_SLICE_TIME &&
952bd48bde8SPaolo Bonzini             s->common.iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
953cf56a3c6SDenis V. Lunev             if (s->in_flight >= MAX_IN_FLIGHT || s->buf_free_count == 0 ||
954402a4741SPaolo Bonzini                 (cnt == 0 && s->in_flight > 0)) {
9559a46dba7SEric Blake                 trace_mirror_yield(s, cnt, s->buf_free_count, s->in_flight);
9561181e19aSMax Reitz                 mirror_wait_for_free_in_flight_slot(s);
957bd48bde8SPaolo Bonzini                 continue;
958bd48bde8SPaolo Bonzini             } else if (cnt != 0) {
959cc8c9d6cSPaolo Bonzini                 delay_ns = mirror_iteration(s);
960893f7ebaSPaolo Bonzini             }
961cc8c9d6cSPaolo Bonzini         }
962893f7ebaSPaolo Bonzini 
963893f7ebaSPaolo Bonzini         should_complete = false;
964bd48bde8SPaolo Bonzini         if (s->in_flight == 0 && cnt == 0) {
965893f7ebaSPaolo Bonzini             trace_mirror_before_flush(s);
966bdffb31dSPaolo Bonzini             if (!s->synced) {
967bdffb31dSPaolo Bonzini                 if (mirror_flush(s) < 0) {
968bdffb31dSPaolo Bonzini                     /* Go check s->ret.  */
969bdffb31dSPaolo Bonzini                     continue;
970893f7ebaSPaolo Bonzini                 }
971893f7ebaSPaolo Bonzini                 /* We're out of the streaming phase.  From now on, if the job
972893f7ebaSPaolo Bonzini                  * is cancelled we will actually complete all pending I/O and
973893f7ebaSPaolo Bonzini                  * report completion.  This way, block-job-cancel will leave
974893f7ebaSPaolo Bonzini                  * the target in a consistent state.
975893f7ebaSPaolo Bonzini                  */
9762e1795b5SKevin Wolf                 job_transition_to_ready(&s->common.job);
977d63ffd87SPaolo Bonzini                 s->synced = true;
978d06107adSMax Reitz                 if (s->copy_mode != MIRROR_COPY_MODE_BACKGROUND) {
979d06107adSMax Reitz                     s->actively_synced = true;
980d06107adSMax Reitz                 }
981d63ffd87SPaolo Bonzini             }
982d63ffd87SPaolo Bonzini 
983d63ffd87SPaolo Bonzini             should_complete = s->should_complete ||
984daa7f2f9SKevin Wolf                 job_is_cancelled(&s->common.job);
98520dca810SJohn Snow             cnt = bdrv_get_dirty_count(s->dirty_bitmap);
986893f7ebaSPaolo Bonzini         }
987893f7ebaSPaolo Bonzini 
988893f7ebaSPaolo Bonzini         if (cnt == 0 && should_complete) {
989893f7ebaSPaolo Bonzini             /* The dirty bitmap is not updated while operations are pending.
990893f7ebaSPaolo Bonzini              * If we're about to exit, wait for pending operations before
991893f7ebaSPaolo Bonzini              * calling bdrv_get_dirty_count(bs), or we may exit while the
992893f7ebaSPaolo Bonzini              * source has dirty data to copy!
993893f7ebaSPaolo Bonzini              *
994893f7ebaSPaolo Bonzini              * Note that I/O can be submitted by the guest while
9959a0cec66SPaolo Bonzini              * mirror_populate runs, so pause it now.  Before deciding
9969a0cec66SPaolo Bonzini              * whether to switch to target check one last time if I/O has
9979a0cec66SPaolo Bonzini              * come in the meanwhile, and if not flush the data to disk.
998893f7ebaSPaolo Bonzini              */
9999a46dba7SEric Blake             trace_mirror_before_drain(s, cnt);
10009a0cec66SPaolo Bonzini 
10019a0cec66SPaolo Bonzini             bdrv_drained_begin(bs);
100220dca810SJohn Snow             cnt = bdrv_get_dirty_count(s->dirty_bitmap);
1003bdffb31dSPaolo Bonzini             if (cnt > 0 || mirror_flush(s) < 0) {
10049a0cec66SPaolo Bonzini                 bdrv_drained_end(bs);
10059a0cec66SPaolo Bonzini                 continue;
10069a0cec66SPaolo Bonzini             }
10079a0cec66SPaolo Bonzini 
10089a0cec66SPaolo Bonzini             /* The two disks are in sync.  Exit and report successful
10099a0cec66SPaolo Bonzini              * completion.
10109a0cec66SPaolo Bonzini              */
10119a0cec66SPaolo Bonzini             assert(QLIST_EMPTY(&bs->tracked_requests));
1012daa7f2f9SKevin Wolf             s->common.job.cancelled = false;
10139a0cec66SPaolo Bonzini             need_drain = false;
10149a0cec66SPaolo Bonzini             break;
1015893f7ebaSPaolo Bonzini         }
1016893f7ebaSPaolo Bonzini 
1017893f7ebaSPaolo Bonzini         ret = 0;
1018ddc4115eSStefan Hajnoczi 
1019ddc4115eSStefan Hajnoczi         if (s->synced && !should_complete) {
102018bb6928SKevin Wolf             delay_ns = (s->in_flight == 0 &&
102118bb6928SKevin Wolf                         cnt == 0 ? BLOCK_JOB_SLICE_TIME : 0);
1022ddc4115eSStefan Hajnoczi         }
10239a46dba7SEric Blake         trace_mirror_before_sleep(s, cnt, s->synced, delay_ns);
10245d43e86eSKevin Wolf         job_sleep_ns(&s->common.job, delay_ns);
1025daa7f2f9SKevin Wolf         if (job_is_cancelled(&s->common.job) &&
1026004e95dfSKevin Wolf             (!s->synced || s->common.job.force_cancel))
1027eb36639fSMax Reitz         {
1028893f7ebaSPaolo Bonzini             break;
1029893f7ebaSPaolo Bonzini         }
103049efb1f5SDenis V. Lunev         s->last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
1031893f7ebaSPaolo Bonzini     }
1032893f7ebaSPaolo Bonzini 
1033893f7ebaSPaolo Bonzini immediate_exit:
1034bd48bde8SPaolo Bonzini     if (s->in_flight > 0) {
1035bd48bde8SPaolo Bonzini         /* We get here only if something went wrong.  Either the job failed,
1036bd48bde8SPaolo Bonzini          * or it was cancelled prematurely so that we do not guarantee that
1037bd48bde8SPaolo Bonzini          * the target is a copy of the source.
1038bd48bde8SPaolo Bonzini          */
1039004e95dfSKevin Wolf         assert(ret < 0 || ((s->common.job.force_cancel || !s->synced) &&
1040daa7f2f9SKevin Wolf                job_is_cancelled(&s->common.job)));
10419a0cec66SPaolo Bonzini         assert(need_drain);
1042bae8196dSPaolo Bonzini         mirror_wait_for_all_io(s);
1043bd48bde8SPaolo Bonzini     }
1044bd48bde8SPaolo Bonzini 
1045bd48bde8SPaolo Bonzini     assert(s->in_flight == 0);
10467191bf31SMarkus Armbruster     qemu_vfree(s->buf);
1047b812f671SPaolo Bonzini     g_free(s->cow_bitmap);
1048402a4741SPaolo Bonzini     g_free(s->in_flight_bitmap);
1049dc162c8eSFam Zheng     bdrv_dirty_iter_free(s->dbi);
10505a7e7a0bSStefan Hajnoczi 
10519a0cec66SPaolo Bonzini     if (need_drain) {
1052e253f4b8SKevin Wolf         bdrv_drained_begin(bs);
10539a0cec66SPaolo Bonzini     }
1054f67432a2SJohn Snow 
1055f67432a2SJohn Snow     return ret;
1056893f7ebaSPaolo Bonzini }
1057893f7ebaSPaolo Bonzini 
10583453d972SKevin Wolf static void mirror_complete(Job *job, Error **errp)
1059d63ffd87SPaolo Bonzini {
10603453d972SKevin Wolf     MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
10614ef85a9cSKevin Wolf     BlockDriverState *target;
1062d63ffd87SPaolo Bonzini 
1063274fcceeSMax Reitz     target = blk_bs(s->target);
1064274fcceeSMax Reitz 
1065d63ffd87SPaolo Bonzini     if (!s->synced) {
10669df229c3SAlberto Garcia         error_setg(errp, "The active block job '%s' cannot be completed",
10673453d972SKevin Wolf                    job->id);
1068d63ffd87SPaolo Bonzini         return;
1069d63ffd87SPaolo Bonzini     }
1070d63ffd87SPaolo Bonzini 
1071274fcceeSMax Reitz     if (s->backing_mode == MIRROR_OPEN_BACKING_CHAIN) {
1072274fcceeSMax Reitz         int ret;
1073274fcceeSMax Reitz 
1074274fcceeSMax Reitz         assert(!target->backing);
1075274fcceeSMax Reitz         ret = bdrv_open_backing_file(target, NULL, "backing", errp);
1076274fcceeSMax Reitz         if (ret < 0) {
1077274fcceeSMax Reitz             return;
1078274fcceeSMax Reitz         }
1079274fcceeSMax Reitz     }
1080274fcceeSMax Reitz 
108115d67298SChanglong Xie     /* block all operations on to_replace bs */
108209158f00SBenoît Canet     if (s->replaces) {
10835a7e7a0bSStefan Hajnoczi         AioContext *replace_aio_context;
10845a7e7a0bSStefan Hajnoczi 
1085e12f3784SWen Congyang         s->to_replace = bdrv_find_node(s->replaces);
108609158f00SBenoît Canet         if (!s->to_replace) {
1087e12f3784SWen Congyang             error_setg(errp, "Node name '%s' not found", s->replaces);
108809158f00SBenoît Canet             return;
108909158f00SBenoît Canet         }
109009158f00SBenoît Canet 
10915a7e7a0bSStefan Hajnoczi         replace_aio_context = bdrv_get_aio_context(s->to_replace);
10925a7e7a0bSStefan Hajnoczi         aio_context_acquire(replace_aio_context);
10935a7e7a0bSStefan Hajnoczi 
10944ef85a9cSKevin Wolf         /* TODO Translate this into permission system. Current definition of
10954ef85a9cSKevin Wolf          * GRAPH_MOD would require to request it for the parents; they might
10964ef85a9cSKevin Wolf          * not even be BlockDriverStates, however, so a BdrvChild can't address
10974ef85a9cSKevin Wolf          * them. May need redefinition of GRAPH_MOD. */
109809158f00SBenoît Canet         error_setg(&s->replace_blocker,
109909158f00SBenoît Canet                    "block device is in use by block-job-complete");
110009158f00SBenoît Canet         bdrv_op_block_all(s->to_replace, s->replace_blocker);
110109158f00SBenoît Canet         bdrv_ref(s->to_replace);
11025a7e7a0bSStefan Hajnoczi 
11035a7e7a0bSStefan Hajnoczi         aio_context_release(replace_aio_context);
110409158f00SBenoît Canet     }
110509158f00SBenoît Canet 
1106d63ffd87SPaolo Bonzini     s->should_complete = true;
11073d70ff53SKevin Wolf     job_enter(job);
1108d63ffd87SPaolo Bonzini }
1109d63ffd87SPaolo Bonzini 
1110da01ff7fSKevin Wolf static void mirror_pause(Job *job)
1111565ac01fSStefan Hajnoczi {
1112da01ff7fSKevin Wolf     MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
1113565ac01fSStefan Hajnoczi 
1114bae8196dSPaolo Bonzini     mirror_wait_for_all_io(s);
1115565ac01fSStefan Hajnoczi }
1116565ac01fSStefan Hajnoczi 
111789bd0305SKevin Wolf static bool mirror_drained_poll(BlockJob *job)
111889bd0305SKevin Wolf {
111989bd0305SKevin Wolf     MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
112089bd0305SKevin Wolf     return !!s->in_flight;
112189bd0305SKevin Wolf }
112289bd0305SKevin Wolf 
1123565ac01fSStefan Hajnoczi static void mirror_attached_aio_context(BlockJob *job, AioContext *new_context)
1124565ac01fSStefan Hajnoczi {
1125565ac01fSStefan Hajnoczi     MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
1126565ac01fSStefan Hajnoczi 
1127565ac01fSStefan Hajnoczi     blk_set_aio_context(s->target, new_context);
1128565ac01fSStefan Hajnoczi }
1129565ac01fSStefan Hajnoczi 
1130bae8196dSPaolo Bonzini static void mirror_drain(BlockJob *job)
1131bae8196dSPaolo Bonzini {
1132bae8196dSPaolo Bonzini     MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
1133bae8196dSPaolo Bonzini 
1134bae8196dSPaolo Bonzini     /* Need to keep a reference in case blk_drain triggers execution
1135bae8196dSPaolo Bonzini      * of mirror_complete...
1136bae8196dSPaolo Bonzini      */
1137bae8196dSPaolo Bonzini     if (s->target) {
1138bae8196dSPaolo Bonzini         BlockBackend *target = s->target;
1139bae8196dSPaolo Bonzini         blk_ref(target);
1140bae8196dSPaolo Bonzini         blk_drain(target);
1141bae8196dSPaolo Bonzini         blk_unref(target);
1142bae8196dSPaolo Bonzini     }
1143bae8196dSPaolo Bonzini }
1144bae8196dSPaolo Bonzini 
11453fc4b10aSFam Zheng static const BlockJobDriver mirror_job_driver = {
114633e9e9bdSKevin Wolf     .job_driver = {
1147893f7ebaSPaolo Bonzini         .instance_size          = sizeof(MirrorBlockJob),
11488e4c8700SKevin Wolf         .job_type               = JOB_TYPE_MIRROR,
114980fa2c75SKevin Wolf         .free                   = block_job_free,
1150b15de828SKevin Wolf         .user_resume            = block_job_user_resume,
1151b69f777dSKevin Wolf         .drain                  = block_job_drain,
1152f67432a2SJohn Snow         .run                    = mirror_run,
1153737efc1eSJohn Snow         .prepare                = mirror_prepare,
1154737efc1eSJohn Snow         .abort                  = mirror_abort,
1155565ac01fSStefan Hajnoczi         .pause                  = mirror_pause,
1156da01ff7fSKevin Wolf         .complete               = mirror_complete,
11573453d972SKevin Wolf     },
115889bd0305SKevin Wolf     .drained_poll           = mirror_drained_poll,
1159565ac01fSStefan Hajnoczi     .attached_aio_context   = mirror_attached_aio_context,
1160bae8196dSPaolo Bonzini     .drain                  = mirror_drain,
1161893f7ebaSPaolo Bonzini };
1162893f7ebaSPaolo Bonzini 
116303544a6eSFam Zheng static const BlockJobDriver commit_active_job_driver = {
116433e9e9bdSKevin Wolf     .job_driver = {
116503544a6eSFam Zheng         .instance_size          = sizeof(MirrorBlockJob),
11668e4c8700SKevin Wolf         .job_type               = JOB_TYPE_COMMIT,
116780fa2c75SKevin Wolf         .free                   = block_job_free,
1168b15de828SKevin Wolf         .user_resume            = block_job_user_resume,
1169b69f777dSKevin Wolf         .drain                  = block_job_drain,
1170f67432a2SJohn Snow         .run                    = mirror_run,
1171737efc1eSJohn Snow         .prepare                = mirror_prepare,
1172737efc1eSJohn Snow         .abort                  = mirror_abort,
1173565ac01fSStefan Hajnoczi         .pause                  = mirror_pause,
1174da01ff7fSKevin Wolf         .complete               = mirror_complete,
11753453d972SKevin Wolf     },
117689bd0305SKevin Wolf     .drained_poll           = mirror_drained_poll,
1177565ac01fSStefan Hajnoczi     .attached_aio_context   = mirror_attached_aio_context,
1178bae8196dSPaolo Bonzini     .drain                  = mirror_drain,
117903544a6eSFam Zheng };
118003544a6eSFam Zheng 
1181d06107adSMax Reitz static void do_sync_target_write(MirrorBlockJob *job, MirrorMethod method,
1182d06107adSMax Reitz                                  uint64_t offset, uint64_t bytes,
1183d06107adSMax Reitz                                  QEMUIOVector *qiov, int flags)
1184d06107adSMax Reitz {
1185d06107adSMax Reitz     BdrvDirtyBitmapIter *iter;
1186d06107adSMax Reitz     QEMUIOVector target_qiov;
1187d06107adSMax Reitz     uint64_t dirty_offset;
1188d06107adSMax Reitz     int dirty_bytes;
1189d06107adSMax Reitz 
1190d06107adSMax Reitz     if (qiov) {
1191d06107adSMax Reitz         qemu_iovec_init(&target_qiov, qiov->niov);
1192d06107adSMax Reitz     }
1193d06107adSMax Reitz 
1194d06107adSMax Reitz     iter = bdrv_dirty_iter_new(job->dirty_bitmap);
1195d06107adSMax Reitz     bdrv_set_dirty_iter(iter, offset);
1196d06107adSMax Reitz 
1197d06107adSMax Reitz     while (true) {
1198d06107adSMax Reitz         bool valid_area;
1199d06107adSMax Reitz         int ret;
1200d06107adSMax Reitz 
1201d06107adSMax Reitz         bdrv_dirty_bitmap_lock(job->dirty_bitmap);
1202d06107adSMax Reitz         valid_area = bdrv_dirty_iter_next_area(iter, offset + bytes,
1203d06107adSMax Reitz                                                &dirty_offset, &dirty_bytes);
1204d06107adSMax Reitz         if (!valid_area) {
1205d06107adSMax Reitz             bdrv_dirty_bitmap_unlock(job->dirty_bitmap);
1206d06107adSMax Reitz             break;
1207d06107adSMax Reitz         }
1208d06107adSMax Reitz 
1209d06107adSMax Reitz         bdrv_reset_dirty_bitmap_locked(job->dirty_bitmap,
1210d06107adSMax Reitz                                        dirty_offset, dirty_bytes);
1211d06107adSMax Reitz         bdrv_dirty_bitmap_unlock(job->dirty_bitmap);
1212d06107adSMax Reitz 
1213d06107adSMax Reitz         job_progress_increase_remaining(&job->common.job, dirty_bytes);
1214d06107adSMax Reitz 
1215d06107adSMax Reitz         assert(dirty_offset - offset <= SIZE_MAX);
1216d06107adSMax Reitz         if (qiov) {
1217d06107adSMax Reitz             qemu_iovec_reset(&target_qiov);
1218d06107adSMax Reitz             qemu_iovec_concat(&target_qiov, qiov,
1219d06107adSMax Reitz                               dirty_offset - offset, dirty_bytes);
1220d06107adSMax Reitz         }
1221d06107adSMax Reitz 
1222d06107adSMax Reitz         switch (method) {
1223d06107adSMax Reitz         case MIRROR_METHOD_COPY:
1224d06107adSMax Reitz             ret = blk_co_pwritev(job->target, dirty_offset, dirty_bytes,
1225d06107adSMax Reitz                                  qiov ? &target_qiov : NULL, flags);
1226d06107adSMax Reitz             break;
1227d06107adSMax Reitz 
1228d06107adSMax Reitz         case MIRROR_METHOD_ZERO:
1229d06107adSMax Reitz             assert(!qiov);
1230d06107adSMax Reitz             ret = blk_co_pwrite_zeroes(job->target, dirty_offset, dirty_bytes,
1231d06107adSMax Reitz                                        flags);
1232d06107adSMax Reitz             break;
1233d06107adSMax Reitz 
1234d06107adSMax Reitz         case MIRROR_METHOD_DISCARD:
1235d06107adSMax Reitz             assert(!qiov);
1236d06107adSMax Reitz             ret = blk_co_pdiscard(job->target, dirty_offset, dirty_bytes);
1237d06107adSMax Reitz             break;
1238d06107adSMax Reitz 
1239d06107adSMax Reitz         default:
1240d06107adSMax Reitz             abort();
1241d06107adSMax Reitz         }
1242d06107adSMax Reitz 
1243d06107adSMax Reitz         if (ret >= 0) {
1244d06107adSMax Reitz             job_progress_update(&job->common.job, dirty_bytes);
1245d06107adSMax Reitz         } else {
1246d06107adSMax Reitz             BlockErrorAction action;
1247d06107adSMax Reitz 
1248d06107adSMax Reitz             bdrv_set_dirty_bitmap(job->dirty_bitmap, dirty_offset, dirty_bytes);
1249d06107adSMax Reitz             job->actively_synced = false;
1250d06107adSMax Reitz 
1251d06107adSMax Reitz             action = mirror_error_action(job, false, -ret);
1252d06107adSMax Reitz             if (action == BLOCK_ERROR_ACTION_REPORT) {
1253d06107adSMax Reitz                 if (!job->ret) {
1254d06107adSMax Reitz                     job->ret = ret;
1255d06107adSMax Reitz                 }
1256d06107adSMax Reitz                 break;
1257d06107adSMax Reitz             }
1258d06107adSMax Reitz         }
1259d06107adSMax Reitz     }
1260d06107adSMax Reitz 
1261d06107adSMax Reitz     bdrv_dirty_iter_free(iter);
1262d06107adSMax Reitz     if (qiov) {
1263d06107adSMax Reitz         qemu_iovec_destroy(&target_qiov);
1264d06107adSMax Reitz     }
1265d06107adSMax Reitz }
1266d06107adSMax Reitz 
1267d06107adSMax Reitz static MirrorOp *coroutine_fn active_write_prepare(MirrorBlockJob *s,
1268d06107adSMax Reitz                                                    uint64_t offset,
1269d06107adSMax Reitz                                                    uint64_t bytes)
1270d06107adSMax Reitz {
1271d06107adSMax Reitz     MirrorOp *op;
1272d06107adSMax Reitz     uint64_t start_chunk = offset / s->granularity;
1273d06107adSMax Reitz     uint64_t end_chunk = DIV_ROUND_UP(offset + bytes, s->granularity);
1274d06107adSMax Reitz 
1275d06107adSMax Reitz     op = g_new(MirrorOp, 1);
1276d06107adSMax Reitz     *op = (MirrorOp){
1277d06107adSMax Reitz         .s                  = s,
1278d06107adSMax Reitz         .offset             = offset,
1279d06107adSMax Reitz         .bytes              = bytes,
1280d06107adSMax Reitz         .is_active_write    = true,
1281d06107adSMax Reitz     };
1282d06107adSMax Reitz     qemu_co_queue_init(&op->waiting_requests);
1283d06107adSMax Reitz     QTAILQ_INSERT_TAIL(&s->ops_in_flight, op, next);
1284d06107adSMax Reitz 
1285d06107adSMax Reitz     s->in_active_write_counter++;
1286d06107adSMax Reitz 
1287d06107adSMax Reitz     mirror_wait_on_conflicts(op, s, offset, bytes);
1288d06107adSMax Reitz 
1289d06107adSMax Reitz     bitmap_set(s->in_flight_bitmap, start_chunk, end_chunk - start_chunk);
1290d06107adSMax Reitz 
1291d06107adSMax Reitz     return op;
1292d06107adSMax Reitz }
1293d06107adSMax Reitz 
1294d06107adSMax Reitz static void coroutine_fn active_write_settle(MirrorOp *op)
1295d06107adSMax Reitz {
1296d06107adSMax Reitz     uint64_t start_chunk = op->offset / op->s->granularity;
1297d06107adSMax Reitz     uint64_t end_chunk = DIV_ROUND_UP(op->offset + op->bytes,
1298d06107adSMax Reitz                                       op->s->granularity);
1299d06107adSMax Reitz 
1300d06107adSMax Reitz     if (!--op->s->in_active_write_counter && op->s->actively_synced) {
1301d06107adSMax Reitz         BdrvChild *source = op->s->mirror_top_bs->backing;
1302d06107adSMax Reitz 
1303d06107adSMax Reitz         if (QLIST_FIRST(&source->bs->parents) == source &&
1304d06107adSMax Reitz             QLIST_NEXT(source, next_parent) == NULL)
1305d06107adSMax Reitz         {
1306d06107adSMax Reitz             /* Assert that we are back in sync once all active write
1307d06107adSMax Reitz              * operations are settled.
1308d06107adSMax Reitz              * Note that we can only assert this if the mirror node
1309d06107adSMax Reitz              * is the source node's only parent. */
1310d06107adSMax Reitz             assert(!bdrv_get_dirty_count(op->s->dirty_bitmap));
1311d06107adSMax Reitz         }
1312d06107adSMax Reitz     }
1313d06107adSMax Reitz     bitmap_clear(op->s->in_flight_bitmap, start_chunk, end_chunk - start_chunk);
1314d06107adSMax Reitz     QTAILQ_REMOVE(&op->s->ops_in_flight, op, next);
1315d06107adSMax Reitz     qemu_co_queue_restart_all(&op->waiting_requests);
1316d06107adSMax Reitz     g_free(op);
1317d06107adSMax Reitz }
1318d06107adSMax Reitz 
13194ef85a9cSKevin Wolf static int coroutine_fn bdrv_mirror_top_preadv(BlockDriverState *bs,
13204ef85a9cSKevin Wolf     uint64_t offset, uint64_t bytes, QEMUIOVector *qiov, int flags)
13214ef85a9cSKevin Wolf {
13224ef85a9cSKevin Wolf     return bdrv_co_preadv(bs->backing, offset, bytes, qiov, flags);
13234ef85a9cSKevin Wolf }
13244ef85a9cSKevin Wolf 
1325d06107adSMax Reitz static int coroutine_fn bdrv_mirror_top_do_write(BlockDriverState *bs,
1326d06107adSMax Reitz     MirrorMethod method, uint64_t offset, uint64_t bytes, QEMUIOVector *qiov,
1327d06107adSMax Reitz     int flags)
1328d06107adSMax Reitz {
1329d06107adSMax Reitz     MirrorOp *op = NULL;
1330d06107adSMax Reitz     MirrorBDSOpaque *s = bs->opaque;
1331d06107adSMax Reitz     int ret = 0;
1332d06107adSMax Reitz     bool copy_to_target;
1333d06107adSMax Reitz 
1334d06107adSMax Reitz     copy_to_target = s->job->ret >= 0 &&
1335d06107adSMax Reitz                      s->job->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING;
1336d06107adSMax Reitz 
1337d06107adSMax Reitz     if (copy_to_target) {
1338d06107adSMax Reitz         op = active_write_prepare(s->job, offset, bytes);
1339d06107adSMax Reitz     }
1340d06107adSMax Reitz 
1341d06107adSMax Reitz     switch (method) {
1342d06107adSMax Reitz     case MIRROR_METHOD_COPY:
1343d06107adSMax Reitz         ret = bdrv_co_pwritev(bs->backing, offset, bytes, qiov, flags);
1344d06107adSMax Reitz         break;
1345d06107adSMax Reitz 
1346d06107adSMax Reitz     case MIRROR_METHOD_ZERO:
1347d06107adSMax Reitz         ret = bdrv_co_pwrite_zeroes(bs->backing, offset, bytes, flags);
1348d06107adSMax Reitz         break;
1349d06107adSMax Reitz 
1350d06107adSMax Reitz     case MIRROR_METHOD_DISCARD:
13510b9fd3f4SFam Zheng         ret = bdrv_co_pdiscard(bs->backing, offset, bytes);
1352d06107adSMax Reitz         break;
1353d06107adSMax Reitz 
1354d06107adSMax Reitz     default:
1355d06107adSMax Reitz         abort();
1356d06107adSMax Reitz     }
1357d06107adSMax Reitz 
1358d06107adSMax Reitz     if (ret < 0) {
1359d06107adSMax Reitz         goto out;
1360d06107adSMax Reitz     }
1361d06107adSMax Reitz 
1362d06107adSMax Reitz     if (copy_to_target) {
1363d06107adSMax Reitz         do_sync_target_write(s->job, method, offset, bytes, qiov, flags);
1364d06107adSMax Reitz     }
1365d06107adSMax Reitz 
1366d06107adSMax Reitz out:
1367d06107adSMax Reitz     if (copy_to_target) {
1368d06107adSMax Reitz         active_write_settle(op);
1369d06107adSMax Reitz     }
1370d06107adSMax Reitz     return ret;
1371d06107adSMax Reitz }
1372d06107adSMax Reitz 
13734ef85a9cSKevin Wolf static int coroutine_fn bdrv_mirror_top_pwritev(BlockDriverState *bs,
13744ef85a9cSKevin Wolf     uint64_t offset, uint64_t bytes, QEMUIOVector *qiov, int flags)
13754ef85a9cSKevin Wolf {
1376d06107adSMax Reitz     MirrorBDSOpaque *s = bs->opaque;
1377d06107adSMax Reitz     QEMUIOVector bounce_qiov;
1378d06107adSMax Reitz     void *bounce_buf;
1379d06107adSMax Reitz     int ret = 0;
1380d06107adSMax Reitz     bool copy_to_target;
1381d06107adSMax Reitz 
1382d06107adSMax Reitz     copy_to_target = s->job->ret >= 0 &&
1383d06107adSMax Reitz                      s->job->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING;
1384d06107adSMax Reitz 
1385d06107adSMax Reitz     if (copy_to_target) {
1386d06107adSMax Reitz         /* The guest might concurrently modify the data to write; but
1387d06107adSMax Reitz          * the data on source and destination must match, so we have
1388d06107adSMax Reitz          * to use a bounce buffer if we are going to write to the
1389d06107adSMax Reitz          * target now. */
1390d06107adSMax Reitz         bounce_buf = qemu_blockalign(bs, bytes);
1391d06107adSMax Reitz         iov_to_buf_full(qiov->iov, qiov->niov, 0, bounce_buf, bytes);
1392d06107adSMax Reitz 
1393d06107adSMax Reitz         qemu_iovec_init(&bounce_qiov, 1);
1394d06107adSMax Reitz         qemu_iovec_add(&bounce_qiov, bounce_buf, bytes);
1395d06107adSMax Reitz         qiov = &bounce_qiov;
1396d06107adSMax Reitz     }
1397d06107adSMax Reitz 
1398d06107adSMax Reitz     ret = bdrv_mirror_top_do_write(bs, MIRROR_METHOD_COPY, offset, bytes, qiov,
1399d06107adSMax Reitz                                    flags);
1400d06107adSMax Reitz 
1401d06107adSMax Reitz     if (copy_to_target) {
1402d06107adSMax Reitz         qemu_iovec_destroy(&bounce_qiov);
1403d06107adSMax Reitz         qemu_vfree(bounce_buf);
1404d06107adSMax Reitz     }
1405d06107adSMax Reitz 
1406d06107adSMax Reitz     return ret;
14074ef85a9cSKevin Wolf }
14084ef85a9cSKevin Wolf 
14094ef85a9cSKevin Wolf static int coroutine_fn bdrv_mirror_top_flush(BlockDriverState *bs)
14104ef85a9cSKevin Wolf {
1411ce960aa9SVladimir Sementsov-Ogievskiy     if (bs->backing == NULL) {
1412ce960aa9SVladimir Sementsov-Ogievskiy         /* we can be here after failed bdrv_append in mirror_start_job */
1413ce960aa9SVladimir Sementsov-Ogievskiy         return 0;
1414ce960aa9SVladimir Sementsov-Ogievskiy     }
14154ef85a9cSKevin Wolf     return bdrv_co_flush(bs->backing->bs);
14164ef85a9cSKevin Wolf }
14174ef85a9cSKevin Wolf 
14184ef85a9cSKevin Wolf static int coroutine_fn bdrv_mirror_top_pwrite_zeroes(BlockDriverState *bs,
1419f5a5ca79SManos Pitsidianakis     int64_t offset, int bytes, BdrvRequestFlags flags)
14204ef85a9cSKevin Wolf {
1421d06107adSMax Reitz     return bdrv_mirror_top_do_write(bs, MIRROR_METHOD_ZERO, offset, bytes, NULL,
1422d06107adSMax Reitz                                     flags);
14234ef85a9cSKevin Wolf }
14244ef85a9cSKevin Wolf 
14254ef85a9cSKevin Wolf static int coroutine_fn bdrv_mirror_top_pdiscard(BlockDriverState *bs,
1426f5a5ca79SManos Pitsidianakis     int64_t offset, int bytes)
14274ef85a9cSKevin Wolf {
1428d06107adSMax Reitz     return bdrv_mirror_top_do_write(bs, MIRROR_METHOD_DISCARD, offset, bytes,
1429d06107adSMax Reitz                                     NULL, 0);
14304ef85a9cSKevin Wolf }
14314ef85a9cSKevin Wolf 
1432fd4a6493SKevin Wolf static void bdrv_mirror_top_refresh_filename(BlockDriverState *bs, QDict *opts)
1433fd4a6493SKevin Wolf {
143418775ff3SVladimir Sementsov-Ogievskiy     if (bs->backing == NULL) {
143518775ff3SVladimir Sementsov-Ogievskiy         /* we can be here after failed bdrv_attach_child in
143618775ff3SVladimir Sementsov-Ogievskiy          * bdrv_set_backing_hd */
143718775ff3SVladimir Sementsov-Ogievskiy         return;
143818775ff3SVladimir Sementsov-Ogievskiy     }
1439fd4a6493SKevin Wolf     bdrv_refresh_filename(bs->backing->bs);
1440fd4a6493SKevin Wolf     pstrcpy(bs->exact_filename, sizeof(bs->exact_filename),
1441fd4a6493SKevin Wolf             bs->backing->bs->filename);
1442fd4a6493SKevin Wolf }
1443fd4a6493SKevin Wolf 
14444ef85a9cSKevin Wolf static void bdrv_mirror_top_child_perm(BlockDriverState *bs, BdrvChild *c,
14454ef85a9cSKevin Wolf                                        const BdrvChildRole *role,
1446e0995dc3SKevin Wolf                                        BlockReopenQueue *reopen_queue,
14474ef85a9cSKevin Wolf                                        uint64_t perm, uint64_t shared,
14484ef85a9cSKevin Wolf                                        uint64_t *nperm, uint64_t *nshared)
14494ef85a9cSKevin Wolf {
14504ef85a9cSKevin Wolf     /* Must be able to forward guest writes to the real image */
14514ef85a9cSKevin Wolf     *nperm = 0;
14524ef85a9cSKevin Wolf     if (perm & BLK_PERM_WRITE) {
14534ef85a9cSKevin Wolf         *nperm |= BLK_PERM_WRITE;
14544ef85a9cSKevin Wolf     }
14554ef85a9cSKevin Wolf 
14564ef85a9cSKevin Wolf     *nshared = BLK_PERM_ALL;
14574ef85a9cSKevin Wolf }
14584ef85a9cSKevin Wolf 
14594ef85a9cSKevin Wolf /* Dummy node that provides consistent read to its users without requiring it
14604ef85a9cSKevin Wolf  * from its backing file and that allows writes on the backing file chain. */
14614ef85a9cSKevin Wolf static BlockDriver bdrv_mirror_top = {
14624ef85a9cSKevin Wolf     .format_name                = "mirror_top",
14634ef85a9cSKevin Wolf     .bdrv_co_preadv             = bdrv_mirror_top_preadv,
14644ef85a9cSKevin Wolf     .bdrv_co_pwritev            = bdrv_mirror_top_pwritev,
14654ef85a9cSKevin Wolf     .bdrv_co_pwrite_zeroes      = bdrv_mirror_top_pwrite_zeroes,
14664ef85a9cSKevin Wolf     .bdrv_co_pdiscard           = bdrv_mirror_top_pdiscard,
14674ef85a9cSKevin Wolf     .bdrv_co_flush              = bdrv_mirror_top_flush,
14683e4d0e72SEric Blake     .bdrv_co_block_status       = bdrv_co_block_status_from_backing,
1469fd4a6493SKevin Wolf     .bdrv_refresh_filename      = bdrv_mirror_top_refresh_filename,
14704ef85a9cSKevin Wolf     .bdrv_child_perm            = bdrv_mirror_top_child_perm,
14714ef85a9cSKevin Wolf };
14724ef85a9cSKevin Wolf 
147371aa9867SAlberto Garcia static void mirror_start_job(const char *job_id, BlockDriverState *bs,
147447970dfbSJohn Snow                              int creation_flags, BlockDriverState *target,
147547970dfbSJohn Snow                              const char *replaces, int64_t speed,
147647970dfbSJohn Snow                              uint32_t granularity, int64_t buf_size,
1477274fcceeSMax Reitz                              BlockMirrorBackingMode backing_mode,
147803544a6eSFam Zheng                              BlockdevOnError on_source_error,
1479b952b558SPaolo Bonzini                              BlockdevOnError on_target_error,
14800fc9f8eaSFam Zheng                              bool unmap,
1481097310b5SMarkus Armbruster                              BlockCompletionFunc *cb,
148251ccfa2dSFam Zheng                              void *opaque,
148303544a6eSFam Zheng                              const BlockJobDriver *driver,
1484b49f7eadSWen Congyang                              bool is_none_mode, BlockDriverState *base,
148551ccfa2dSFam Zheng                              bool auto_complete, const char *filter_node_name,
1486481debaaSMax Reitz                              bool is_mirror, MirrorCopyMode copy_mode,
148751ccfa2dSFam Zheng                              Error **errp)
1488893f7ebaSPaolo Bonzini {
1489893f7ebaSPaolo Bonzini     MirrorBlockJob *s;
1490429076e8SMax Reitz     MirrorBDSOpaque *bs_opaque;
14914ef85a9cSKevin Wolf     BlockDriverState *mirror_top_bs;
14924ef85a9cSKevin Wolf     bool target_graph_mod;
14934ef85a9cSKevin Wolf     bool target_is_backing;
1494b2c2832cSKevin Wolf     Error *local_err = NULL;
1495d7086422SKevin Wolf     int ret;
1496893f7ebaSPaolo Bonzini 
1497eee13dfeSPaolo Bonzini     if (granularity == 0) {
1498341ebc2fSJohn Snow         granularity = bdrv_get_default_bitmap_granularity(target);
1499eee13dfeSPaolo Bonzini     }
1500eee13dfeSPaolo Bonzini 
150131826642SEric Blake     assert(is_power_of_2(granularity));
1502eee13dfeSPaolo Bonzini 
150348ac0a4dSWen Congyang     if (buf_size < 0) {
150448ac0a4dSWen Congyang         error_setg(errp, "Invalid parameter 'buf-size'");
150548ac0a4dSWen Congyang         return;
150648ac0a4dSWen Congyang     }
150748ac0a4dSWen Congyang 
150848ac0a4dSWen Congyang     if (buf_size == 0) {
150948ac0a4dSWen Congyang         buf_size = DEFAULT_MIRROR_BUF_SIZE;
151048ac0a4dSWen Congyang     }
15115bc361b8SFam Zheng 
151286fae10cSKevin Wolf     if (bs == target) {
151386fae10cSKevin Wolf         error_setg(errp, "Can't mirror node into itself");
151486fae10cSKevin Wolf         return;
151586fae10cSKevin Wolf     }
151686fae10cSKevin Wolf 
15174ef85a9cSKevin Wolf     /* In the case of active commit, add dummy driver to provide consistent
15184ef85a9cSKevin Wolf      * reads on the top, while disabling it in the intermediate nodes, and make
15194ef85a9cSKevin Wolf      * the backing chain writable. */
15206cdbceb1SKevin Wolf     mirror_top_bs = bdrv_new_open_driver(&bdrv_mirror_top, filter_node_name,
15216cdbceb1SKevin Wolf                                          BDRV_O_RDWR, errp);
15224ef85a9cSKevin Wolf     if (mirror_top_bs == NULL) {
1523893f7ebaSPaolo Bonzini         return;
1524893f7ebaSPaolo Bonzini     }
1525d3c8c674SKevin Wolf     if (!filter_node_name) {
1526d3c8c674SKevin Wolf         mirror_top_bs->implicit = true;
1527d3c8c674SKevin Wolf     }
15284ef85a9cSKevin Wolf     mirror_top_bs->total_sectors = bs->total_sectors;
1529228345bfSMax Reitz     mirror_top_bs->supported_write_flags = BDRV_REQ_WRITE_UNCHANGED;
1530228345bfSMax Reitz     mirror_top_bs->supported_zero_flags = BDRV_REQ_WRITE_UNCHANGED;
1531429076e8SMax Reitz     bs_opaque = g_new0(MirrorBDSOpaque, 1);
1532429076e8SMax Reitz     mirror_top_bs->opaque = bs_opaque;
153319dd29e8SFam Zheng     bdrv_set_aio_context(mirror_top_bs, bdrv_get_aio_context(bs));
1534893f7ebaSPaolo Bonzini 
15354ef85a9cSKevin Wolf     /* bdrv_append takes ownership of the mirror_top_bs reference, need to keep
15367a25fcd0SMax Reitz      * it alive until block_job_create() succeeds even if bs has no parent. */
15374ef85a9cSKevin Wolf     bdrv_ref(mirror_top_bs);
15384ef85a9cSKevin Wolf     bdrv_drained_begin(bs);
1539b2c2832cSKevin Wolf     bdrv_append(mirror_top_bs, bs, &local_err);
15404ef85a9cSKevin Wolf     bdrv_drained_end(bs);
15414ef85a9cSKevin Wolf 
1542b2c2832cSKevin Wolf     if (local_err) {
1543b2c2832cSKevin Wolf         bdrv_unref(mirror_top_bs);
1544b2c2832cSKevin Wolf         error_propagate(errp, local_err);
1545b2c2832cSKevin Wolf         return;
1546b2c2832cSKevin Wolf     }
1547b2c2832cSKevin Wolf 
15484ef85a9cSKevin Wolf     /* Make sure that the source is not resized while the job is running */
154975859b94SJohn Snow     s = block_job_create(job_id, driver, NULL, mirror_top_bs,
15504ef85a9cSKevin Wolf                          BLK_PERM_CONSISTENT_READ,
15514ef85a9cSKevin Wolf                          BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE_UNCHANGED |
15524ef85a9cSKevin Wolf                          BLK_PERM_WRITE | BLK_PERM_GRAPH_MOD, speed,
15534ef85a9cSKevin Wolf                          creation_flags, cb, opaque, errp);
15544ef85a9cSKevin Wolf     if (!s) {
15554ef85a9cSKevin Wolf         goto fail;
15564ef85a9cSKevin Wolf     }
1557429076e8SMax Reitz     bs_opaque->job = s;
1558429076e8SMax Reitz 
15597a25fcd0SMax Reitz     /* The block job now has a reference to this node */
15607a25fcd0SMax Reitz     bdrv_unref(mirror_top_bs);
15617a25fcd0SMax Reitz 
15624ef85a9cSKevin Wolf     s->mirror_top_bs = mirror_top_bs;
15634ef85a9cSKevin Wolf 
15644ef85a9cSKevin Wolf     /* No resize for the target either; while the mirror is still running, a
15654ef85a9cSKevin Wolf      * consistent read isn't necessarily possible. We could possibly allow
15664ef85a9cSKevin Wolf      * writes and graph modifications, though it would likely defeat the
15674ef85a9cSKevin Wolf      * purpose of a mirror, so leave them blocked for now.
15684ef85a9cSKevin Wolf      *
15694ef85a9cSKevin Wolf      * In the case of active commit, things look a bit different, though,
15704ef85a9cSKevin Wolf      * because the target is an already populated backing file in active use.
15714ef85a9cSKevin Wolf      * We can allow anything except resize there.*/
15724ef85a9cSKevin Wolf     target_is_backing = bdrv_chain_contains(bs, target);
15734ef85a9cSKevin Wolf     target_graph_mod = (backing_mode != MIRROR_LEAVE_BACKING_CHAIN);
15744ef85a9cSKevin Wolf     s->target = blk_new(BLK_PERM_WRITE | BLK_PERM_RESIZE |
15754ef85a9cSKevin Wolf                         (target_graph_mod ? BLK_PERM_GRAPH_MOD : 0),
15764ef85a9cSKevin Wolf                         BLK_PERM_WRITE_UNCHANGED |
15774ef85a9cSKevin Wolf                         (target_is_backing ? BLK_PERM_CONSISTENT_READ |
15784ef85a9cSKevin Wolf                                              BLK_PERM_WRITE |
15794ef85a9cSKevin Wolf                                              BLK_PERM_GRAPH_MOD : 0));
1580d7086422SKevin Wolf     ret = blk_insert_bs(s->target, target, errp);
1581d7086422SKevin Wolf     if (ret < 0) {
15824ef85a9cSKevin Wolf         goto fail;
1583d7086422SKevin Wolf     }
1584045a2f82SFam Zheng     if (is_mirror) {
1585045a2f82SFam Zheng         /* XXX: Mirror target could be a NBD server of target QEMU in the case
1586045a2f82SFam Zheng          * of non-shared block migration. To allow migration completion, we
1587045a2f82SFam Zheng          * have to allow "inactivate" of the target BB.  When that happens, we
1588045a2f82SFam Zheng          * know the job is drained, and the vcpus are stopped, so no write
1589045a2f82SFam Zheng          * operation will be performed. Block layer already has assertions to
1590045a2f82SFam Zheng          * ensure that. */
1591045a2f82SFam Zheng         blk_set_force_allow_inactivate(s->target);
1592045a2f82SFam Zheng     }
1593e253f4b8SKevin Wolf 
159409158f00SBenoît Canet     s->replaces = g_strdup(replaces);
1595b952b558SPaolo Bonzini     s->on_source_error = on_source_error;
1596b952b558SPaolo Bonzini     s->on_target_error = on_target_error;
159703544a6eSFam Zheng     s->is_none_mode = is_none_mode;
1598274fcceeSMax Reitz     s->backing_mode = backing_mode;
1599481debaaSMax Reitz     s->copy_mode = copy_mode;
16005bc361b8SFam Zheng     s->base = base;
1601eee13dfeSPaolo Bonzini     s->granularity = granularity;
160248ac0a4dSWen Congyang     s->buf_size = ROUND_UP(buf_size, granularity);
16030fc9f8eaSFam Zheng     s->unmap = unmap;
1604b49f7eadSWen Congyang     if (auto_complete) {
1605b49f7eadSWen Congyang         s->should_complete = true;
1606b49f7eadSWen Congyang     }
1607b812f671SPaolo Bonzini 
16080db6e54aSFam Zheng     s->dirty_bitmap = bdrv_create_dirty_bitmap(bs, granularity, NULL, errp);
1609b8afb520SFam Zheng     if (!s->dirty_bitmap) {
161088f9d1b3SKevin Wolf         goto fail;
1611b8afb520SFam Zheng     }
161210f3cd15SAlberto Garcia 
16134ef85a9cSKevin Wolf     /* Required permissions are already taken with blk_new() */
161476d554e2SKevin Wolf     block_job_add_bdrv(&s->common, "target", target, 0, BLK_PERM_ALL,
161576d554e2SKevin Wolf                        &error_abort);
161676d554e2SKevin Wolf 
1617f3ede4b0SAlberto Garcia     /* In commit_active_start() all intermediate nodes disappear, so
1618f3ede4b0SAlberto Garcia      * any jobs in them must be blocked */
16194ef85a9cSKevin Wolf     if (target_is_backing) {
1620f3ede4b0SAlberto Garcia         BlockDriverState *iter;
1621f3ede4b0SAlberto Garcia         for (iter = backing_bs(bs); iter != target; iter = backing_bs(iter)) {
16224ef85a9cSKevin Wolf             /* XXX BLK_PERM_WRITE needs to be allowed so we don't block
16234ef85a9cSKevin Wolf              * ourselves at s->base (if writes are blocked for a node, they are
16244ef85a9cSKevin Wolf              * also blocked for its backing file). The other options would be a
16254ef85a9cSKevin Wolf              * second filter driver above s->base (== target). */
16264ef85a9cSKevin Wolf             ret = block_job_add_bdrv(&s->common, "intermediate node", iter, 0,
16274ef85a9cSKevin Wolf                                      BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE,
16284ef85a9cSKevin Wolf                                      errp);
16294ef85a9cSKevin Wolf             if (ret < 0) {
16304ef85a9cSKevin Wolf                 goto fail;
16314ef85a9cSKevin Wolf             }
1632f3ede4b0SAlberto Garcia         }
1633f3ede4b0SAlberto Garcia     }
163410f3cd15SAlberto Garcia 
163512aa4082SMax Reitz     QTAILQ_INIT(&s->ops_in_flight);
163612aa4082SMax Reitz 
16375ccac6f1SJohn Snow     trace_mirror_start(bs, s, opaque);
1638da01ff7fSKevin Wolf     job_start(&s->common.job);
16394ef85a9cSKevin Wolf     return;
16404ef85a9cSKevin Wolf 
16414ef85a9cSKevin Wolf fail:
16424ef85a9cSKevin Wolf     if (s) {
16437a25fcd0SMax Reitz         /* Make sure this BDS does not go away until we have completed the graph
16447a25fcd0SMax Reitz          * changes below */
16457a25fcd0SMax Reitz         bdrv_ref(mirror_top_bs);
16467a25fcd0SMax Reitz 
16474ef85a9cSKevin Wolf         g_free(s->replaces);
16484ef85a9cSKevin Wolf         blk_unref(s->target);
1649429076e8SMax Reitz         bs_opaque->job = NULL;
16504ad35181SKevin Wolf         job_early_fail(&s->common.job);
16514ef85a9cSKevin Wolf     }
16524ef85a9cSKevin Wolf 
1653c1cef672SFam Zheng     bdrv_child_try_set_perm(mirror_top_bs->backing, 0, BLK_PERM_ALL,
1654c1cef672SFam Zheng                             &error_abort);
16555fe31c25SKevin Wolf     bdrv_replace_node(mirror_top_bs, backing_bs(mirror_top_bs), &error_abort);
16567a25fcd0SMax Reitz 
16577a25fcd0SMax Reitz     bdrv_unref(mirror_top_bs);
1658893f7ebaSPaolo Bonzini }
165903544a6eSFam Zheng 
166071aa9867SAlberto Garcia void mirror_start(const char *job_id, BlockDriverState *bs,
166171aa9867SAlberto Garcia                   BlockDriverState *target, const char *replaces,
1662a1999b33SJohn Snow                   int creation_flags, int64_t speed,
1663a1999b33SJohn Snow                   uint32_t granularity, int64_t buf_size,
1664274fcceeSMax Reitz                   MirrorSyncMode mode, BlockMirrorBackingMode backing_mode,
1665274fcceeSMax Reitz                   BlockdevOnError on_source_error,
166603544a6eSFam Zheng                   BlockdevOnError on_target_error,
1667481debaaSMax Reitz                   bool unmap, const char *filter_node_name,
1668481debaaSMax Reitz                   MirrorCopyMode copy_mode, Error **errp)
166903544a6eSFam Zheng {
167003544a6eSFam Zheng     bool is_none_mode;
167103544a6eSFam Zheng     BlockDriverState *base;
167203544a6eSFam Zheng 
16734b80ab2bSJohn Snow     if (mode == MIRROR_SYNC_MODE_INCREMENTAL) {
16744b80ab2bSJohn Snow         error_setg(errp, "Sync mode 'incremental' not supported");
1675d58d8453SJohn Snow         return;
1676d58d8453SJohn Snow     }
167703544a6eSFam Zheng     is_none_mode = mode == MIRROR_SYNC_MODE_NONE;
1678760e0063SKevin Wolf     base = mode == MIRROR_SYNC_MODE_TOP ? backing_bs(bs) : NULL;
1679a1999b33SJohn Snow     mirror_start_job(job_id, bs, creation_flags, target, replaces,
1680274fcceeSMax Reitz                      speed, granularity, buf_size, backing_mode,
168151ccfa2dSFam Zheng                      on_source_error, on_target_error, unmap, NULL, NULL,
16826cdbceb1SKevin Wolf                      &mirror_job_driver, is_none_mode, base, false,
1683481debaaSMax Reitz                      filter_node_name, true, copy_mode, errp);
168403544a6eSFam Zheng }
168503544a6eSFam Zheng 
1686fd62c609SAlberto Garcia void commit_active_start(const char *job_id, BlockDriverState *bs,
168747970dfbSJohn Snow                          BlockDriverState *base, int creation_flags,
168847970dfbSJohn Snow                          int64_t speed, BlockdevOnError on_error,
16890db832f4SKevin Wolf                          const char *filter_node_name,
169078bbd910SFam Zheng                          BlockCompletionFunc *cb, void *opaque,
169178bbd910SFam Zheng                          bool auto_complete, Error **errp)
169203544a6eSFam Zheng {
1693*1ba79388SAlberto Garcia     bool base_read_only;
1694cc67f4d1SJeff Cody     Error *local_err = NULL;
16954da83585SJeff Cody 
1696*1ba79388SAlberto Garcia     base_read_only = bdrv_is_read_only(base);
16974da83585SJeff Cody 
1698*1ba79388SAlberto Garcia     if (base_read_only) {
1699*1ba79388SAlberto Garcia         if (bdrv_reopen_set_read_only(base, false, errp) < 0) {
170020a63d2cSFam Zheng             return;
170120a63d2cSFam Zheng         }
1702*1ba79388SAlberto Garcia     }
17034da83585SJeff Cody 
170447970dfbSJohn Snow     mirror_start_job(job_id, bs, creation_flags, base, NULL, speed, 0, 0,
170571aa9867SAlberto Garcia                      MIRROR_LEAVE_BACKING_CHAIN,
170651ccfa2dSFam Zheng                      on_error, on_error, true, cb, opaque,
17076cdbceb1SKevin Wolf                      &commit_active_job_driver, false, base, auto_complete,
1708481debaaSMax Reitz                      filter_node_name, false, MIRROR_COPY_MODE_BACKGROUND,
1709481debaaSMax Reitz                      &local_err);
17100fb6395cSMarkus Armbruster     if (local_err) {
1711cc67f4d1SJeff Cody         error_propagate(errp, local_err);
17124da83585SJeff Cody         goto error_restore_flags;
17134da83585SJeff Cody     }
17144da83585SJeff Cody 
17154da83585SJeff Cody     return;
17164da83585SJeff Cody 
17174da83585SJeff Cody error_restore_flags:
17184da83585SJeff Cody     /* ignore error and errp for bdrv_reopen, because we want to propagate
17194da83585SJeff Cody      * the original error */
1720*1ba79388SAlberto Garcia     if (base_read_only) {
1721*1ba79388SAlberto Garcia         bdrv_reopen_set_read_only(base, true, NULL);
1722*1ba79388SAlberto Garcia     }
17234da83585SJeff Cody     return;
172403544a6eSFam Zheng }
1725