xref: /qemu/block/mirror.c (revision 018e5987b57589e6e9089c2d2ef31db4e7519fd5)
1893f7ebaSPaolo Bonzini /*
2893f7ebaSPaolo Bonzini  * Image mirroring
3893f7ebaSPaolo Bonzini  *
4893f7ebaSPaolo Bonzini  * Copyright Red Hat, Inc. 2012
5893f7ebaSPaolo Bonzini  *
6893f7ebaSPaolo Bonzini  * Authors:
7893f7ebaSPaolo Bonzini  *  Paolo Bonzini  <pbonzini@redhat.com>
8893f7ebaSPaolo Bonzini  *
9893f7ebaSPaolo Bonzini  * This work is licensed under the terms of the GNU LGPL, version 2 or later.
10893f7ebaSPaolo Bonzini  * See the COPYING.LIB file in the top-level directory.
11893f7ebaSPaolo Bonzini  *
12893f7ebaSPaolo Bonzini  */
13893f7ebaSPaolo Bonzini 
1480c71a24SPeter Maydell #include "qemu/osdep.h"
15fd4a6493SKevin Wolf #include "qemu/cutils.h"
1612aa4082SMax Reitz #include "qemu/coroutine.h"
171181e19aSMax Reitz #include "qemu/range.h"
18893f7ebaSPaolo Bonzini #include "trace.h"
19c87621eaSJohn Snow #include "block/blockjob_int.h"
20737e150eSPaolo Bonzini #include "block/block_int.h"
21e2c1c34fSMarkus Armbruster #include "block/dirty-bitmap.h"
22373340b2SMax Reitz #include "sysemu/block-backend.h"
23da34e65cSMarkus Armbruster #include "qapi/error.h"
24893f7ebaSPaolo Bonzini #include "qemu/ratelimit.h"
25b812f671SPaolo Bonzini #include "qemu/bitmap.h"
265df022cfSPeter Maydell #include "qemu/memalign.h"
27893f7ebaSPaolo Bonzini 
28402a4741SPaolo Bonzini #define MAX_IN_FLIGHT 16
29b436982fSEric Blake #define MAX_IO_BYTES (1 << 20) /* 1 Mb */
30b436982fSEric Blake #define DEFAULT_MIRROR_BUF_SIZE (MAX_IN_FLIGHT * MAX_IO_BYTES)
31402a4741SPaolo Bonzini 
32402a4741SPaolo Bonzini /* The mirroring buffer is a list of granularity-sized chunks.
33402a4741SPaolo Bonzini  * Free chunks are organized in a list.
34402a4741SPaolo Bonzini  */
35402a4741SPaolo Bonzini typedef struct MirrorBuffer {
36402a4741SPaolo Bonzini     QSIMPLEQ_ENTRY(MirrorBuffer) next;
37402a4741SPaolo Bonzini } MirrorBuffer;
38893f7ebaSPaolo Bonzini 
3912aa4082SMax Reitz typedef struct MirrorOp MirrorOp;
4012aa4082SMax Reitz 
41893f7ebaSPaolo Bonzini typedef struct MirrorBlockJob {
42893f7ebaSPaolo Bonzini     BlockJob common;
43e253f4b8SKevin Wolf     BlockBackend *target;
444ef85a9cSKevin Wolf     BlockDriverState *mirror_top_bs;
455bc361b8SFam Zheng     BlockDriverState *base;
463f072a7fSMax Reitz     BlockDriverState *base_overlay;
474ef85a9cSKevin Wolf 
4809158f00SBenoît Canet     /* The name of the graph node to replace */
4909158f00SBenoît Canet     char *replaces;
5009158f00SBenoît Canet     /* The BDS to replace */
5109158f00SBenoît Canet     BlockDriverState *to_replace;
5209158f00SBenoît Canet     /* Used to block operations on the drive-mirror-replace target */
5309158f00SBenoît Canet     Error *replace_blocker;
5403544a6eSFam Zheng     bool is_none_mode;
55274fcceeSMax Reitz     BlockMirrorBackingMode backing_mode;
56cdf3bc93SMax Reitz     /* Whether the target image requires explicit zero-initialization */
57cdf3bc93SMax Reitz     bool zero_target;
58d06107adSMax Reitz     MirrorCopyMode copy_mode;
59b952b558SPaolo Bonzini     BlockdevOnError on_source_error, on_target_error;
60d06107adSMax Reitz     /* Set when the target is synced (dirty bitmap is clean, nothing
61d06107adSMax Reitz      * in flight) and the job is running in active mode */
62d06107adSMax Reitz     bool actively_synced;
63d63ffd87SPaolo Bonzini     bool should_complete;
64eee13dfeSPaolo Bonzini     int64_t granularity;
65b812f671SPaolo Bonzini     size_t buf_size;
66b21c7652SMax Reitz     int64_t bdev_length;
67b812f671SPaolo Bonzini     unsigned long *cow_bitmap;
68e4654d2dSFam Zheng     BdrvDirtyBitmap *dirty_bitmap;
69dc162c8eSFam Zheng     BdrvDirtyBitmapIter *dbi;
70893f7ebaSPaolo Bonzini     uint8_t *buf;
71402a4741SPaolo Bonzini     QSIMPLEQ_HEAD(, MirrorBuffer) buf_free;
72402a4741SPaolo Bonzini     int buf_free_count;
73bd48bde8SPaolo Bonzini 
7449efb1f5SDenis V. Lunev     uint64_t last_pause_ns;
75402a4741SPaolo Bonzini     unsigned long *in_flight_bitmap;
761b8f7776SDenis V. Lunev     unsigned in_flight;
77b436982fSEric Blake     int64_t bytes_in_flight;
78b58deb34SPaolo Bonzini     QTAILQ_HEAD(, MirrorOp) ops_in_flight;
79bd48bde8SPaolo Bonzini     int ret;
800fc9f8eaSFam Zheng     bool unmap;
81b436982fSEric Blake     int target_cluster_size;
82e5b43573SFam Zheng     int max_iov;
8390ab48ebSAnton Nefedov     bool initial_zeroing_ongoing;
84d06107adSMax Reitz     int in_active_write_counter;
85d69a879bSHanna Reitz     int64_t active_write_bytes_in_flight;
86737efc1eSJohn Snow     bool prepared;
875e771752SSergio Lopez     bool in_drain;
88893f7ebaSPaolo Bonzini } MirrorBlockJob;
89893f7ebaSPaolo Bonzini 
90429076e8SMax Reitz typedef struct MirrorBDSOpaque {
91429076e8SMax Reitz     MirrorBlockJob *job;
92f94dc3b4SMax Reitz     bool stop;
9353431b90SMax Reitz     bool is_commit;
94429076e8SMax Reitz } MirrorBDSOpaque;
95429076e8SMax Reitz 
9612aa4082SMax Reitz struct MirrorOp {
97bd48bde8SPaolo Bonzini     MirrorBlockJob *s;
98bd48bde8SPaolo Bonzini     QEMUIOVector qiov;
99b436982fSEric Blake     int64_t offset;
100b436982fSEric Blake     uint64_t bytes;
1012e1990b2SMax Reitz 
1022e1990b2SMax Reitz     /* The pointee is set by mirror_co_read(), mirror_co_zero(), and
1032e1990b2SMax Reitz      * mirror_co_discard() before yielding for the first time */
1042e1990b2SMax Reitz     int64_t *bytes_handled;
10512aa4082SMax Reitz 
1061181e19aSMax Reitz     bool is_pseudo_op;
107d06107adSMax Reitz     bool is_active_write;
108ce8cabbdSKevin Wolf     bool is_in_flight;
10912aa4082SMax Reitz     CoQueue waiting_requests;
110eed325b9SKevin Wolf     Coroutine *co;
111d44dae1aSVladimir Sementsov-Ogievskiy     MirrorOp *waiting_for_op;
11212aa4082SMax Reitz 
11312aa4082SMax Reitz     QTAILQ_ENTRY(MirrorOp) next;
11412aa4082SMax Reitz };
115bd48bde8SPaolo Bonzini 
1164295c5fcSMax Reitz typedef enum MirrorMethod {
1174295c5fcSMax Reitz     MIRROR_METHOD_COPY,
1184295c5fcSMax Reitz     MIRROR_METHOD_ZERO,
1194295c5fcSMax Reitz     MIRROR_METHOD_DISCARD,
1204295c5fcSMax Reitz } MirrorMethod;
1214295c5fcSMax Reitz 
122b952b558SPaolo Bonzini static BlockErrorAction mirror_error_action(MirrorBlockJob *s, bool read,
123b952b558SPaolo Bonzini                                             int error)
124b952b558SPaolo Bonzini {
125d06107adSMax Reitz     s->actively_synced = false;
126b952b558SPaolo Bonzini     if (read) {
12781e254dcSKevin Wolf         return block_job_error_action(&s->common, s->on_source_error,
12881e254dcSKevin Wolf                                       true, error);
129b952b558SPaolo Bonzini     } else {
13081e254dcSKevin Wolf         return block_job_error_action(&s->common, s->on_target_error,
13181e254dcSKevin Wolf                                       false, error);
132b952b558SPaolo Bonzini     }
133b952b558SPaolo Bonzini }
134b952b558SPaolo Bonzini 
1351181e19aSMax Reitz static void coroutine_fn mirror_wait_on_conflicts(MirrorOp *self,
1361181e19aSMax Reitz                                                   MirrorBlockJob *s,
1371181e19aSMax Reitz                                                   uint64_t offset,
1381181e19aSMax Reitz                                                   uint64_t bytes)
1391181e19aSMax Reitz {
1401181e19aSMax Reitz     uint64_t self_start_chunk = offset / s->granularity;
1411181e19aSMax Reitz     uint64_t self_end_chunk = DIV_ROUND_UP(offset + bytes, s->granularity);
1421181e19aSMax Reitz     uint64_t self_nb_chunks = self_end_chunk - self_start_chunk;
1431181e19aSMax Reitz 
1441181e19aSMax Reitz     while (find_next_bit(s->in_flight_bitmap, self_end_chunk,
1451181e19aSMax Reitz                          self_start_chunk) < self_end_chunk &&
1461181e19aSMax Reitz            s->ret >= 0)
1471181e19aSMax Reitz     {
1481181e19aSMax Reitz         MirrorOp *op;
1491181e19aSMax Reitz 
1501181e19aSMax Reitz         QTAILQ_FOREACH(op, &s->ops_in_flight, next) {
1511181e19aSMax Reitz             uint64_t op_start_chunk = op->offset / s->granularity;
1521181e19aSMax Reitz             uint64_t op_nb_chunks = DIV_ROUND_UP(op->offset + op->bytes,
1531181e19aSMax Reitz                                                  s->granularity) -
1541181e19aSMax Reitz                                     op_start_chunk;
1551181e19aSMax Reitz 
1561181e19aSMax Reitz             if (op == self) {
1571181e19aSMax Reitz                 continue;
1581181e19aSMax Reitz             }
1591181e19aSMax Reitz 
1601181e19aSMax Reitz             if (ranges_overlap(self_start_chunk, self_nb_chunks,
1611181e19aSMax Reitz                                op_start_chunk, op_nb_chunks))
1621181e19aSMax Reitz             {
16366fed30cSStefano Garzarella                 if (self) {
164d44dae1aSVladimir Sementsov-Ogievskiy                     /*
16566fed30cSStefano Garzarella                      * If the operation is already (indirectly) waiting for us,
16666fed30cSStefano Garzarella                      * or will wait for us as soon as it wakes up, then just go
16766fed30cSStefano Garzarella                      * on (instead of producing a deadlock in the former case).
168d44dae1aSVladimir Sementsov-Ogievskiy                      */
169d44dae1aSVladimir Sementsov-Ogievskiy                     if (op->waiting_for_op) {
170d44dae1aSVladimir Sementsov-Ogievskiy                         continue;
171d44dae1aSVladimir Sementsov-Ogievskiy                     }
172d44dae1aSVladimir Sementsov-Ogievskiy 
173d44dae1aSVladimir Sementsov-Ogievskiy                     self->waiting_for_op = op;
17466fed30cSStefano Garzarella                 }
17566fed30cSStefano Garzarella 
1761181e19aSMax Reitz                 qemu_co_queue_wait(&op->waiting_requests, NULL);
17766fed30cSStefano Garzarella 
17866fed30cSStefano Garzarella                 if (self) {
179d44dae1aSVladimir Sementsov-Ogievskiy                     self->waiting_for_op = NULL;
18066fed30cSStefano Garzarella                 }
18166fed30cSStefano Garzarella 
1821181e19aSMax Reitz                 break;
1831181e19aSMax Reitz             }
1841181e19aSMax Reitz         }
1851181e19aSMax Reitz     }
1861181e19aSMax Reitz }
1871181e19aSMax Reitz 
1882e1990b2SMax Reitz static void coroutine_fn mirror_iteration_done(MirrorOp *op, int ret)
189bd48bde8SPaolo Bonzini {
190bd48bde8SPaolo Bonzini     MirrorBlockJob *s = op->s;
191402a4741SPaolo Bonzini     struct iovec *iov;
192bd48bde8SPaolo Bonzini     int64_t chunk_num;
193b436982fSEric Blake     int i, nb_chunks;
194bd48bde8SPaolo Bonzini 
195b436982fSEric Blake     trace_mirror_iteration_done(s, op->offset, op->bytes, ret);
196bd48bde8SPaolo Bonzini 
197bd48bde8SPaolo Bonzini     s->in_flight--;
198b436982fSEric Blake     s->bytes_in_flight -= op->bytes;
199402a4741SPaolo Bonzini     iov = op->qiov.iov;
200402a4741SPaolo Bonzini     for (i = 0; i < op->qiov.niov; i++) {
201402a4741SPaolo Bonzini         MirrorBuffer *buf = (MirrorBuffer *) iov[i].iov_base;
202402a4741SPaolo Bonzini         QSIMPLEQ_INSERT_TAIL(&s->buf_free, buf, next);
203402a4741SPaolo Bonzini         s->buf_free_count++;
204402a4741SPaolo Bonzini     }
205402a4741SPaolo Bonzini 
206b436982fSEric Blake     chunk_num = op->offset / s->granularity;
207b436982fSEric Blake     nb_chunks = DIV_ROUND_UP(op->bytes, s->granularity);
20812aa4082SMax Reitz 
209402a4741SPaolo Bonzini     bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks);
21012aa4082SMax Reitz     QTAILQ_REMOVE(&s->ops_in_flight, op, next);
211b21c7652SMax Reitz     if (ret >= 0) {
212b21c7652SMax Reitz         if (s->cow_bitmap) {
213bd48bde8SPaolo Bonzini             bitmap_set(s->cow_bitmap, chunk_num, nb_chunks);
214bd48bde8SPaolo Bonzini         }
21590ab48ebSAnton Nefedov         if (!s->initial_zeroing_ongoing) {
21630a5c887SKevin Wolf             job_progress_update(&s->common.job, op->bytes);
217b21c7652SMax Reitz         }
21890ab48ebSAnton Nefedov     }
2196df3bf8eSZhang Min     qemu_iovec_destroy(&op->qiov);
2207b770c72SStefan Hajnoczi 
22112aa4082SMax Reitz     qemu_co_queue_restart_all(&op->waiting_requests);
22212aa4082SMax Reitz     g_free(op);
2237b770c72SStefan Hajnoczi }
224bd48bde8SPaolo Bonzini 
2252e1990b2SMax Reitz static void coroutine_fn mirror_write_complete(MirrorOp *op, int ret)
226bd48bde8SPaolo Bonzini {
227bd48bde8SPaolo Bonzini     MirrorBlockJob *s = op->s;
228b9e413ddSPaolo Bonzini 
229bd48bde8SPaolo Bonzini     if (ret < 0) {
230bd48bde8SPaolo Bonzini         BlockErrorAction action;
231bd48bde8SPaolo Bonzini 
232e0d7f73eSEric Blake         bdrv_set_dirty_bitmap(s->dirty_bitmap, op->offset, op->bytes);
233bd48bde8SPaolo Bonzini         action = mirror_error_action(s, false, -ret);
234a589569fSWenchao Xia         if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) {
235bd48bde8SPaolo Bonzini             s->ret = ret;
236bd48bde8SPaolo Bonzini         }
237bd48bde8SPaolo Bonzini     }
238d12ade57SVladimir Sementsov-Ogievskiy 
239bd48bde8SPaolo Bonzini     mirror_iteration_done(op, ret);
240bd48bde8SPaolo Bonzini }
241bd48bde8SPaolo Bonzini 
2422e1990b2SMax Reitz static void coroutine_fn mirror_read_complete(MirrorOp *op, int ret)
243bd48bde8SPaolo Bonzini {
244bd48bde8SPaolo Bonzini     MirrorBlockJob *s = op->s;
245b9e413ddSPaolo Bonzini 
246bd48bde8SPaolo Bonzini     if (ret < 0) {
247bd48bde8SPaolo Bonzini         BlockErrorAction action;
248bd48bde8SPaolo Bonzini 
249e0d7f73eSEric Blake         bdrv_set_dirty_bitmap(s->dirty_bitmap, op->offset, op->bytes);
250bd48bde8SPaolo Bonzini         action = mirror_error_action(s, true, -ret);
251a589569fSWenchao Xia         if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) {
252bd48bde8SPaolo Bonzini             s->ret = ret;
253bd48bde8SPaolo Bonzini         }
254bd48bde8SPaolo Bonzini 
255bd48bde8SPaolo Bonzini         mirror_iteration_done(op, ret);
256d12ade57SVladimir Sementsov-Ogievskiy         return;
257bd48bde8SPaolo Bonzini     }
258d12ade57SVladimir Sementsov-Ogievskiy 
259d12ade57SVladimir Sementsov-Ogievskiy     ret = blk_co_pwritev(s->target, op->offset, op->qiov.size, &op->qiov, 0);
260d12ade57SVladimir Sementsov-Ogievskiy     mirror_write_complete(op, ret);
261b9e413ddSPaolo Bonzini }
262bd48bde8SPaolo Bonzini 
263782d97efSEric Blake /* Clip bytes relative to offset to not exceed end-of-file */
264782d97efSEric Blake static inline int64_t mirror_clip_bytes(MirrorBlockJob *s,
265782d97efSEric Blake                                         int64_t offset,
266782d97efSEric Blake                                         int64_t bytes)
267782d97efSEric Blake {
268782d97efSEric Blake     return MIN(bytes, s->bdev_length - offset);
269782d97efSEric Blake }
270782d97efSEric Blake 
271782d97efSEric Blake /* Round offset and/or bytes to target cluster if COW is needed, and
272782d97efSEric Blake  * return the offset of the adjusted tail against original. */
27317ac39c3SPaolo Bonzini static int coroutine_fn mirror_cow_align(MirrorBlockJob *s, int64_t *offset,
274ae4cc877SEric Blake                                          uint64_t *bytes)
275893f7ebaSPaolo Bonzini {
276e5b43573SFam Zheng     bool need_cow;
277e5b43573SFam Zheng     int ret = 0;
278782d97efSEric Blake     int64_t align_offset = *offset;
2797cfd5275SEric Blake     int64_t align_bytes = *bytes;
280782d97efSEric Blake     int max_bytes = s->granularity * s->max_iov;
281893f7ebaSPaolo Bonzini 
282782d97efSEric Blake     need_cow = !test_bit(*offset / s->granularity, s->cow_bitmap);
283782d97efSEric Blake     need_cow |= !test_bit((*offset + *bytes - 1) / s->granularity,
284e5b43573SFam Zheng                           s->cow_bitmap);
285e5b43573SFam Zheng     if (need_cow) {
286782d97efSEric Blake         bdrv_round_to_clusters(blk_bs(s->target), *offset, *bytes,
287782d97efSEric Blake                                &align_offset, &align_bytes);
2888f0720ecSPaolo Bonzini     }
2898f0720ecSPaolo Bonzini 
290782d97efSEric Blake     if (align_bytes > max_bytes) {
291782d97efSEric Blake         align_bytes = max_bytes;
292e5b43573SFam Zheng         if (need_cow) {
293782d97efSEric Blake             align_bytes = QEMU_ALIGN_DOWN(align_bytes, s->target_cluster_size);
294e5b43573SFam Zheng         }
295e5b43573SFam Zheng     }
296782d97efSEric Blake     /* Clipping may result in align_bytes unaligned to chunk boundary, but
2974150ae60SFam Zheng      * that doesn't matter because it's already the end of source image. */
298782d97efSEric Blake     align_bytes = mirror_clip_bytes(s, align_offset, align_bytes);
299402a4741SPaolo Bonzini 
300782d97efSEric Blake     ret = align_offset + align_bytes - (*offset + *bytes);
301782d97efSEric Blake     *offset = align_offset;
302782d97efSEric Blake     *bytes = align_bytes;
303e5b43573SFam Zheng     assert(ret >= 0);
304e5b43573SFam Zheng     return ret;
305e5b43573SFam Zheng }
306e5b43573SFam Zheng 
307537c3d4fSStefan Hajnoczi static inline void coroutine_fn
308eb994912SHanna Reitz mirror_wait_for_free_in_flight_slot(MirrorBlockJob *s)
30921cd917fSFam Zheng {
31012aa4082SMax Reitz     MirrorOp *op;
31112aa4082SMax Reitz 
3121181e19aSMax Reitz     QTAILQ_FOREACH(op, &s->ops_in_flight, next) {
313eb994912SHanna Reitz         /*
314eb994912SHanna Reitz          * Do not wait on pseudo ops, because it may in turn wait on
3151181e19aSMax Reitz          * some other operation to start, which may in fact be the
3161181e19aSMax Reitz          * caller of this function.  Since there is only one pseudo op
3171181e19aSMax Reitz          * at any given time, we will always find some real operation
318eb994912SHanna Reitz          * to wait on.
319eb994912SHanna Reitz          * Also, do not wait on active operations, because they do not
320eb994912SHanna Reitz          * use up in-flight slots.
321eb994912SHanna Reitz          */
322eb994912SHanna Reitz         if (!op->is_pseudo_op && op->is_in_flight && !op->is_active_write) {
32312aa4082SMax Reitz             qemu_co_queue_wait(&op->waiting_requests, NULL);
3241181e19aSMax Reitz             return;
3251181e19aSMax Reitz         }
3261181e19aSMax Reitz     }
3271181e19aSMax Reitz     abort();
32821cd917fSFam Zheng }
32921cd917fSFam Zheng 
3302e1990b2SMax Reitz /* Perform a mirror copy operation.
3312e1990b2SMax Reitz  *
3322e1990b2SMax Reitz  * *op->bytes_handled is set to the number of bytes copied after and
3332e1990b2SMax Reitz  * including offset, excluding any bytes copied prior to offset due
3342e1990b2SMax Reitz  * to alignment.  This will be op->bytes if no alignment is necessary,
3352e1990b2SMax Reitz  * or (new_end - op->offset) if the tail is rounded up or down due to
336e5b43573SFam Zheng  * alignment or buffer limit.
337402a4741SPaolo Bonzini  */
3382e1990b2SMax Reitz static void coroutine_fn mirror_co_read(void *opaque)
339e5b43573SFam Zheng {
3402e1990b2SMax Reitz     MirrorOp *op = opaque;
3412e1990b2SMax Reitz     MirrorBlockJob *s = op->s;
342ae4cc877SEric Blake     int nb_chunks;
343ae4cc877SEric Blake     uint64_t ret;
344ae4cc877SEric Blake     uint64_t max_bytes;
345402a4741SPaolo Bonzini 
346ae4cc877SEric Blake     max_bytes = s->granularity * s->max_iov;
347e5b43573SFam Zheng 
348e5b43573SFam Zheng     /* We can only handle as much as buf_size at a time. */
3492e1990b2SMax Reitz     op->bytes = MIN(s->buf_size, MIN(max_bytes, op->bytes));
3502e1990b2SMax Reitz     assert(op->bytes);
3512e1990b2SMax Reitz     assert(op->bytes < BDRV_REQUEST_MAX_BYTES);
3522e1990b2SMax Reitz     *op->bytes_handled = op->bytes;
353e5b43573SFam Zheng 
354e5b43573SFam Zheng     if (s->cow_bitmap) {
3552e1990b2SMax Reitz         *op->bytes_handled += mirror_cow_align(s, &op->offset, &op->bytes);
356e5b43573SFam Zheng     }
3572e1990b2SMax Reitz     /* Cannot exceed BDRV_REQUEST_MAX_BYTES + INT_MAX */
3582e1990b2SMax Reitz     assert(*op->bytes_handled <= UINT_MAX);
3592e1990b2SMax Reitz     assert(op->bytes <= s->buf_size);
360ae4cc877SEric Blake     /* The offset is granularity-aligned because:
361e5b43573SFam Zheng      * 1) Caller passes in aligned values;
362e5b43573SFam Zheng      * 2) mirror_cow_align is used only when target cluster is larger. */
3632e1990b2SMax Reitz     assert(QEMU_IS_ALIGNED(op->offset, s->granularity));
364ae4cc877SEric Blake     /* The range is sector-aligned, since bdrv_getlength() rounds up. */
3652e1990b2SMax Reitz     assert(QEMU_IS_ALIGNED(op->bytes, BDRV_SECTOR_SIZE));
3662e1990b2SMax Reitz     nb_chunks = DIV_ROUND_UP(op->bytes, s->granularity);
367e5b43573SFam Zheng 
368e5b43573SFam Zheng     while (s->buf_free_count < nb_chunks) {
3692e1990b2SMax Reitz         trace_mirror_yield_in_flight(s, op->offset, s->in_flight);
3709178f4feSKevin Wolf         mirror_wait_for_free_in_flight_slot(s);
371b812f671SPaolo Bonzini     }
372b812f671SPaolo Bonzini 
373402a4741SPaolo Bonzini     /* Now make a QEMUIOVector taking enough granularity-sized chunks
374402a4741SPaolo Bonzini      * from s->buf_free.
375402a4741SPaolo Bonzini      */
376402a4741SPaolo Bonzini     qemu_iovec_init(&op->qiov, nb_chunks);
377402a4741SPaolo Bonzini     while (nb_chunks-- > 0) {
378402a4741SPaolo Bonzini         MirrorBuffer *buf = QSIMPLEQ_FIRST(&s->buf_free);
3792e1990b2SMax Reitz         size_t remaining = op->bytes - op->qiov.size;
3805a0f6fd5SKevin Wolf 
381402a4741SPaolo Bonzini         QSIMPLEQ_REMOVE_HEAD(&s->buf_free, next);
382402a4741SPaolo Bonzini         s->buf_free_count--;
3835a0f6fd5SKevin Wolf         qemu_iovec_add(&op->qiov, buf, MIN(s->granularity, remaining));
384402a4741SPaolo Bonzini     }
385402a4741SPaolo Bonzini 
386893f7ebaSPaolo Bonzini     /* Copy the dirty cluster.  */
387bd48bde8SPaolo Bonzini     s->in_flight++;
3882e1990b2SMax Reitz     s->bytes_in_flight += op->bytes;
389ce8cabbdSKevin Wolf     op->is_in_flight = true;
3902e1990b2SMax Reitz     trace_mirror_one_iteration(s, op->offset, op->bytes);
391dcfb3bebSFam Zheng 
392b9b10c35SKevin Wolf     WITH_GRAPH_RDLOCK_GUARD() {
393138f9fffSMax Reitz         ret = bdrv_co_preadv(s->mirror_top_bs->backing, op->offset, op->bytes,
394138f9fffSMax Reitz                              &op->qiov, 0);
395b9b10c35SKevin Wolf     }
3962e1990b2SMax Reitz     mirror_read_complete(op, ret);
397e5b43573SFam Zheng }
398e5b43573SFam Zheng 
3992e1990b2SMax Reitz static void coroutine_fn mirror_co_zero(void *opaque)
400e5b43573SFam Zheng {
4012e1990b2SMax Reitz     MirrorOp *op = opaque;
4022e1990b2SMax Reitz     int ret;
403e5b43573SFam Zheng 
4042e1990b2SMax Reitz     op->s->in_flight++;
4052e1990b2SMax Reitz     op->s->bytes_in_flight += op->bytes;
4062e1990b2SMax Reitz     *op->bytes_handled = op->bytes;
407ce8cabbdSKevin Wolf     op->is_in_flight = true;
408e5b43573SFam Zheng 
4092e1990b2SMax Reitz     ret = blk_co_pwrite_zeroes(op->s->target, op->offset, op->bytes,
4102e1990b2SMax Reitz                                op->s->unmap ? BDRV_REQ_MAY_UNMAP : 0);
4112e1990b2SMax Reitz     mirror_write_complete(op, ret);
412e5b43573SFam Zheng }
4132e1990b2SMax Reitz 
4142e1990b2SMax Reitz static void coroutine_fn mirror_co_discard(void *opaque)
4152e1990b2SMax Reitz {
4162e1990b2SMax Reitz     MirrorOp *op = opaque;
4172e1990b2SMax Reitz     int ret;
4182e1990b2SMax Reitz 
4192e1990b2SMax Reitz     op->s->in_flight++;
4202e1990b2SMax Reitz     op->s->bytes_in_flight += op->bytes;
4212e1990b2SMax Reitz     *op->bytes_handled = op->bytes;
422ce8cabbdSKevin Wolf     op->is_in_flight = true;
4232e1990b2SMax Reitz 
4242e1990b2SMax Reitz     ret = blk_co_pdiscard(op->s->target, op->offset, op->bytes);
4252e1990b2SMax Reitz     mirror_write_complete(op, ret);
426e5b43573SFam Zheng }
427e5b43573SFam Zheng 
4284295c5fcSMax Reitz static unsigned mirror_perform(MirrorBlockJob *s, int64_t offset,
4294295c5fcSMax Reitz                                unsigned bytes, MirrorMethod mirror_method)
4304295c5fcSMax Reitz {
4312e1990b2SMax Reitz     MirrorOp *op;
4322e1990b2SMax Reitz     Coroutine *co;
4332e1990b2SMax Reitz     int64_t bytes_handled = -1;
4342e1990b2SMax Reitz 
4352e1990b2SMax Reitz     op = g_new(MirrorOp, 1);
4362e1990b2SMax Reitz     *op = (MirrorOp){
4372e1990b2SMax Reitz         .s              = s,
4382e1990b2SMax Reitz         .offset         = offset,
4392e1990b2SMax Reitz         .bytes          = bytes,
4402e1990b2SMax Reitz         .bytes_handled  = &bytes_handled,
4412e1990b2SMax Reitz     };
44212aa4082SMax Reitz     qemu_co_queue_init(&op->waiting_requests);
4432e1990b2SMax Reitz 
4444295c5fcSMax Reitz     switch (mirror_method) {
4454295c5fcSMax Reitz     case MIRROR_METHOD_COPY:
4462e1990b2SMax Reitz         co = qemu_coroutine_create(mirror_co_read, op);
4472e1990b2SMax Reitz         break;
4484295c5fcSMax Reitz     case MIRROR_METHOD_ZERO:
4492e1990b2SMax Reitz         co = qemu_coroutine_create(mirror_co_zero, op);
4502e1990b2SMax Reitz         break;
4514295c5fcSMax Reitz     case MIRROR_METHOD_DISCARD:
4522e1990b2SMax Reitz         co = qemu_coroutine_create(mirror_co_discard, op);
4532e1990b2SMax Reitz         break;
4544295c5fcSMax Reitz     default:
4554295c5fcSMax Reitz         abort();
4564295c5fcSMax Reitz     }
457eed325b9SKevin Wolf     op->co = co;
4582e1990b2SMax Reitz 
45912aa4082SMax Reitz     QTAILQ_INSERT_TAIL(&s->ops_in_flight, op, next);
4602e1990b2SMax Reitz     qemu_coroutine_enter(co);
4612e1990b2SMax Reitz     /* At this point, ownership of op has been moved to the coroutine
4622e1990b2SMax Reitz      * and the object may already be freed */
4632e1990b2SMax Reitz 
4642e1990b2SMax Reitz     /* Assert that this value has been set */
4652e1990b2SMax Reitz     assert(bytes_handled >= 0);
4662e1990b2SMax Reitz 
4672e1990b2SMax Reitz     /* Same assertion as in mirror_co_read() (and for mirror_co_read()
4682e1990b2SMax Reitz      * and mirror_co_discard(), bytes_handled == op->bytes, which
4692e1990b2SMax Reitz      * is the @bytes parameter given to this function) */
4702e1990b2SMax Reitz     assert(bytes_handled <= UINT_MAX);
4712e1990b2SMax Reitz     return bytes_handled;
4724295c5fcSMax Reitz }
4734295c5fcSMax Reitz 
474*018e5987SKevin Wolf static void coroutine_fn mirror_iteration(MirrorBlockJob *s)
475e5b43573SFam Zheng {
476138f9fffSMax Reitz     BlockDriverState *source = s->mirror_top_bs->backing->bs;
4771181e19aSMax Reitz     MirrorOp *pseudo_op;
4781181e19aSMax Reitz     int64_t offset;
479e5b43573SFam Zheng     /* At least the first dirty chunk is mirrored in one iteration. */
480e5b43573SFam Zheng     int nb_chunks = 1;
4814b5004d9SDenis V. Lunev     bool write_zeroes_ok = bdrv_can_write_zeroes_with_unmap(blk_bs(s->target));
482b436982fSEric Blake     int max_io_bytes = MAX(s->buf_size / MAX_IN_FLIGHT, MAX_IO_BYTES);
483e5b43573SFam Zheng 
484b64bd51eSPaolo Bonzini     bdrv_dirty_bitmap_lock(s->dirty_bitmap);
485f798184cSEric Blake     offset = bdrv_dirty_iter_next(s->dbi);
486fb2ef791SEric Blake     if (offset < 0) {
487dc162c8eSFam Zheng         bdrv_set_dirty_iter(s->dbi, 0);
488f798184cSEric Blake         offset = bdrv_dirty_iter_next(s->dbi);
4899a46dba7SEric Blake         trace_mirror_restart_iter(s, bdrv_get_dirty_count(s->dirty_bitmap));
490fb2ef791SEric Blake         assert(offset >= 0);
491e5b43573SFam Zheng     }
492b64bd51eSPaolo Bonzini     bdrv_dirty_bitmap_unlock(s->dirty_bitmap);
493e5b43573SFam Zheng 
494d69a879bSHanna Reitz     /*
495d69a879bSHanna Reitz      * Wait for concurrent requests to @offset.  The next loop will limit the
496d69a879bSHanna Reitz      * copied area based on in_flight_bitmap so we only copy an area that does
497d69a879bSHanna Reitz      * not overlap with concurrent in-flight requests.  Still, we would like to
498d69a879bSHanna Reitz      * copy something, so wait until there are at least no more requests to the
499d69a879bSHanna Reitz      * very beginning of the area.
500d69a879bSHanna Reitz      */
5011181e19aSMax Reitz     mirror_wait_on_conflicts(NULL, s, offset, 1);
5029c83625bSMax Reitz 
503da01ff7fSKevin Wolf     job_pause_point(&s->common.job);
504565ac01fSStefan Hajnoczi 
505e5b43573SFam Zheng     /* Find the number of consective dirty chunks following the first dirty
506e5b43573SFam Zheng      * one, and wait for in flight requests in them. */
507b64bd51eSPaolo Bonzini     bdrv_dirty_bitmap_lock(s->dirty_bitmap);
508fb2ef791SEric Blake     while (nb_chunks * s->granularity < s->buf_size) {
509dc162c8eSFam Zheng         int64_t next_dirty;
510fb2ef791SEric Blake         int64_t next_offset = offset + nb_chunks * s->granularity;
511fb2ef791SEric Blake         int64_t next_chunk = next_offset / s->granularity;
512fb2ef791SEric Blake         if (next_offset >= s->bdev_length ||
51328636b82SJohn Snow             !bdrv_dirty_bitmap_get_locked(s->dirty_bitmap, next_offset)) {
514e5b43573SFam Zheng             break;
515e5b43573SFam Zheng         }
516e5b43573SFam Zheng         if (test_bit(next_chunk, s->in_flight_bitmap)) {
517e5b43573SFam Zheng             break;
518e5b43573SFam Zheng         }
5199c83625bSMax Reitz 
520f798184cSEric Blake         next_dirty = bdrv_dirty_iter_next(s->dbi);
521fb2ef791SEric Blake         if (next_dirty > next_offset || next_dirty < 0) {
522f27a2742SMax Reitz             /* The bitmap iterator's cache is stale, refresh it */
523715a74d8SEric Blake             bdrv_set_dirty_iter(s->dbi, next_offset);
524f798184cSEric Blake             next_dirty = bdrv_dirty_iter_next(s->dbi);
525f27a2742SMax Reitz         }
526fb2ef791SEric Blake         assert(next_dirty == next_offset);
527e5b43573SFam Zheng         nb_chunks++;
528e5b43573SFam Zheng     }
529e5b43573SFam Zheng 
530e5b43573SFam Zheng     /* Clear dirty bits before querying the block status, because
53131826642SEric Blake      * calling bdrv_block_status_above could yield - if some blocks are
532e5b43573SFam Zheng      * marked dirty in this window, we need to know.
533e5b43573SFam Zheng      */
534e0d7f73eSEric Blake     bdrv_reset_dirty_bitmap_locked(s->dirty_bitmap, offset,
535e0d7f73eSEric Blake                                    nb_chunks * s->granularity);
536b64bd51eSPaolo Bonzini     bdrv_dirty_bitmap_unlock(s->dirty_bitmap);
537b64bd51eSPaolo Bonzini 
5381181e19aSMax Reitz     /* Before claiming an area in the in-flight bitmap, we have to
5391181e19aSMax Reitz      * create a MirrorOp for it so that conflicting requests can wait
5401181e19aSMax Reitz      * for it.  mirror_perform() will create the real MirrorOps later,
5411181e19aSMax Reitz      * for now we just create a pseudo operation that will wake up all
5421181e19aSMax Reitz      * conflicting requests once all real operations have been
5431181e19aSMax Reitz      * launched. */
5441181e19aSMax Reitz     pseudo_op = g_new(MirrorOp, 1);
5451181e19aSMax Reitz     *pseudo_op = (MirrorOp){
5461181e19aSMax Reitz         .offset         = offset,
5471181e19aSMax Reitz         .bytes          = nb_chunks * s->granularity,
5481181e19aSMax Reitz         .is_pseudo_op   = true,
5491181e19aSMax Reitz     };
5501181e19aSMax Reitz     qemu_co_queue_init(&pseudo_op->waiting_requests);
5511181e19aSMax Reitz     QTAILQ_INSERT_TAIL(&s->ops_in_flight, pseudo_op, next);
5521181e19aSMax Reitz 
553fb2ef791SEric Blake     bitmap_set(s->in_flight_bitmap, offset / s->granularity, nb_chunks);
554fb2ef791SEric Blake     while (nb_chunks > 0 && offset < s->bdev_length) {
55531826642SEric Blake         int ret;
5567cfd5275SEric Blake         int64_t io_bytes;
557f3e4ce4aSEric Blake         int64_t io_bytes_acct;
5584295c5fcSMax Reitz         MirrorMethod mirror_method = MIRROR_METHOD_COPY;
559e5b43573SFam Zheng 
560fb2ef791SEric Blake         assert(!(offset % s->granularity));
5617ff9579eSKevin Wolf         WITH_GRAPH_RDLOCK_GUARD() {
56231826642SEric Blake             ret = bdrv_block_status_above(source, NULL, offset,
56331826642SEric Blake                                         nb_chunks * s->granularity,
56431826642SEric Blake                                         &io_bytes, NULL, NULL);
5657ff9579eSKevin Wolf         }
566e5b43573SFam Zheng         if (ret < 0) {
567fb2ef791SEric Blake             io_bytes = MIN(nb_chunks * s->granularity, max_io_bytes);
5680965a41eSVladimir Sementsov-Ogievskiy         } else if (ret & BDRV_BLOCK_DATA) {
569fb2ef791SEric Blake             io_bytes = MIN(io_bytes, max_io_bytes);
570e5b43573SFam Zheng         }
571e5b43573SFam Zheng 
572fb2ef791SEric Blake         io_bytes -= io_bytes % s->granularity;
573fb2ef791SEric Blake         if (io_bytes < s->granularity) {
574fb2ef791SEric Blake             io_bytes = s->granularity;
575e5b43573SFam Zheng         } else if (ret >= 0 && !(ret & BDRV_BLOCK_DATA)) {
576fb2ef791SEric Blake             int64_t target_offset;
5777cfd5275SEric Blake             int64_t target_bytes;
578a00e70c0SEmanuele Giuseppe Esposito             WITH_GRAPH_RDLOCK_GUARD() {
579fb2ef791SEric Blake                 bdrv_round_to_clusters(blk_bs(s->target), offset, io_bytes,
580fb2ef791SEric Blake                                        &target_offset, &target_bytes);
581a00e70c0SEmanuele Giuseppe Esposito             }
582fb2ef791SEric Blake             if (target_offset == offset &&
583fb2ef791SEric Blake                 target_bytes == io_bytes) {
584e5b43573SFam Zheng                 mirror_method = ret & BDRV_BLOCK_ZERO ?
585e5b43573SFam Zheng                                     MIRROR_METHOD_ZERO :
586e5b43573SFam Zheng                                     MIRROR_METHOD_DISCARD;
587e5b43573SFam Zheng             }
588e5b43573SFam Zheng         }
589e5b43573SFam Zheng 
590cf56a3c6SDenis V. Lunev         while (s->in_flight >= MAX_IN_FLIGHT) {
591fb2ef791SEric Blake             trace_mirror_yield_in_flight(s, offset, s->in_flight);
5929178f4feSKevin Wolf             mirror_wait_for_free_in_flight_slot(s);
593cf56a3c6SDenis V. Lunev         }
594cf56a3c6SDenis V. Lunev 
595dbaa7b57SVladimir Sementsov-Ogievskiy         if (s->ret < 0) {
5961181e19aSMax Reitz             ret = 0;
5971181e19aSMax Reitz             goto fail;
598dbaa7b57SVladimir Sementsov-Ogievskiy         }
599dbaa7b57SVladimir Sementsov-Ogievskiy 
600fb2ef791SEric Blake         io_bytes = mirror_clip_bytes(s, offset, io_bytes);
6014295c5fcSMax Reitz         io_bytes = mirror_perform(s, offset, io_bytes, mirror_method);
6024295c5fcSMax Reitz         if (mirror_method != MIRROR_METHOD_COPY && write_zeroes_ok) {
603f3e4ce4aSEric Blake             io_bytes_acct = 0;
6044b5004d9SDenis V. Lunev         } else {
605fb2ef791SEric Blake             io_bytes_acct = io_bytes;
6064b5004d9SDenis V. Lunev         }
607fb2ef791SEric Blake         assert(io_bytes);
608fb2ef791SEric Blake         offset += io_bytes;
609fb2ef791SEric Blake         nb_chunks -= DIV_ROUND_UP(io_bytes, s->granularity);
610*018e5987SKevin Wolf         block_job_ratelimit_processed_bytes(&s->common, io_bytes_acct);
611dcfb3bebSFam Zheng     }
6121181e19aSMax Reitz 
6131181e19aSMax Reitz fail:
6141181e19aSMax Reitz     QTAILQ_REMOVE(&s->ops_in_flight, pseudo_op, next);
6151181e19aSMax Reitz     qemu_co_queue_restart_all(&pseudo_op->waiting_requests);
6161181e19aSMax Reitz     g_free(pseudo_op);
617893f7ebaSPaolo Bonzini }
618b952b558SPaolo Bonzini 
619402a4741SPaolo Bonzini static void mirror_free_init(MirrorBlockJob *s)
620402a4741SPaolo Bonzini {
621402a4741SPaolo Bonzini     int granularity = s->granularity;
622402a4741SPaolo Bonzini     size_t buf_size = s->buf_size;
623402a4741SPaolo Bonzini     uint8_t *buf = s->buf;
624402a4741SPaolo Bonzini 
625402a4741SPaolo Bonzini     assert(s->buf_free_count == 0);
626402a4741SPaolo Bonzini     QSIMPLEQ_INIT(&s->buf_free);
627402a4741SPaolo Bonzini     while (buf_size != 0) {
628402a4741SPaolo Bonzini         MirrorBuffer *cur = (MirrorBuffer *)buf;
629402a4741SPaolo Bonzini         QSIMPLEQ_INSERT_TAIL(&s->buf_free, cur, next);
630402a4741SPaolo Bonzini         s->buf_free_count++;
631402a4741SPaolo Bonzini         buf_size -= granularity;
632402a4741SPaolo Bonzini         buf += granularity;
633402a4741SPaolo Bonzini     }
634402a4741SPaolo Bonzini }
635402a4741SPaolo Bonzini 
636bae8196dSPaolo Bonzini /* This is also used for the .pause callback. There is no matching
637bae8196dSPaolo Bonzini  * mirror_resume() because mirror_run() will begin iterating again
638bae8196dSPaolo Bonzini  * when the job is resumed.
639bae8196dSPaolo Bonzini  */
640537c3d4fSStefan Hajnoczi static void coroutine_fn mirror_wait_for_all_io(MirrorBlockJob *s)
641bd48bde8SPaolo Bonzini {
642bd48bde8SPaolo Bonzini     while (s->in_flight > 0) {
6439178f4feSKevin Wolf         mirror_wait_for_free_in_flight_slot(s);
644bd48bde8SPaolo Bonzini     }
645893f7ebaSPaolo Bonzini }
646893f7ebaSPaolo Bonzini 
647737efc1eSJohn Snow /**
648737efc1eSJohn Snow  * mirror_exit_common: handle both abort() and prepare() cases.
649737efc1eSJohn Snow  * for .prepare, returns 0 on success and -errno on failure.
650737efc1eSJohn Snow  * for .abort cases, denoted by abort = true, MUST return 0.
651737efc1eSJohn Snow  */
652737efc1eSJohn Snow static int mirror_exit_common(Job *job)
6535a7e7a0bSStefan Hajnoczi {
6541908a559SKevin Wolf     MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
6551908a559SKevin Wolf     BlockJob *bjob = &s->common;
656f93c3addSMax Reitz     MirrorBDSOpaque *bs_opaque;
6575a7e7a0bSStefan Hajnoczi     AioContext *replace_aio_context = NULL;
658f93c3addSMax Reitz     BlockDriverState *src;
659f93c3addSMax Reitz     BlockDriverState *target_bs;
660f93c3addSMax Reitz     BlockDriverState *mirror_top_bs;
66112fa4af6SKevin Wolf     Error *local_err = NULL;
662737efc1eSJohn Snow     bool abort = job->ret < 0;
663737efc1eSJohn Snow     int ret = 0;
664737efc1eSJohn Snow 
665737efc1eSJohn Snow     if (s->prepared) {
666737efc1eSJohn Snow         return 0;
667737efc1eSJohn Snow     }
668737efc1eSJohn Snow     s->prepared = true;
6693f09bfbcSKevin Wolf 
670f93c3addSMax Reitz     mirror_top_bs = s->mirror_top_bs;
671f93c3addSMax Reitz     bs_opaque = mirror_top_bs->opaque;
672f93c3addSMax Reitz     src = mirror_top_bs->backing->bs;
673f93c3addSMax Reitz     target_bs = blk_bs(s->target);
674f93c3addSMax Reitz 
675ef53dc09SAlberto Garcia     if (bdrv_chain_contains(src, target_bs)) {
676ef53dc09SAlberto Garcia         bdrv_unfreeze_backing_chain(mirror_top_bs, target_bs);
677ef53dc09SAlberto Garcia     }
678ef53dc09SAlberto Garcia 
6795deb6cbdSVladimir Sementsov-Ogievskiy     bdrv_release_dirty_bitmap(s->dirty_bitmap);
6802119882cSPaolo Bonzini 
6817b508f6bSJohn Snow     /* Make sure that the source BDS doesn't go away during bdrv_replace_node,
6827b508f6bSJohn Snow      * before we can call bdrv_drained_end */
6833f09bfbcSKevin Wolf     bdrv_ref(src);
6844ef85a9cSKevin Wolf     bdrv_ref(mirror_top_bs);
6857d9fcb39SKevin Wolf     bdrv_ref(target_bs);
6867d9fcb39SKevin Wolf 
687bb0c9409SVladimir Sementsov-Ogievskiy     /*
688bb0c9409SVladimir Sementsov-Ogievskiy      * Remove target parent that still uses BLK_PERM_WRITE/RESIZE before
6897d9fcb39SKevin Wolf      * inserting target_bs at s->to_replace, where we might not be able to get
69063c8ef28SKevin Wolf      * these permissions.
691bb0c9409SVladimir Sementsov-Ogievskiy      */
6927d9fcb39SKevin Wolf     blk_unref(s->target);
6937d9fcb39SKevin Wolf     s->target = NULL;
6944ef85a9cSKevin Wolf 
6954ef85a9cSKevin Wolf     /* We don't access the source any more. Dropping any WRITE/RESIZE is
696d2da5e28SKevin Wolf      * required before it could become a backing file of target_bs. Not having
697d2da5e28SKevin Wolf      * these permissions any more means that we can't allow any new requests on
698d2da5e28SKevin Wolf      * mirror_top_bs from now on, so keep it drained. */
699d2da5e28SKevin Wolf     bdrv_drained_begin(mirror_top_bs);
700f94dc3b4SMax Reitz     bs_opaque->stop = true;
701f94dc3b4SMax Reitz     bdrv_child_refresh_perms(mirror_top_bs, mirror_top_bs->backing,
7024ef85a9cSKevin Wolf                              &error_abort);
703737efc1eSJohn Snow     if (!abort && s->backing_mode == MIRROR_SOURCE_BACKING_CHAIN) {
7044ef85a9cSKevin Wolf         BlockDriverState *backing = s->is_none_mode ? src : s->base;
7053f072a7fSMax Reitz         BlockDriverState *unfiltered_target = bdrv_skip_filters(target_bs);
7063f072a7fSMax Reitz 
7073f072a7fSMax Reitz         if (bdrv_cow_bs(unfiltered_target) != backing) {
7083f072a7fSMax Reitz             bdrv_set_backing_hd(unfiltered_target, backing, &local_err);
70912fa4af6SKevin Wolf             if (local_err) {
71012fa4af6SKevin Wolf                 error_report_err(local_err);
71166c8672dSVladimir Sementsov-Ogievskiy                 local_err = NULL;
7127b508f6bSJohn Snow                 ret = -EPERM;
71312fa4af6SKevin Wolf             }
7144ef85a9cSKevin Wolf         }
715c41f5b96SMax Reitz     } else if (!abort && s->backing_mode == MIRROR_OPEN_BACKING_CHAIN) {
716c41f5b96SMax Reitz         assert(!bdrv_backing_chain_next(target_bs));
717c41f5b96SMax Reitz         ret = bdrv_open_backing_file(bdrv_skip_filters(target_bs), NULL,
718c41f5b96SMax Reitz                                      "backing", &local_err);
719c41f5b96SMax Reitz         if (ret < 0) {
720c41f5b96SMax Reitz             error_report_err(local_err);
721c41f5b96SMax Reitz             local_err = NULL;
722c41f5b96SMax Reitz         }
7234ef85a9cSKevin Wolf     }
7245a7e7a0bSStefan Hajnoczi 
7255a7e7a0bSStefan Hajnoczi     if (s->to_replace) {
7265a7e7a0bSStefan Hajnoczi         replace_aio_context = bdrv_get_aio_context(s->to_replace);
7275a7e7a0bSStefan Hajnoczi         aio_context_acquire(replace_aio_context);
7285a7e7a0bSStefan Hajnoczi     }
7295a7e7a0bSStefan Hajnoczi 
730737efc1eSJohn Snow     if (s->should_complete && !abort) {
731737efc1eSJohn Snow         BlockDriverState *to_replace = s->to_replace ?: src;
7321ba79388SAlberto Garcia         bool ro = bdrv_is_read_only(to_replace);
73340365552SKevin Wolf 
7341ba79388SAlberto Garcia         if (ro != bdrv_is_read_only(target_bs)) {
7351ba79388SAlberto Garcia             bdrv_reopen_set_read_only(target_bs, ro, NULL);
7365a7e7a0bSStefan Hajnoczi         }
737b8804815SKevin Wolf 
738b8804815SKevin Wolf         /* The mirror job has no requests in flight any more, but we need to
739b8804815SKevin Wolf          * drain potential other users of the BDS before changing the graph. */
7405e771752SSergio Lopez         assert(s->in_drain);
741e253f4b8SKevin Wolf         bdrv_drained_begin(target_bs);
7426e9cc051SMax Reitz         /*
7436e9cc051SMax Reitz          * Cannot use check_to_replace_node() here, because that would
7446e9cc051SMax Reitz          * check for an op blocker on @to_replace, and we have our own
7456e9cc051SMax Reitz          * there.
746533c6e4eSKevin Wolf          *
747533c6e4eSKevin Wolf          * TODO Pull out the writer lock from bdrv_replace_node() to here
7486e9cc051SMax Reitz          */
749533c6e4eSKevin Wolf         bdrv_graph_rdlock_main_loop();
7506e9cc051SMax Reitz         if (bdrv_recurse_can_replace(src, to_replace)) {
7515fe31c25SKevin Wolf             bdrv_replace_node(to_replace, target_bs, &local_err);
7526e9cc051SMax Reitz         } else {
7536e9cc051SMax Reitz             error_setg(&local_err, "Can no longer replace '%s' by '%s', "
7546e9cc051SMax Reitz                        "because it can no longer be guaranteed that doing so "
7556e9cc051SMax Reitz                        "would not lead to an abrupt change of visible data",
7566e9cc051SMax Reitz                        to_replace->node_name, target_bs->node_name);
7576e9cc051SMax Reitz         }
758533c6e4eSKevin Wolf         bdrv_graph_rdunlock_main_loop();
759e253f4b8SKevin Wolf         bdrv_drained_end(target_bs);
7605fe31c25SKevin Wolf         if (local_err) {
7615fe31c25SKevin Wolf             error_report_err(local_err);
7627b508f6bSJohn Snow             ret = -EPERM;
7635fe31c25SKevin Wolf         }
7645a7e7a0bSStefan Hajnoczi     }
7655a7e7a0bSStefan Hajnoczi     if (s->to_replace) {
7665a7e7a0bSStefan Hajnoczi         bdrv_op_unblock_all(s->to_replace, s->replace_blocker);
7675a7e7a0bSStefan Hajnoczi         error_free(s->replace_blocker);
7685a7e7a0bSStefan Hajnoczi         bdrv_unref(s->to_replace);
7695a7e7a0bSStefan Hajnoczi     }
7705a7e7a0bSStefan Hajnoczi     if (replace_aio_context) {
7715a7e7a0bSStefan Hajnoczi         aio_context_release(replace_aio_context);
7725a7e7a0bSStefan Hajnoczi     }
7735a7e7a0bSStefan Hajnoczi     g_free(s->replaces);
7747d9fcb39SKevin Wolf     bdrv_unref(target_bs);
7754ef85a9cSKevin Wolf 
776f94dc3b4SMax Reitz     /*
777f94dc3b4SMax Reitz      * Remove the mirror filter driver from the graph. Before this, get rid of
7784ef85a9cSKevin Wolf      * the blockers on the intermediate nodes so that the resulting state is
779f94dc3b4SMax Reitz      * valid.
780f94dc3b4SMax Reitz      */
7811908a559SKevin Wolf     block_job_remove_all_bdrv(bjob);
7823f072a7fSMax Reitz     bdrv_replace_node(mirror_top_bs, mirror_top_bs->backing->bs, &error_abort);
7834ef85a9cSKevin Wolf 
784429076e8SMax Reitz     bs_opaque->job = NULL;
7854ef85a9cSKevin Wolf 
786176c3699SFam Zheng     bdrv_drained_end(src);
787d2da5e28SKevin Wolf     bdrv_drained_end(mirror_top_bs);
7885e771752SSergio Lopez     s->in_drain = false;
7894ef85a9cSKevin Wolf     bdrv_unref(mirror_top_bs);
7903f09bfbcSKevin Wolf     bdrv_unref(src);
7917b508f6bSJohn Snow 
792737efc1eSJohn Snow     return ret;
793737efc1eSJohn Snow }
794737efc1eSJohn Snow 
795737efc1eSJohn Snow static int mirror_prepare(Job *job)
796737efc1eSJohn Snow {
797737efc1eSJohn Snow     return mirror_exit_common(job);
798737efc1eSJohn Snow }
799737efc1eSJohn Snow 
800737efc1eSJohn Snow static void mirror_abort(Job *job)
801737efc1eSJohn Snow {
802737efc1eSJohn Snow     int ret = mirror_exit_common(job);
803737efc1eSJohn Snow     assert(ret == 0);
8045a7e7a0bSStefan Hajnoczi }
8055a7e7a0bSStefan Hajnoczi 
806537c3d4fSStefan Hajnoczi static void coroutine_fn mirror_throttle(MirrorBlockJob *s)
80749efb1f5SDenis V. Lunev {
80849efb1f5SDenis V. Lunev     int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
80949efb1f5SDenis V. Lunev 
81018bb6928SKevin Wolf     if (now - s->last_pause_ns > BLOCK_JOB_SLICE_TIME) {
81149efb1f5SDenis V. Lunev         s->last_pause_ns = now;
8125d43e86eSKevin Wolf         job_sleep_ns(&s->common.job, 0);
81349efb1f5SDenis V. Lunev     } else {
814da01ff7fSKevin Wolf         job_pause_point(&s->common.job);
81549efb1f5SDenis V. Lunev     }
81649efb1f5SDenis V. Lunev }
81749efb1f5SDenis V. Lunev 
818c0b363adSDenis V. Lunev static int coroutine_fn mirror_dirty_init(MirrorBlockJob *s)
819c0b363adSDenis V. Lunev {
82023ca459aSEric Blake     int64_t offset;
821138f9fffSMax Reitz     BlockDriverState *bs = s->mirror_top_bs->backing->bs;
822c0b363adSDenis V. Lunev     BlockDriverState *target_bs = blk_bs(s->target);
82323ca459aSEric Blake     int ret;
82451b0a488SEric Blake     int64_t count;
825c0b363adSDenis V. Lunev 
826cdf3bc93SMax Reitz     if (s->zero_target) {
827c7c2769cSDenis V. Lunev         if (!bdrv_can_write_zeroes_with_unmap(target_bs)) {
828e0d7f73eSEric Blake             bdrv_set_dirty_bitmap(s->dirty_bitmap, 0, s->bdev_length);
829b7d5062cSDenis V. Lunev             return 0;
830b7d5062cSDenis V. Lunev         }
831b7d5062cSDenis V. Lunev 
83290ab48ebSAnton Nefedov         s->initial_zeroing_ongoing = true;
83323ca459aSEric Blake         for (offset = 0; offset < s->bdev_length; ) {
83423ca459aSEric Blake             int bytes = MIN(s->bdev_length - offset,
83523ca459aSEric Blake                             QEMU_ALIGN_DOWN(INT_MAX, s->granularity));
836c7c2769cSDenis V. Lunev 
837c7c2769cSDenis V. Lunev             mirror_throttle(s);
838c7c2769cSDenis V. Lunev 
839daa7f2f9SKevin Wolf             if (job_is_cancelled(&s->common.job)) {
84090ab48ebSAnton Nefedov                 s->initial_zeroing_ongoing = false;
841c7c2769cSDenis V. Lunev                 return 0;
842c7c2769cSDenis V. Lunev             }
843c7c2769cSDenis V. Lunev 
844c7c2769cSDenis V. Lunev             if (s->in_flight >= MAX_IN_FLIGHT) {
84567adf4b3SEric Blake                 trace_mirror_yield(s, UINT64_MAX, s->buf_free_count,
84667adf4b3SEric Blake                                    s->in_flight);
8479178f4feSKevin Wolf                 mirror_wait_for_free_in_flight_slot(s);
848c7c2769cSDenis V. Lunev                 continue;
849c7c2769cSDenis V. Lunev             }
850c7c2769cSDenis V. Lunev 
8514295c5fcSMax Reitz             mirror_perform(s, offset, bytes, MIRROR_METHOD_ZERO);
85223ca459aSEric Blake             offset += bytes;
853c7c2769cSDenis V. Lunev         }
854c7c2769cSDenis V. Lunev 
855bae8196dSPaolo Bonzini         mirror_wait_for_all_io(s);
85690ab48ebSAnton Nefedov         s->initial_zeroing_ongoing = false;
857c7c2769cSDenis V. Lunev     }
858c7c2769cSDenis V. Lunev 
859c0b363adSDenis V. Lunev     /* First part, loop on the sectors and initialize the dirty bitmap.  */
86023ca459aSEric Blake     for (offset = 0; offset < s->bdev_length; ) {
861c0b363adSDenis V. Lunev         /* Just to make sure we are not exceeding int limit. */
86223ca459aSEric Blake         int bytes = MIN(s->bdev_length - offset,
86323ca459aSEric Blake                         QEMU_ALIGN_DOWN(INT_MAX, s->granularity));
864c0b363adSDenis V. Lunev 
865c0b363adSDenis V. Lunev         mirror_throttle(s);
866c0b363adSDenis V. Lunev 
867daa7f2f9SKevin Wolf         if (job_is_cancelled(&s->common.job)) {
868c0b363adSDenis V. Lunev             return 0;
869c0b363adSDenis V. Lunev         }
870c0b363adSDenis V. Lunev 
8717ff9579eSKevin Wolf         WITH_GRAPH_RDLOCK_GUARD() {
8727ff9579eSKevin Wolf             ret = bdrv_is_allocated_above(bs, s->base_overlay, true, offset,
8737ff9579eSKevin Wolf                                           bytes, &count);
8747ff9579eSKevin Wolf         }
875c0b363adSDenis V. Lunev         if (ret < 0) {
876c0b363adSDenis V. Lunev             return ret;
877c0b363adSDenis V. Lunev         }
878c0b363adSDenis V. Lunev 
87923ca459aSEric Blake         assert(count);
880a92b1b06SEric Blake         if (ret > 0) {
88123ca459aSEric Blake             bdrv_set_dirty_bitmap(s->dirty_bitmap, offset, count);
882c0b363adSDenis V. Lunev         }
88323ca459aSEric Blake         offset += count;
884c0b363adSDenis V. Lunev     }
885c0b363adSDenis V. Lunev     return 0;
886c0b363adSDenis V. Lunev }
887c0b363adSDenis V. Lunev 
888bdffb31dSPaolo Bonzini /* Called when going out of the streaming phase to flush the bulk of the
889bdffb31dSPaolo Bonzini  * data to the medium, or just before completing.
890bdffb31dSPaolo Bonzini  */
89126bef102SPaolo Bonzini static int coroutine_fn mirror_flush(MirrorBlockJob *s)
892bdffb31dSPaolo Bonzini {
89326bef102SPaolo Bonzini     int ret = blk_co_flush(s->target);
894bdffb31dSPaolo Bonzini     if (ret < 0) {
895bdffb31dSPaolo Bonzini         if (mirror_error_action(s, false, -ret) == BLOCK_ERROR_ACTION_REPORT) {
896bdffb31dSPaolo Bonzini             s->ret = ret;
897bdffb31dSPaolo Bonzini         }
898bdffb31dSPaolo Bonzini     }
899bdffb31dSPaolo Bonzini     return ret;
900bdffb31dSPaolo Bonzini }
901bdffb31dSPaolo Bonzini 
902f67432a2SJohn Snow static int coroutine_fn mirror_run(Job *job, Error **errp)
903893f7ebaSPaolo Bonzini {
904f67432a2SJohn Snow     MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
905138f9fffSMax Reitz     BlockDriverState *bs = s->mirror_top_bs->backing->bs;
90632125b14SKevin Wolf     MirrorBDSOpaque *mirror_top_opaque = s->mirror_top_bs->opaque;
907e253f4b8SKevin Wolf     BlockDriverState *target_bs = blk_bs(s->target);
9089a0cec66SPaolo Bonzini     bool need_drain = true;
909d59cb66dSEmanuele Giuseppe Esposito     BlockDeviceIoStatus iostatus;
910c0b363adSDenis V. Lunev     int64_t length;
911e83dd680SKevin Wolf     int64_t target_length;
912b812f671SPaolo Bonzini     BlockDriverInfo bdi;
9131d33936eSJeff Cody     char backing_filename[2]; /* we only need 2 characters because we are only
9141d33936eSJeff Cody                                  checking for a NULL string */
915893f7ebaSPaolo Bonzini     int ret = 0;
916893f7ebaSPaolo Bonzini 
917daa7f2f9SKevin Wolf     if (job_is_cancelled(&s->common.job)) {
918893f7ebaSPaolo Bonzini         goto immediate_exit;
919893f7ebaSPaolo Bonzini     }
920893f7ebaSPaolo Bonzini 
9218ab8140aSKevin Wolf     bdrv_graph_co_rdlock();
922c86422c5SEmanuele Giuseppe Esposito     s->bdev_length = bdrv_co_getlength(bs);
9238ab8140aSKevin Wolf     bdrv_graph_co_rdunlock();
9248ab8140aSKevin Wolf 
925b21c7652SMax Reitz     if (s->bdev_length < 0) {
926b21c7652SMax Reitz         ret = s->bdev_length;
927373df5b1SFam Zheng         goto immediate_exit;
928becc347eSKevin Wolf     }
929becc347eSKevin Wolf 
930c86422c5SEmanuele Giuseppe Esposito     target_length = blk_co_getlength(s->target);
931e83dd680SKevin Wolf     if (target_length < 0) {
932e83dd680SKevin Wolf         ret = target_length;
933becc347eSKevin Wolf         goto immediate_exit;
934becc347eSKevin Wolf     }
935becc347eSKevin Wolf 
936e83dd680SKevin Wolf     /* Active commit must resize the base image if its size differs from the
937e83dd680SKevin Wolf      * active layer. */
938e83dd680SKevin Wolf     if (s->base == blk_bs(s->target)) {
939e83dd680SKevin Wolf         if (s->bdev_length > target_length) {
94088276216SAlberto Faria             ret = blk_co_truncate(s->target, s->bdev_length, false,
9418c6242b6SKevin Wolf                                   PREALLOC_MODE_OFF, 0, NULL);
942becc347eSKevin Wolf             if (ret < 0) {
943becc347eSKevin Wolf                 goto immediate_exit;
944becc347eSKevin Wolf             }
945becc347eSKevin Wolf         }
946e83dd680SKevin Wolf     } else if (s->bdev_length != target_length) {
947e83dd680SKevin Wolf         error_setg(errp, "Source and target image have different sizes");
948e83dd680SKevin Wolf         ret = -EINVAL;
949e83dd680SKevin Wolf         goto immediate_exit;
950becc347eSKevin Wolf     }
951becc347eSKevin Wolf 
952becc347eSKevin Wolf     if (s->bdev_length == 0) {
9532e1795b5SKevin Wolf         /* Transition to the READY state and wait for complete. */
9542e1795b5SKevin Wolf         job_transition_to_ready(&s->common.job);
955d06107adSMax Reitz         s->actively_synced = true;
95608b83bffSHanna Reitz         while (!job_cancel_requested(&s->common.job) && !s->should_complete) {
957198c49ccSKevin Wolf             job_yield(&s->common.job);
9589e48b025SFam Zheng         }
9599e48b025SFam Zheng         goto immediate_exit;
960893f7ebaSPaolo Bonzini     }
961893f7ebaSPaolo Bonzini 
962b21c7652SMax Reitz     length = DIV_ROUND_UP(s->bdev_length, s->granularity);
963402a4741SPaolo Bonzini     s->in_flight_bitmap = bitmap_new(length);
964402a4741SPaolo Bonzini 
965b812f671SPaolo Bonzini     /* If we have no backing file yet in the destination, we cannot let
966b812f671SPaolo Bonzini      * the destination do COW.  Instead, we copy sectors around the
967b812f671SPaolo Bonzini      * dirty data if needed.  We need a bitmap to do that.
968b812f671SPaolo Bonzini      */
969e253f4b8SKevin Wolf     bdrv_get_backing_filename(target_bs, backing_filename,
970b812f671SPaolo Bonzini                               sizeof(backing_filename));
971a00e70c0SEmanuele Giuseppe Esposito     bdrv_graph_co_rdlock();
9723d47eb0aSEmanuele Giuseppe Esposito     if (!bdrv_co_get_info(target_bs, &bdi) && bdi.cluster_size) {
973b436982fSEric Blake         s->target_cluster_size = bdi.cluster_size;
974b436982fSEric Blake     } else {
975b436982fSEric Blake         s->target_cluster_size = BDRV_SECTOR_SIZE;
976c3cc95bdSFam Zheng     }
977a00e70c0SEmanuele Giuseppe Esposito     bdrv_graph_co_rdunlock();
9783f072a7fSMax Reitz     if (backing_filename[0] && !bdrv_backing_chain_next(target_bs) &&
979b436982fSEric Blake         s->granularity < s->target_cluster_size) {
980b436982fSEric Blake         s->buf_size = MAX(s->buf_size, s->target_cluster_size);
981b812f671SPaolo Bonzini         s->cow_bitmap = bitmap_new(length);
982b812f671SPaolo Bonzini     }
983e253f4b8SKevin Wolf     s->max_iov = MIN(bs->bl.max_iov, target_bs->bl.max_iov);
984b812f671SPaolo Bonzini 
9857504edf4SKevin Wolf     s->buf = qemu_try_blockalign(bs, s->buf_size);
9867504edf4SKevin Wolf     if (s->buf == NULL) {
9877504edf4SKevin Wolf         ret = -ENOMEM;
9887504edf4SKevin Wolf         goto immediate_exit;
9897504edf4SKevin Wolf     }
9907504edf4SKevin Wolf 
991402a4741SPaolo Bonzini     mirror_free_init(s);
992893f7ebaSPaolo Bonzini 
99349efb1f5SDenis V. Lunev     s->last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
99403544a6eSFam Zheng     if (!s->is_none_mode) {
995c0b363adSDenis V. Lunev         ret = mirror_dirty_init(s);
996daa7f2f9SKevin Wolf         if (ret < 0 || job_is_cancelled(&s->common.job)) {
9974c0cbd6fSFam Zheng             goto immediate_exit;
9984c0cbd6fSFam Zheng         }
999893f7ebaSPaolo Bonzini     }
1000893f7ebaSPaolo Bonzini 
100132125b14SKevin Wolf     /*
100232125b14SKevin Wolf      * Only now the job is fully initialised and mirror_top_bs should start
100332125b14SKevin Wolf      * accessing it.
100432125b14SKevin Wolf      */
100532125b14SKevin Wolf     mirror_top_opaque->job = s;
100632125b14SKevin Wolf 
1007dc162c8eSFam Zheng     assert(!s->dbi);
1008715a74d8SEric Blake     s->dbi = bdrv_dirty_iter_new(s->dirty_bitmap);
1009893f7ebaSPaolo Bonzini     for (;;) {
101049efb1f5SDenis V. Lunev         int64_t cnt, delta;
1011893f7ebaSPaolo Bonzini         bool should_complete;
1012893f7ebaSPaolo Bonzini 
1013bd48bde8SPaolo Bonzini         if (s->ret < 0) {
1014bd48bde8SPaolo Bonzini             ret = s->ret;
1015893f7ebaSPaolo Bonzini             goto immediate_exit;
1016893f7ebaSPaolo Bonzini         }
1017bd48bde8SPaolo Bonzini 
1018da01ff7fSKevin Wolf         job_pause_point(&s->common.job);
1019565ac01fSStefan Hajnoczi 
10204feeec7eSHanna Reitz         if (job_is_cancelled(&s->common.job)) {
10214feeec7eSHanna Reitz             ret = 0;
10224feeec7eSHanna Reitz             goto immediate_exit;
10234feeec7eSHanna Reitz         }
10244feeec7eSHanna Reitz 
102520dca810SJohn Snow         cnt = bdrv_get_dirty_count(s->dirty_bitmap);
102605df8a6aSKevin Wolf         /* cnt is the number of dirty bytes remaining and s->bytes_in_flight is
102705df8a6aSKevin Wolf          * the number of bytes currently being processed; together those are
102805df8a6aSKevin Wolf          * the current remaining operation length */
1029d69a879bSHanna Reitz         job_progress_set_remaining(&s->common.job,
1030d69a879bSHanna Reitz                                    s->bytes_in_flight + cnt +
1031d69a879bSHanna Reitz                                    s->active_write_bytes_in_flight);
1032bd48bde8SPaolo Bonzini 
1033bd48bde8SPaolo Bonzini         /* Note that even when no rate limit is applied we need to yield
1034a7282330SFam Zheng          * periodically with no pending I/O so that bdrv_drain_all() returns.
103518bb6928SKevin Wolf          * We do so every BLKOCK_JOB_SLICE_TIME nanoseconds, or when there is
103618bb6928SKevin Wolf          * an error, or when the source is clean, whichever comes first. */
103749efb1f5SDenis V. Lunev         delta = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - s->last_pause_ns;
1038d59cb66dSEmanuele Giuseppe Esposito         WITH_JOB_LOCK_GUARD() {
1039d59cb66dSEmanuele Giuseppe Esposito             iostatus = s->common.iostatus;
1040d59cb66dSEmanuele Giuseppe Esposito         }
104118bb6928SKevin Wolf         if (delta < BLOCK_JOB_SLICE_TIME &&
1042d59cb66dSEmanuele Giuseppe Esposito             iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
1043cf56a3c6SDenis V. Lunev             if (s->in_flight >= MAX_IN_FLIGHT || s->buf_free_count == 0 ||
1044402a4741SPaolo Bonzini                 (cnt == 0 && s->in_flight > 0)) {
10459a46dba7SEric Blake                 trace_mirror_yield(s, cnt, s->buf_free_count, s->in_flight);
10469178f4feSKevin Wolf                 mirror_wait_for_free_in_flight_slot(s);
1047bd48bde8SPaolo Bonzini                 continue;
1048bd48bde8SPaolo Bonzini             } else if (cnt != 0) {
1049*018e5987SKevin Wolf                 mirror_iteration(s);
1050893f7ebaSPaolo Bonzini             }
1051cc8c9d6cSPaolo Bonzini         }
1052893f7ebaSPaolo Bonzini 
1053893f7ebaSPaolo Bonzini         should_complete = false;
1054bd48bde8SPaolo Bonzini         if (s->in_flight == 0 && cnt == 0) {
1055893f7ebaSPaolo Bonzini             trace_mirror_before_flush(s);
105644716224SHanna Reitz             if (!job_is_ready(&s->common.job)) {
1057bdffb31dSPaolo Bonzini                 if (mirror_flush(s) < 0) {
1058bdffb31dSPaolo Bonzini                     /* Go check s->ret.  */
1059bdffb31dSPaolo Bonzini                     continue;
1060893f7ebaSPaolo Bonzini                 }
1061893f7ebaSPaolo Bonzini                 /* We're out of the streaming phase.  From now on, if the job
1062893f7ebaSPaolo Bonzini                  * is cancelled we will actually complete all pending I/O and
1063893f7ebaSPaolo Bonzini                  * report completion.  This way, block-job-cancel will leave
1064893f7ebaSPaolo Bonzini                  * the target in a consistent state.
1065893f7ebaSPaolo Bonzini                  */
10662e1795b5SKevin Wolf                 job_transition_to_ready(&s->common.job);
1067d06107adSMax Reitz                 if (s->copy_mode != MIRROR_COPY_MODE_BACKGROUND) {
1068d06107adSMax Reitz                     s->actively_synced = true;
1069d06107adSMax Reitz                 }
1070d63ffd87SPaolo Bonzini             }
1071d63ffd87SPaolo Bonzini 
1072d63ffd87SPaolo Bonzini             should_complete = s->should_complete ||
107308b83bffSHanna Reitz                 job_cancel_requested(&s->common.job);
107420dca810SJohn Snow             cnt = bdrv_get_dirty_count(s->dirty_bitmap);
1075893f7ebaSPaolo Bonzini         }
1076893f7ebaSPaolo Bonzini 
1077893f7ebaSPaolo Bonzini         if (cnt == 0 && should_complete) {
1078893f7ebaSPaolo Bonzini             /* The dirty bitmap is not updated while operations are pending.
1079893f7ebaSPaolo Bonzini              * If we're about to exit, wait for pending operations before
1080893f7ebaSPaolo Bonzini              * calling bdrv_get_dirty_count(bs), or we may exit while the
1081893f7ebaSPaolo Bonzini              * source has dirty data to copy!
1082893f7ebaSPaolo Bonzini              *
1083893f7ebaSPaolo Bonzini              * Note that I/O can be submitted by the guest while
10849a0cec66SPaolo Bonzini              * mirror_populate runs, so pause it now.  Before deciding
10859a0cec66SPaolo Bonzini              * whether to switch to target check one last time if I/O has
10869a0cec66SPaolo Bonzini              * come in the meanwhile, and if not flush the data to disk.
1087893f7ebaSPaolo Bonzini              */
10889a46dba7SEric Blake             trace_mirror_before_drain(s, cnt);
10899a0cec66SPaolo Bonzini 
10905e771752SSergio Lopez             s->in_drain = true;
10919a0cec66SPaolo Bonzini             bdrv_drained_begin(bs);
1092d69a879bSHanna Reitz 
1093d69a879bSHanna Reitz             /* Must be zero because we are drained */
1094d69a879bSHanna Reitz             assert(s->in_active_write_counter == 0);
1095d69a879bSHanna Reitz 
109620dca810SJohn Snow             cnt = bdrv_get_dirty_count(s->dirty_bitmap);
1097bdffb31dSPaolo Bonzini             if (cnt > 0 || mirror_flush(s) < 0) {
10989a0cec66SPaolo Bonzini                 bdrv_drained_end(bs);
10995e771752SSergio Lopez                 s->in_drain = false;
11009a0cec66SPaolo Bonzini                 continue;
11019a0cec66SPaolo Bonzini             }
11029a0cec66SPaolo Bonzini 
11039a0cec66SPaolo Bonzini             /* The two disks are in sync.  Exit and report successful
11049a0cec66SPaolo Bonzini              * completion.
11059a0cec66SPaolo Bonzini              */
11069a0cec66SPaolo Bonzini             assert(QLIST_EMPTY(&bs->tracked_requests));
11079a0cec66SPaolo Bonzini             need_drain = false;
11089a0cec66SPaolo Bonzini             break;
1109893f7ebaSPaolo Bonzini         }
1110893f7ebaSPaolo Bonzini 
111144716224SHanna Reitz         if (job_is_ready(&s->common.job) && !should_complete) {
1112*018e5987SKevin Wolf             if (s->in_flight == 0 && cnt == 0) {
111344716224SHanna Reitz                 trace_mirror_before_sleep(s, cnt, job_is_ready(&s->common.job),
1114*018e5987SKevin Wolf                                           BLOCK_JOB_SLICE_TIME);
1115*018e5987SKevin Wolf                 job_sleep_ns(&s->common.job, BLOCK_JOB_SLICE_TIME);
1116*018e5987SKevin Wolf             }
1117*018e5987SKevin Wolf         } else {
1118*018e5987SKevin Wolf             block_job_ratelimit_sleep(&s->common);
1119*018e5987SKevin Wolf         }
112049efb1f5SDenis V. Lunev         s->last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
1121893f7ebaSPaolo Bonzini     }
1122893f7ebaSPaolo Bonzini 
1123893f7ebaSPaolo Bonzini immediate_exit:
1124bd48bde8SPaolo Bonzini     if (s->in_flight > 0) {
1125bd48bde8SPaolo Bonzini         /* We get here only if something went wrong.  Either the job failed,
1126bd48bde8SPaolo Bonzini          * or it was cancelled prematurely so that we do not guarantee that
1127bd48bde8SPaolo Bonzini          * the target is a copy of the source.
1128bd48bde8SPaolo Bonzini          */
112908b83bffSHanna Reitz         assert(ret < 0 || job_is_cancelled(&s->common.job));
11309a0cec66SPaolo Bonzini         assert(need_drain);
1131bae8196dSPaolo Bonzini         mirror_wait_for_all_io(s);
1132bd48bde8SPaolo Bonzini     }
1133bd48bde8SPaolo Bonzini 
1134bd48bde8SPaolo Bonzini     assert(s->in_flight == 0);
11357191bf31SMarkus Armbruster     qemu_vfree(s->buf);
1136b812f671SPaolo Bonzini     g_free(s->cow_bitmap);
1137402a4741SPaolo Bonzini     g_free(s->in_flight_bitmap);
1138dc162c8eSFam Zheng     bdrv_dirty_iter_free(s->dbi);
11395a7e7a0bSStefan Hajnoczi 
11409a0cec66SPaolo Bonzini     if (need_drain) {
11415e771752SSergio Lopez         s->in_drain = true;
1142e253f4b8SKevin Wolf         bdrv_drained_begin(bs);
11439a0cec66SPaolo Bonzini     }
1144f67432a2SJohn Snow 
1145f67432a2SJohn Snow     return ret;
1146893f7ebaSPaolo Bonzini }
1147893f7ebaSPaolo Bonzini 
11483453d972SKevin Wolf static void mirror_complete(Job *job, Error **errp)
1149d63ffd87SPaolo Bonzini {
11503453d972SKevin Wolf     MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
1151274fcceeSMax Reitz 
115244716224SHanna Reitz     if (!job_is_ready(job)) {
11539df229c3SAlberto Garcia         error_setg(errp, "The active block job '%s' cannot be completed",
11543453d972SKevin Wolf                    job->id);
1155d63ffd87SPaolo Bonzini         return;
1156d63ffd87SPaolo Bonzini     }
1157d63ffd87SPaolo Bonzini 
115815d67298SChanglong Xie     /* block all operations on to_replace bs */
115909158f00SBenoît Canet     if (s->replaces) {
11605a7e7a0bSStefan Hajnoczi         AioContext *replace_aio_context;
11615a7e7a0bSStefan Hajnoczi 
1162e12f3784SWen Congyang         s->to_replace = bdrv_find_node(s->replaces);
116309158f00SBenoît Canet         if (!s->to_replace) {
1164e12f3784SWen Congyang             error_setg(errp, "Node name '%s' not found", s->replaces);
116509158f00SBenoît Canet             return;
116609158f00SBenoît Canet         }
116709158f00SBenoît Canet 
11685a7e7a0bSStefan Hajnoczi         replace_aio_context = bdrv_get_aio_context(s->to_replace);
11695a7e7a0bSStefan Hajnoczi         aio_context_acquire(replace_aio_context);
11705a7e7a0bSStefan Hajnoczi 
117164631f36SVladimir Sementsov-Ogievskiy         /* TODO Translate this into child freeze system. */
117209158f00SBenoît Canet         error_setg(&s->replace_blocker,
117309158f00SBenoît Canet                    "block device is in use by block-job-complete");
117409158f00SBenoît Canet         bdrv_op_block_all(s->to_replace, s->replace_blocker);
117509158f00SBenoît Canet         bdrv_ref(s->to_replace);
11765a7e7a0bSStefan Hajnoczi 
11775a7e7a0bSStefan Hajnoczi         aio_context_release(replace_aio_context);
117809158f00SBenoît Canet     }
117909158f00SBenoît Canet 
1180d63ffd87SPaolo Bonzini     s->should_complete = true;
118100769414SMax Reitz 
118200769414SMax Reitz     /* If the job is paused, it will be re-entered when it is resumed */
1183279ac06eSEmanuele Giuseppe Esposito     WITH_JOB_LOCK_GUARD() {
118400769414SMax Reitz         if (!job->paused) {
1185279ac06eSEmanuele Giuseppe Esposito             job_enter_cond_locked(job, NULL);
1186279ac06eSEmanuele Giuseppe Esposito         }
1187d63ffd87SPaolo Bonzini     }
118800769414SMax Reitz }
1189d63ffd87SPaolo Bonzini 
1190537c3d4fSStefan Hajnoczi static void coroutine_fn mirror_pause(Job *job)
1191565ac01fSStefan Hajnoczi {
1192da01ff7fSKevin Wolf     MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
1193565ac01fSStefan Hajnoczi 
1194bae8196dSPaolo Bonzini     mirror_wait_for_all_io(s);
1195565ac01fSStefan Hajnoczi }
1196565ac01fSStefan Hajnoczi 
119789bd0305SKevin Wolf static bool mirror_drained_poll(BlockJob *job)
119889bd0305SKevin Wolf {
119989bd0305SKevin Wolf     MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
12005e771752SSergio Lopez 
12015e771752SSergio Lopez     /* If the job isn't paused nor cancelled, we can't be sure that it won't
12025e771752SSergio Lopez      * issue more requests. We make an exception if we've reached this point
12035e771752SSergio Lopez      * from one of our own drain sections, to avoid a deadlock waiting for
12045e771752SSergio Lopez      * ourselves.
12055e771752SSergio Lopez      */
1206279ac06eSEmanuele Giuseppe Esposito     WITH_JOB_LOCK_GUARD() {
1207279ac06eSEmanuele Giuseppe Esposito         if (!s->common.job.paused && !job_is_cancelled_locked(&job->job)
1208279ac06eSEmanuele Giuseppe Esposito             && !s->in_drain) {
12095e771752SSergio Lopez             return true;
12105e771752SSergio Lopez         }
1211279ac06eSEmanuele Giuseppe Esposito     }
12125e771752SSergio Lopez 
121389bd0305SKevin Wolf     return !!s->in_flight;
121489bd0305SKevin Wolf }
121589bd0305SKevin Wolf 
121673895f38SHanna Reitz static bool mirror_cancel(Job *job, bool force)
1217521ff8b7SVladimir Sementsov-Ogievskiy {
1218521ff8b7SVladimir Sementsov-Ogievskiy     MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
1219521ff8b7SVladimir Sementsov-Ogievskiy     BlockDriverState *target = blk_bs(s->target);
1220521ff8b7SVladimir Sementsov-Ogievskiy 
122173895f38SHanna Reitz     /*
122273895f38SHanna Reitz      * Before the job is READY, we treat any cancellation like a
122373895f38SHanna Reitz      * force-cancellation.
122473895f38SHanna Reitz      */
122573895f38SHanna Reitz     force = force || !job_is_ready(job);
122673895f38SHanna Reitz 
122773895f38SHanna Reitz     if (force) {
1228521ff8b7SVladimir Sementsov-Ogievskiy         bdrv_cancel_in_flight(target);
1229521ff8b7SVladimir Sementsov-Ogievskiy     }
123073895f38SHanna Reitz     return force;
123173895f38SHanna Reitz }
123273895f38SHanna Reitz 
123373895f38SHanna Reitz static bool commit_active_cancel(Job *job, bool force)
123473895f38SHanna Reitz {
123573895f38SHanna Reitz     /* Same as above in mirror_cancel() */
123673895f38SHanna Reitz     return force || !job_is_ready(job);
12379c785cd7SVladimir Sementsov-Ogievskiy }
1238521ff8b7SVladimir Sementsov-Ogievskiy 
12393fc4b10aSFam Zheng static const BlockJobDriver mirror_job_driver = {
124033e9e9bdSKevin Wolf     .job_driver = {
1241893f7ebaSPaolo Bonzini         .instance_size          = sizeof(MirrorBlockJob),
12428e4c8700SKevin Wolf         .job_type               = JOB_TYPE_MIRROR,
124380fa2c75SKevin Wolf         .free                   = block_job_free,
1244b15de828SKevin Wolf         .user_resume            = block_job_user_resume,
1245f67432a2SJohn Snow         .run                    = mirror_run,
1246737efc1eSJohn Snow         .prepare                = mirror_prepare,
1247737efc1eSJohn Snow         .abort                  = mirror_abort,
1248565ac01fSStefan Hajnoczi         .pause                  = mirror_pause,
1249da01ff7fSKevin Wolf         .complete               = mirror_complete,
1250521ff8b7SVladimir Sementsov-Ogievskiy         .cancel                 = mirror_cancel,
12513453d972SKevin Wolf     },
125289bd0305SKevin Wolf     .drained_poll           = mirror_drained_poll,
1253893f7ebaSPaolo Bonzini };
1254893f7ebaSPaolo Bonzini 
125503544a6eSFam Zheng static const BlockJobDriver commit_active_job_driver = {
125633e9e9bdSKevin Wolf     .job_driver = {
125703544a6eSFam Zheng         .instance_size          = sizeof(MirrorBlockJob),
12588e4c8700SKevin Wolf         .job_type               = JOB_TYPE_COMMIT,
125980fa2c75SKevin Wolf         .free                   = block_job_free,
1260b15de828SKevin Wolf         .user_resume            = block_job_user_resume,
1261f67432a2SJohn Snow         .run                    = mirror_run,
1262737efc1eSJohn Snow         .prepare                = mirror_prepare,
1263737efc1eSJohn Snow         .abort                  = mirror_abort,
1264565ac01fSStefan Hajnoczi         .pause                  = mirror_pause,
1265da01ff7fSKevin Wolf         .complete               = mirror_complete,
126673895f38SHanna Reitz         .cancel                 = commit_active_cancel,
12673453d972SKevin Wolf     },
126889bd0305SKevin Wolf     .drained_poll           = mirror_drained_poll,
126903544a6eSFam Zheng };
127003544a6eSFam Zheng 
1271537c3d4fSStefan Hajnoczi static void coroutine_fn
1272537c3d4fSStefan Hajnoczi do_sync_target_write(MirrorBlockJob *job, MirrorMethod method,
1273d06107adSMax Reitz                      uint64_t offset, uint64_t bytes,
1274d06107adSMax Reitz                      QEMUIOVector *qiov, int flags)
1275d06107adSMax Reitz {
1276d06107adSMax Reitz     int ret;
1277dbdf699cSVladimir Sementsov-Ogievskiy     size_t qiov_offset = 0;
1278dbdf699cSVladimir Sementsov-Ogievskiy     int64_t bitmap_offset, bitmap_end;
1279d06107adSMax Reitz 
1280dbdf699cSVladimir Sementsov-Ogievskiy     if (!QEMU_IS_ALIGNED(offset, job->granularity) &&
1281dbdf699cSVladimir Sementsov-Ogievskiy         bdrv_dirty_bitmap_get(job->dirty_bitmap, offset))
1282dbdf699cSVladimir Sementsov-Ogievskiy     {
1283dbdf699cSVladimir Sementsov-Ogievskiy             /*
1284dbdf699cSVladimir Sementsov-Ogievskiy              * Dirty unaligned padding: ignore it.
1285dbdf699cSVladimir Sementsov-Ogievskiy              *
1286dbdf699cSVladimir Sementsov-Ogievskiy              * Reasoning:
1287dbdf699cSVladimir Sementsov-Ogievskiy              * 1. If we copy it, we can't reset corresponding bit in
1288dbdf699cSVladimir Sementsov-Ogievskiy              *    dirty_bitmap as there may be some "dirty" bytes still not
1289dbdf699cSVladimir Sementsov-Ogievskiy              *    copied.
1290dbdf699cSVladimir Sementsov-Ogievskiy              * 2. It's already dirty, so skipping it we don't diverge mirror
1291dbdf699cSVladimir Sementsov-Ogievskiy              *    progress.
1292dbdf699cSVladimir Sementsov-Ogievskiy              *
1293dbdf699cSVladimir Sementsov-Ogievskiy              * Note, that because of this, guest write may have no contribution
1294dbdf699cSVladimir Sementsov-Ogievskiy              * into mirror converge, but that's not bad, as we have background
1295dbdf699cSVladimir Sementsov-Ogievskiy              * process of mirroring. If under some bad circumstances (high guest
1296dbdf699cSVladimir Sementsov-Ogievskiy              * IO load) background process starve, we will not converge anyway,
1297dbdf699cSVladimir Sementsov-Ogievskiy              * even if each write will contribute, as guest is not guaranteed to
1298dbdf699cSVladimir Sementsov-Ogievskiy              * rewrite the whole disk.
1299dbdf699cSVladimir Sementsov-Ogievskiy              */
1300dbdf699cSVladimir Sementsov-Ogievskiy             qiov_offset = QEMU_ALIGN_UP(offset, job->granularity) - offset;
1301dbdf699cSVladimir Sementsov-Ogievskiy             if (bytes <= qiov_offset) {
1302dbdf699cSVladimir Sementsov-Ogievskiy                 /* nothing to do after shrink */
1303dbdf699cSVladimir Sementsov-Ogievskiy                 return;
1304dbdf699cSVladimir Sementsov-Ogievskiy             }
1305dbdf699cSVladimir Sementsov-Ogievskiy             offset += qiov_offset;
1306dbdf699cSVladimir Sementsov-Ogievskiy             bytes -= qiov_offset;
1307dbdf699cSVladimir Sementsov-Ogievskiy     }
1308dbdf699cSVladimir Sementsov-Ogievskiy 
1309dbdf699cSVladimir Sementsov-Ogievskiy     if (!QEMU_IS_ALIGNED(offset + bytes, job->granularity) &&
1310dbdf699cSVladimir Sementsov-Ogievskiy         bdrv_dirty_bitmap_get(job->dirty_bitmap, offset + bytes - 1))
1311dbdf699cSVladimir Sementsov-Ogievskiy     {
1312dbdf699cSVladimir Sementsov-Ogievskiy         uint64_t tail = (offset + bytes) % job->granularity;
1313dbdf699cSVladimir Sementsov-Ogievskiy 
1314dbdf699cSVladimir Sementsov-Ogievskiy         if (bytes <= tail) {
1315dbdf699cSVladimir Sementsov-Ogievskiy             /* nothing to do after shrink */
1316dbdf699cSVladimir Sementsov-Ogievskiy             return;
1317dbdf699cSVladimir Sementsov-Ogievskiy         }
1318dbdf699cSVladimir Sementsov-Ogievskiy         bytes -= tail;
1319dbdf699cSVladimir Sementsov-Ogievskiy     }
1320dbdf699cSVladimir Sementsov-Ogievskiy 
1321dbdf699cSVladimir Sementsov-Ogievskiy     /*
1322dbdf699cSVladimir Sementsov-Ogievskiy      * Tails are either clean or shrunk, so for bitmap resetting
1323dbdf699cSVladimir Sementsov-Ogievskiy      * we safely align the range down.
1324dbdf699cSVladimir Sementsov-Ogievskiy      */
1325dbdf699cSVladimir Sementsov-Ogievskiy     bitmap_offset = QEMU_ALIGN_UP(offset, job->granularity);
1326dbdf699cSVladimir Sementsov-Ogievskiy     bitmap_end = QEMU_ALIGN_DOWN(offset + bytes, job->granularity);
1327dbdf699cSVladimir Sementsov-Ogievskiy     if (bitmap_offset < bitmap_end) {
1328dbdf699cSVladimir Sementsov-Ogievskiy         bdrv_reset_dirty_bitmap(job->dirty_bitmap, bitmap_offset,
1329dbdf699cSVladimir Sementsov-Ogievskiy                                 bitmap_end - bitmap_offset);
1330dbdf699cSVladimir Sementsov-Ogievskiy     }
1331d06107adSMax Reitz 
13325c511ac3SVladimir Sementsov-Ogievskiy     job_progress_increase_remaining(&job->common.job, bytes);
1333d69a879bSHanna Reitz     job->active_write_bytes_in_flight += bytes;
1334d06107adSMax Reitz 
1335d06107adSMax Reitz     switch (method) {
1336d06107adSMax Reitz     case MIRROR_METHOD_COPY:
1337dbdf699cSVladimir Sementsov-Ogievskiy         ret = blk_co_pwritev_part(job->target, offset, bytes,
1338dbdf699cSVladimir Sementsov-Ogievskiy                                   qiov, qiov_offset, flags);
1339d06107adSMax Reitz         break;
1340d06107adSMax Reitz 
1341d06107adSMax Reitz     case MIRROR_METHOD_ZERO:
1342d06107adSMax Reitz         assert(!qiov);
13435c511ac3SVladimir Sementsov-Ogievskiy         ret = blk_co_pwrite_zeroes(job->target, offset, bytes, flags);
1344d06107adSMax Reitz         break;
1345d06107adSMax Reitz 
1346d06107adSMax Reitz     case MIRROR_METHOD_DISCARD:
1347d06107adSMax Reitz         assert(!qiov);
13485c511ac3SVladimir Sementsov-Ogievskiy         ret = blk_co_pdiscard(job->target, offset, bytes);
1349d06107adSMax Reitz         break;
1350d06107adSMax Reitz 
1351d06107adSMax Reitz     default:
1352d06107adSMax Reitz         abort();
1353d06107adSMax Reitz     }
1354d06107adSMax Reitz 
1355d69a879bSHanna Reitz     job->active_write_bytes_in_flight -= bytes;
1356d06107adSMax Reitz     if (ret >= 0) {
13575c511ac3SVladimir Sementsov-Ogievskiy         job_progress_update(&job->common.job, bytes);
1358d06107adSMax Reitz     } else {
1359d06107adSMax Reitz         BlockErrorAction action;
1360d06107adSMax Reitz 
1361dbdf699cSVladimir Sementsov-Ogievskiy         /*
1362dbdf699cSVladimir Sementsov-Ogievskiy          * We failed, so we should mark dirty the whole area, aligned up.
1363dbdf699cSVladimir Sementsov-Ogievskiy          * Note that we don't care about shrunk tails if any: they were dirty
1364dbdf699cSVladimir Sementsov-Ogievskiy          * at function start, and they must be still dirty, as we've locked
1365dbdf699cSVladimir Sementsov-Ogievskiy          * the region for in-flight op.
1366dbdf699cSVladimir Sementsov-Ogievskiy          */
1367dbdf699cSVladimir Sementsov-Ogievskiy         bitmap_offset = QEMU_ALIGN_DOWN(offset, job->granularity);
1368dbdf699cSVladimir Sementsov-Ogievskiy         bitmap_end = QEMU_ALIGN_UP(offset + bytes, job->granularity);
1369dbdf699cSVladimir Sementsov-Ogievskiy         bdrv_set_dirty_bitmap(job->dirty_bitmap, bitmap_offset,
1370dbdf699cSVladimir Sementsov-Ogievskiy                               bitmap_end - bitmap_offset);
1371d06107adSMax Reitz         job->actively_synced = false;
1372d06107adSMax Reitz 
1373d06107adSMax Reitz         action = mirror_error_action(job, false, -ret);
1374d06107adSMax Reitz         if (action == BLOCK_ERROR_ACTION_REPORT) {
1375d06107adSMax Reitz             if (!job->ret) {
1376d06107adSMax Reitz                 job->ret = ret;
1377d06107adSMax Reitz             }
1378d06107adSMax Reitz         }
1379d06107adSMax Reitz     }
1380d06107adSMax Reitz }
1381d06107adSMax Reitz 
1382d06107adSMax Reitz static MirrorOp *coroutine_fn active_write_prepare(MirrorBlockJob *s,
1383d06107adSMax Reitz                                                    uint64_t offset,
1384d06107adSMax Reitz                                                    uint64_t bytes)
1385d06107adSMax Reitz {
1386d06107adSMax Reitz     MirrorOp *op;
1387d06107adSMax Reitz     uint64_t start_chunk = offset / s->granularity;
1388d06107adSMax Reitz     uint64_t end_chunk = DIV_ROUND_UP(offset + bytes, s->granularity);
1389d06107adSMax Reitz 
1390d06107adSMax Reitz     op = g_new(MirrorOp, 1);
1391d06107adSMax Reitz     *op = (MirrorOp){
1392d06107adSMax Reitz         .s                  = s,
1393d06107adSMax Reitz         .offset             = offset,
1394d06107adSMax Reitz         .bytes              = bytes,
1395d06107adSMax Reitz         .is_active_write    = true,
1396ce8cabbdSKevin Wolf         .is_in_flight       = true,
1397ead3f1bfSVladimir Sementsov-Ogievskiy         .co                 = qemu_coroutine_self(),
1398d06107adSMax Reitz     };
1399d06107adSMax Reitz     qemu_co_queue_init(&op->waiting_requests);
1400d06107adSMax Reitz     QTAILQ_INSERT_TAIL(&s->ops_in_flight, op, next);
1401d06107adSMax Reitz 
1402d06107adSMax Reitz     s->in_active_write_counter++;
1403d06107adSMax Reitz 
1404d69a879bSHanna Reitz     /*
1405d69a879bSHanna Reitz      * Wait for concurrent requests affecting the area.  If there are already
1406d69a879bSHanna Reitz      * running requests that are copying off now-to-be stale data in the area,
1407d69a879bSHanna Reitz      * we must wait for them to finish before we begin writing fresh data to the
1408d69a879bSHanna Reitz      * target so that the write operations appear in the correct order.
1409d69a879bSHanna Reitz      * Note that background requests (see mirror_iteration()) in contrast only
1410d69a879bSHanna Reitz      * wait for conflicting requests at the start of the dirty area, and then
1411d69a879bSHanna Reitz      * (based on the in_flight_bitmap) truncate the area to copy so it will not
1412d69a879bSHanna Reitz      * conflict with any requests beyond that.  For active writes, however, we
1413d69a879bSHanna Reitz      * cannot truncate that area.  The request from our parent must be blocked
1414d69a879bSHanna Reitz      * until the area is copied in full.  Therefore, we must wait for the whole
1415d69a879bSHanna Reitz      * area to become free of concurrent requests.
1416d69a879bSHanna Reitz      */
1417d06107adSMax Reitz     mirror_wait_on_conflicts(op, s, offset, bytes);
1418d06107adSMax Reitz 
1419d06107adSMax Reitz     bitmap_set(s->in_flight_bitmap, start_chunk, end_chunk - start_chunk);
1420d06107adSMax Reitz 
1421d06107adSMax Reitz     return op;
1422d06107adSMax Reitz }
1423d06107adSMax Reitz 
14249c93652dSKevin Wolf static void coroutine_fn GRAPH_RDLOCK active_write_settle(MirrorOp *op)
1425d06107adSMax Reitz {
1426d06107adSMax Reitz     uint64_t start_chunk = op->offset / op->s->granularity;
1427d06107adSMax Reitz     uint64_t end_chunk = DIV_ROUND_UP(op->offset + op->bytes,
1428d06107adSMax Reitz                                       op->s->granularity);
1429d06107adSMax Reitz 
1430d06107adSMax Reitz     if (!--op->s->in_active_write_counter && op->s->actively_synced) {
1431d06107adSMax Reitz         BdrvChild *source = op->s->mirror_top_bs->backing;
1432d06107adSMax Reitz 
1433d06107adSMax Reitz         if (QLIST_FIRST(&source->bs->parents) == source &&
1434d06107adSMax Reitz             QLIST_NEXT(source, next_parent) == NULL)
1435d06107adSMax Reitz         {
1436d06107adSMax Reitz             /* Assert that we are back in sync once all active write
1437d06107adSMax Reitz              * operations are settled.
1438d06107adSMax Reitz              * Note that we can only assert this if the mirror node
1439d06107adSMax Reitz              * is the source node's only parent. */
1440d06107adSMax Reitz             assert(!bdrv_get_dirty_count(op->s->dirty_bitmap));
1441d06107adSMax Reitz         }
1442d06107adSMax Reitz     }
1443d06107adSMax Reitz     bitmap_clear(op->s->in_flight_bitmap, start_chunk, end_chunk - start_chunk);
1444d06107adSMax Reitz     QTAILQ_REMOVE(&op->s->ops_in_flight, op, next);
1445d06107adSMax Reitz     qemu_co_queue_restart_all(&op->waiting_requests);
1446d06107adSMax Reitz     g_free(op);
1447d06107adSMax Reitz }
1448d06107adSMax Reitz 
1449b9b10c35SKevin Wolf static int coroutine_fn GRAPH_RDLOCK
1450b9b10c35SKevin Wolf bdrv_mirror_top_preadv(BlockDriverState *bs, int64_t offset, int64_t bytes,
1451b9b10c35SKevin Wolf                        QEMUIOVector *qiov, BdrvRequestFlags flags)
14524ef85a9cSKevin Wolf {
14534ef85a9cSKevin Wolf     return bdrv_co_preadv(bs->backing, offset, bytes, qiov, flags);
14544ef85a9cSKevin Wolf }
14554ef85a9cSKevin Wolf 
14569a5a1c62SEmanuele Giuseppe Esposito static int coroutine_fn GRAPH_RDLOCK
14579a5a1c62SEmanuele Giuseppe Esposito bdrv_mirror_top_do_write(BlockDriverState *bs, MirrorMethod method,
14589a5a1c62SEmanuele Giuseppe Esposito                          uint64_t offset, uint64_t bytes, QEMUIOVector *qiov,
1459d06107adSMax Reitz                          int flags)
1460d06107adSMax Reitz {
1461d06107adSMax Reitz     MirrorOp *op = NULL;
1462d06107adSMax Reitz     MirrorBDSOpaque *s = bs->opaque;
1463d06107adSMax Reitz     int ret = 0;
1464da93d5c8SHanna Reitz     bool copy_to_target = false;
1465d06107adSMax Reitz 
1466da93d5c8SHanna Reitz     if (s->job) {
1467d06107adSMax Reitz         copy_to_target = s->job->ret >= 0 &&
14689b230ef9SHanna Reitz                          !job_is_cancelled(&s->job->common.job) &&
1469d06107adSMax Reitz                          s->job->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING;
1470da93d5c8SHanna Reitz     }
1471d06107adSMax Reitz 
1472d06107adSMax Reitz     if (copy_to_target) {
1473d06107adSMax Reitz         op = active_write_prepare(s->job, offset, bytes);
1474d06107adSMax Reitz     }
1475d06107adSMax Reitz 
1476d06107adSMax Reitz     switch (method) {
1477d06107adSMax Reitz     case MIRROR_METHOD_COPY:
1478d06107adSMax Reitz         ret = bdrv_co_pwritev(bs->backing, offset, bytes, qiov, flags);
1479d06107adSMax Reitz         break;
1480d06107adSMax Reitz 
1481d06107adSMax Reitz     case MIRROR_METHOD_ZERO:
1482d06107adSMax Reitz         ret = bdrv_co_pwrite_zeroes(bs->backing, offset, bytes, flags);
1483d06107adSMax Reitz         break;
1484d06107adSMax Reitz 
1485d06107adSMax Reitz     case MIRROR_METHOD_DISCARD:
14860b9fd3f4SFam Zheng         ret = bdrv_co_pdiscard(bs->backing, offset, bytes);
1487d06107adSMax Reitz         break;
1488d06107adSMax Reitz 
1489d06107adSMax Reitz     default:
1490d06107adSMax Reitz         abort();
1491d06107adSMax Reitz     }
1492d06107adSMax Reitz 
1493d06107adSMax Reitz     if (ret < 0) {
1494d06107adSMax Reitz         goto out;
1495d06107adSMax Reitz     }
1496d06107adSMax Reitz 
1497d06107adSMax Reitz     if (copy_to_target) {
1498d06107adSMax Reitz         do_sync_target_write(s->job, method, offset, bytes, qiov, flags);
1499d06107adSMax Reitz     }
1500d06107adSMax Reitz 
1501d06107adSMax Reitz out:
1502d06107adSMax Reitz     if (copy_to_target) {
1503d06107adSMax Reitz         active_write_settle(op);
1504d06107adSMax Reitz     }
1505d06107adSMax Reitz     return ret;
1506d06107adSMax Reitz }
1507d06107adSMax Reitz 
1508b9b10c35SKevin Wolf static int coroutine_fn GRAPH_RDLOCK
1509b9b10c35SKevin Wolf bdrv_mirror_top_pwritev(BlockDriverState *bs, int64_t offset, int64_t bytes,
1510b9b10c35SKevin Wolf                         QEMUIOVector *qiov, BdrvRequestFlags flags)
15114ef85a9cSKevin Wolf {
1512d06107adSMax Reitz     MirrorBDSOpaque *s = bs->opaque;
1513d06107adSMax Reitz     QEMUIOVector bounce_qiov;
1514d06107adSMax Reitz     void *bounce_buf;
1515d06107adSMax Reitz     int ret = 0;
1516da93d5c8SHanna Reitz     bool copy_to_target = false;
1517d06107adSMax Reitz 
1518da93d5c8SHanna Reitz     if (s->job) {
1519d06107adSMax Reitz         copy_to_target = s->job->ret >= 0 &&
15209b230ef9SHanna Reitz                          !job_is_cancelled(&s->job->common.job) &&
1521d06107adSMax Reitz                          s->job->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING;
1522da93d5c8SHanna Reitz     }
1523d06107adSMax Reitz 
1524d06107adSMax Reitz     if (copy_to_target) {
1525d06107adSMax Reitz         /* The guest might concurrently modify the data to write; but
1526d06107adSMax Reitz          * the data on source and destination must match, so we have
1527d06107adSMax Reitz          * to use a bounce buffer if we are going to write to the
1528d06107adSMax Reitz          * target now. */
1529d06107adSMax Reitz         bounce_buf = qemu_blockalign(bs, bytes);
1530d06107adSMax Reitz         iov_to_buf_full(qiov->iov, qiov->niov, 0, bounce_buf, bytes);
1531d06107adSMax Reitz 
1532d06107adSMax Reitz         qemu_iovec_init(&bounce_qiov, 1);
1533d06107adSMax Reitz         qemu_iovec_add(&bounce_qiov, bounce_buf, bytes);
1534d06107adSMax Reitz         qiov = &bounce_qiov;
1535e8b65355SStefan Hajnoczi 
1536e8b65355SStefan Hajnoczi         flags &= ~BDRV_REQ_REGISTERED_BUF;
1537d06107adSMax Reitz     }
1538d06107adSMax Reitz 
1539d06107adSMax Reitz     ret = bdrv_mirror_top_do_write(bs, MIRROR_METHOD_COPY, offset, bytes, qiov,
1540d06107adSMax Reitz                                    flags);
1541d06107adSMax Reitz 
1542d06107adSMax Reitz     if (copy_to_target) {
1543d06107adSMax Reitz         qemu_iovec_destroy(&bounce_qiov);
1544d06107adSMax Reitz         qemu_vfree(bounce_buf);
1545d06107adSMax Reitz     }
1546d06107adSMax Reitz 
1547d06107adSMax Reitz     return ret;
15484ef85a9cSKevin Wolf }
15494ef85a9cSKevin Wolf 
155088095349SEmanuele Giuseppe Esposito static int coroutine_fn GRAPH_RDLOCK bdrv_mirror_top_flush(BlockDriverState *bs)
15514ef85a9cSKevin Wolf {
1552ce960aa9SVladimir Sementsov-Ogievskiy     if (bs->backing == NULL) {
1553ce960aa9SVladimir Sementsov-Ogievskiy         /* we can be here after failed bdrv_append in mirror_start_job */
1554ce960aa9SVladimir Sementsov-Ogievskiy         return 0;
1555ce960aa9SVladimir Sementsov-Ogievskiy     }
15564ef85a9cSKevin Wolf     return bdrv_co_flush(bs->backing->bs);
15574ef85a9cSKevin Wolf }
15584ef85a9cSKevin Wolf 
1559abaf8b75SKevin Wolf static int coroutine_fn GRAPH_RDLOCK
1560abaf8b75SKevin Wolf bdrv_mirror_top_pwrite_zeroes(BlockDriverState *bs, int64_t offset,
1561abaf8b75SKevin Wolf                               int64_t bytes, BdrvRequestFlags flags)
15624ef85a9cSKevin Wolf {
1563d06107adSMax Reitz     return bdrv_mirror_top_do_write(bs, MIRROR_METHOD_ZERO, offset, bytes, NULL,
1564d06107adSMax Reitz                                     flags);
15654ef85a9cSKevin Wolf }
15664ef85a9cSKevin Wolf 
15679a5a1c62SEmanuele Giuseppe Esposito static int coroutine_fn GRAPH_RDLOCK
15689a5a1c62SEmanuele Giuseppe Esposito bdrv_mirror_top_pdiscard(BlockDriverState *bs, int64_t offset, int64_t bytes)
15694ef85a9cSKevin Wolf {
1570d06107adSMax Reitz     return bdrv_mirror_top_do_write(bs, MIRROR_METHOD_DISCARD, offset, bytes,
1571d06107adSMax Reitz                                     NULL, 0);
15724ef85a9cSKevin Wolf }
15734ef85a9cSKevin Wolf 
1574998b3a1eSMax Reitz static void bdrv_mirror_top_refresh_filename(BlockDriverState *bs)
1575fd4a6493SKevin Wolf {
157618775ff3SVladimir Sementsov-Ogievskiy     if (bs->backing == NULL) {
157718775ff3SVladimir Sementsov-Ogievskiy         /* we can be here after failed bdrv_attach_child in
157818775ff3SVladimir Sementsov-Ogievskiy          * bdrv_set_backing_hd */
157918775ff3SVladimir Sementsov-Ogievskiy         return;
158018775ff3SVladimir Sementsov-Ogievskiy     }
1581fd4a6493SKevin Wolf     pstrcpy(bs->exact_filename, sizeof(bs->exact_filename),
1582fd4a6493SKevin Wolf             bs->backing->bs->filename);
1583fd4a6493SKevin Wolf }
1584fd4a6493SKevin Wolf 
15854ef85a9cSKevin Wolf static void bdrv_mirror_top_child_perm(BlockDriverState *bs, BdrvChild *c,
1586bf8e925eSMax Reitz                                        BdrvChildRole role,
1587e0995dc3SKevin Wolf                                        BlockReopenQueue *reopen_queue,
15884ef85a9cSKevin Wolf                                        uint64_t perm, uint64_t shared,
15894ef85a9cSKevin Wolf                                        uint64_t *nperm, uint64_t *nshared)
15904ef85a9cSKevin Wolf {
1591f94dc3b4SMax Reitz     MirrorBDSOpaque *s = bs->opaque;
1592f94dc3b4SMax Reitz 
1593f94dc3b4SMax Reitz     if (s->stop) {
1594f94dc3b4SMax Reitz         /*
1595f94dc3b4SMax Reitz          * If the job is to be stopped, we do not need to forward
1596f94dc3b4SMax Reitz          * anything to the real image.
1597f94dc3b4SMax Reitz          */
1598f94dc3b4SMax Reitz         *nperm = 0;
1599f94dc3b4SMax Reitz         *nshared = BLK_PERM_ALL;
1600f94dc3b4SMax Reitz         return;
1601f94dc3b4SMax Reitz     }
1602f94dc3b4SMax Reitz 
160353431b90SMax Reitz     bdrv_default_perms(bs, c, role, reopen_queue,
160453431b90SMax Reitz                        perm, shared, nperm, nshared);
16054ef85a9cSKevin Wolf 
160653431b90SMax Reitz     if (s->is_commit) {
160753431b90SMax Reitz         /*
160853431b90SMax Reitz          * For commit jobs, we cannot take CONSISTENT_READ, because
160953431b90SMax Reitz          * that permission is unshared for everything above the base
161053431b90SMax Reitz          * node (except for filters on the base node).
161153431b90SMax Reitz          * We also have to force-share the WRITE permission, or
161253431b90SMax Reitz          * otherwise we would block ourselves at the base node (if
161353431b90SMax Reitz          * writes are blocked for a node, they are also blocked for
161453431b90SMax Reitz          * its backing file).
161553431b90SMax Reitz          * (We could also share RESIZE, because it may be needed for
161653431b90SMax Reitz          * the target if its size is less than the top node's; but
161753431b90SMax Reitz          * bdrv_default_perms_for_cow() automatically shares RESIZE
161853431b90SMax Reitz          * for backing nodes if WRITE is shared, so there is no need
161953431b90SMax Reitz          * to do it here.)
162053431b90SMax Reitz          */
162153431b90SMax Reitz         *nperm &= ~BLK_PERM_CONSISTENT_READ;
162253431b90SMax Reitz         *nshared |= BLK_PERM_WRITE;
162353431b90SMax Reitz     }
16244ef85a9cSKevin Wolf }
16254ef85a9cSKevin Wolf 
16264ef85a9cSKevin Wolf /* Dummy node that provides consistent read to its users without requiring it
16274ef85a9cSKevin Wolf  * from its backing file and that allows writes on the backing file chain. */
16284ef85a9cSKevin Wolf static BlockDriver bdrv_mirror_top = {
16294ef85a9cSKevin Wolf     .format_name                = "mirror_top",
16304ef85a9cSKevin Wolf     .bdrv_co_preadv             = bdrv_mirror_top_preadv,
16314ef85a9cSKevin Wolf     .bdrv_co_pwritev            = bdrv_mirror_top_pwritev,
16324ef85a9cSKevin Wolf     .bdrv_co_pwrite_zeroes      = bdrv_mirror_top_pwrite_zeroes,
16334ef85a9cSKevin Wolf     .bdrv_co_pdiscard           = bdrv_mirror_top_pdiscard,
16344ef85a9cSKevin Wolf     .bdrv_co_flush              = bdrv_mirror_top_flush,
1635fd4a6493SKevin Wolf     .bdrv_refresh_filename      = bdrv_mirror_top_refresh_filename,
16364ef85a9cSKevin Wolf     .bdrv_child_perm            = bdrv_mirror_top_child_perm,
16376540fd15SMax Reitz 
16386540fd15SMax Reitz     .is_filter                  = true,
1639046fd84fSVladimir Sementsov-Ogievskiy     .filtered_child_is_backing  = true,
16404ef85a9cSKevin Wolf };
16414ef85a9cSKevin Wolf 
1642cc19f177SVladimir Sementsov-Ogievskiy static BlockJob *mirror_start_job(
1643cc19f177SVladimir Sementsov-Ogievskiy                              const char *job_id, BlockDriverState *bs,
164447970dfbSJohn Snow                              int creation_flags, BlockDriverState *target,
164547970dfbSJohn Snow                              const char *replaces, int64_t speed,
164647970dfbSJohn Snow                              uint32_t granularity, int64_t buf_size,
1647274fcceeSMax Reitz                              BlockMirrorBackingMode backing_mode,
1648cdf3bc93SMax Reitz                              bool zero_target,
164903544a6eSFam Zheng                              BlockdevOnError on_source_error,
1650b952b558SPaolo Bonzini                              BlockdevOnError on_target_error,
16510fc9f8eaSFam Zheng                              bool unmap,
1652097310b5SMarkus Armbruster                              BlockCompletionFunc *cb,
165351ccfa2dSFam Zheng                              void *opaque,
165403544a6eSFam Zheng                              const BlockJobDriver *driver,
1655b49f7eadSWen Congyang                              bool is_none_mode, BlockDriverState *base,
165651ccfa2dSFam Zheng                              bool auto_complete, const char *filter_node_name,
1657481debaaSMax Reitz                              bool is_mirror, MirrorCopyMode copy_mode,
165851ccfa2dSFam Zheng                              Error **errp)
1659893f7ebaSPaolo Bonzini {
1660893f7ebaSPaolo Bonzini     MirrorBlockJob *s;
1661429076e8SMax Reitz     MirrorBDSOpaque *bs_opaque;
16624ef85a9cSKevin Wolf     BlockDriverState *mirror_top_bs;
16634ef85a9cSKevin Wolf     bool target_is_backing;
16643f072a7fSMax Reitz     uint64_t target_perms, target_shared_perms;
1665d7086422SKevin Wolf     int ret;
1666893f7ebaSPaolo Bonzini 
1667eee13dfeSPaolo Bonzini     if (granularity == 0) {
1668341ebc2fSJohn Snow         granularity = bdrv_get_default_bitmap_granularity(target);
1669eee13dfeSPaolo Bonzini     }
1670eee13dfeSPaolo Bonzini 
167131826642SEric Blake     assert(is_power_of_2(granularity));
1672eee13dfeSPaolo Bonzini 
167348ac0a4dSWen Congyang     if (buf_size < 0) {
167448ac0a4dSWen Congyang         error_setg(errp, "Invalid parameter 'buf-size'");
1675cc19f177SVladimir Sementsov-Ogievskiy         return NULL;
167648ac0a4dSWen Congyang     }
167748ac0a4dSWen Congyang 
167848ac0a4dSWen Congyang     if (buf_size == 0) {
167948ac0a4dSWen Congyang         buf_size = DEFAULT_MIRROR_BUF_SIZE;
168048ac0a4dSWen Congyang     }
16815bc361b8SFam Zheng 
16823f072a7fSMax Reitz     if (bdrv_skip_filters(bs) == bdrv_skip_filters(target)) {
168386fae10cSKevin Wolf         error_setg(errp, "Can't mirror node into itself");
1684cc19f177SVladimir Sementsov-Ogievskiy         return NULL;
168586fae10cSKevin Wolf     }
168686fae10cSKevin Wolf 
168753431b90SMax Reitz     target_is_backing = bdrv_chain_contains(bs, target);
168853431b90SMax Reitz 
16894ef85a9cSKevin Wolf     /* In the case of active commit, add dummy driver to provide consistent
16904ef85a9cSKevin Wolf      * reads on the top, while disabling it in the intermediate nodes, and make
16914ef85a9cSKevin Wolf      * the backing chain writable. */
16926cdbceb1SKevin Wolf     mirror_top_bs = bdrv_new_open_driver(&bdrv_mirror_top, filter_node_name,
16936cdbceb1SKevin Wolf                                          BDRV_O_RDWR, errp);
16944ef85a9cSKevin Wolf     if (mirror_top_bs == NULL) {
1695cc19f177SVladimir Sementsov-Ogievskiy         return NULL;
1696893f7ebaSPaolo Bonzini     }
1697d3c8c674SKevin Wolf     if (!filter_node_name) {
1698d3c8c674SKevin Wolf         mirror_top_bs->implicit = true;
1699d3c8c674SKevin Wolf     }
1700e5182c1cSMax Reitz 
1701e5182c1cSMax Reitz     /* So that we can always drop this node */
1702e5182c1cSMax Reitz     mirror_top_bs->never_freeze = true;
1703e5182c1cSMax Reitz 
17044ef85a9cSKevin Wolf     mirror_top_bs->total_sectors = bs->total_sectors;
1705228345bfSMax Reitz     mirror_top_bs->supported_write_flags = BDRV_REQ_WRITE_UNCHANGED;
170680f5c33fSKevin Wolf     mirror_top_bs->supported_zero_flags = BDRV_REQ_WRITE_UNCHANGED |
170780f5c33fSKevin Wolf                                           BDRV_REQ_NO_FALLBACK;
1708429076e8SMax Reitz     bs_opaque = g_new0(MirrorBDSOpaque, 1);
1709429076e8SMax Reitz     mirror_top_bs->opaque = bs_opaque;
1710893f7ebaSPaolo Bonzini 
171153431b90SMax Reitz     bs_opaque->is_commit = target_is_backing;
171253431b90SMax Reitz 
17134ef85a9cSKevin Wolf     bdrv_drained_begin(bs);
1714934aee14SVladimir Sementsov-Ogievskiy     ret = bdrv_append(mirror_top_bs, bs, errp);
17154ef85a9cSKevin Wolf     bdrv_drained_end(bs);
17164ef85a9cSKevin Wolf 
1717934aee14SVladimir Sementsov-Ogievskiy     if (ret < 0) {
1718b2c2832cSKevin Wolf         bdrv_unref(mirror_top_bs);
1719cc19f177SVladimir Sementsov-Ogievskiy         return NULL;
1720b2c2832cSKevin Wolf     }
1721b2c2832cSKevin Wolf 
17224ef85a9cSKevin Wolf     /* Make sure that the source is not resized while the job is running */
172375859b94SJohn Snow     s = block_job_create(job_id, driver, NULL, mirror_top_bs,
17244ef85a9cSKevin Wolf                          BLK_PERM_CONSISTENT_READ,
17254ef85a9cSKevin Wolf                          BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE_UNCHANGED |
172664631f36SVladimir Sementsov-Ogievskiy                          BLK_PERM_WRITE, speed,
17274ef85a9cSKevin Wolf                          creation_flags, cb, opaque, errp);
17284ef85a9cSKevin Wolf     if (!s) {
17294ef85a9cSKevin Wolf         goto fail;
17304ef85a9cSKevin Wolf     }
1731429076e8SMax Reitz 
17327a25fcd0SMax Reitz     /* The block job now has a reference to this node */
17337a25fcd0SMax Reitz     bdrv_unref(mirror_top_bs);
17347a25fcd0SMax Reitz 
17354ef85a9cSKevin Wolf     s->mirror_top_bs = mirror_top_bs;
17364ef85a9cSKevin Wolf 
17374ef85a9cSKevin Wolf     /* No resize for the target either; while the mirror is still running, a
17384ef85a9cSKevin Wolf      * consistent read isn't necessarily possible. We could possibly allow
17394ef85a9cSKevin Wolf      * writes and graph modifications, though it would likely defeat the
17404ef85a9cSKevin Wolf      * purpose of a mirror, so leave them blocked for now.
17414ef85a9cSKevin Wolf      *
17424ef85a9cSKevin Wolf      * In the case of active commit, things look a bit different, though,
17434ef85a9cSKevin Wolf      * because the target is an already populated backing file in active use.
17444ef85a9cSKevin Wolf      * We can allow anything except resize there.*/
17453f072a7fSMax Reitz 
17463f072a7fSMax Reitz     target_perms = BLK_PERM_WRITE;
17473f072a7fSMax Reitz     target_shared_perms = BLK_PERM_WRITE_UNCHANGED;
17483f072a7fSMax Reitz 
17493f072a7fSMax Reitz     if (target_is_backing) {
17503f072a7fSMax Reitz         int64_t bs_size, target_size;
17513f072a7fSMax Reitz         bs_size = bdrv_getlength(bs);
17523f072a7fSMax Reitz         if (bs_size < 0) {
17533f072a7fSMax Reitz             error_setg_errno(errp, -bs_size,
17543f072a7fSMax Reitz                              "Could not inquire top image size");
17553f072a7fSMax Reitz             goto fail;
17563f072a7fSMax Reitz         }
17573f072a7fSMax Reitz 
17583f072a7fSMax Reitz         target_size = bdrv_getlength(target);
17593f072a7fSMax Reitz         if (target_size < 0) {
17603f072a7fSMax Reitz             error_setg_errno(errp, -target_size,
17613f072a7fSMax Reitz                              "Could not inquire base image size");
17623f072a7fSMax Reitz             goto fail;
17633f072a7fSMax Reitz         }
17643f072a7fSMax Reitz 
17653f072a7fSMax Reitz         if (target_size < bs_size) {
17663f072a7fSMax Reitz             target_perms |= BLK_PERM_RESIZE;
17673f072a7fSMax Reitz         }
17683f072a7fSMax Reitz 
176964631f36SVladimir Sementsov-Ogievskiy         target_shared_perms |= BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE;
17703f072a7fSMax Reitz     } else if (bdrv_chain_contains(bs, bdrv_skip_filters(target))) {
17713f072a7fSMax Reitz         /*
17723f072a7fSMax Reitz          * We may want to allow this in the future, but it would
17733f072a7fSMax Reitz          * require taking some extra care.
17743f072a7fSMax Reitz          */
17753f072a7fSMax Reitz         error_setg(errp, "Cannot mirror to a filter on top of a node in the "
17763f072a7fSMax Reitz                    "source's backing chain");
17773f072a7fSMax Reitz         goto fail;
17783f072a7fSMax Reitz     }
17793f072a7fSMax Reitz 
1780d861ab3aSKevin Wolf     s->target = blk_new(s->common.job.aio_context,
17813f072a7fSMax Reitz                         target_perms, target_shared_perms);
1782d7086422SKevin Wolf     ret = blk_insert_bs(s->target, target, errp);
1783d7086422SKevin Wolf     if (ret < 0) {
17844ef85a9cSKevin Wolf         goto fail;
1785d7086422SKevin Wolf     }
1786045a2f82SFam Zheng     if (is_mirror) {
1787045a2f82SFam Zheng         /* XXX: Mirror target could be a NBD server of target QEMU in the case
1788045a2f82SFam Zheng          * of non-shared block migration. To allow migration completion, we
1789045a2f82SFam Zheng          * have to allow "inactivate" of the target BB.  When that happens, we
1790045a2f82SFam Zheng          * know the job is drained, and the vcpus are stopped, so no write
1791045a2f82SFam Zheng          * operation will be performed. Block layer already has assertions to
1792045a2f82SFam Zheng          * ensure that. */
1793045a2f82SFam Zheng         blk_set_force_allow_inactivate(s->target);
1794045a2f82SFam Zheng     }
17959ff7f0dfSKevin Wolf     blk_set_allow_aio_context_change(s->target, true);
1796cf312932SKevin Wolf     blk_set_disable_request_queuing(s->target, true);
1797e253f4b8SKevin Wolf 
179809158f00SBenoît Canet     s->replaces = g_strdup(replaces);
1799b952b558SPaolo Bonzini     s->on_source_error = on_source_error;
1800b952b558SPaolo Bonzini     s->on_target_error = on_target_error;
180103544a6eSFam Zheng     s->is_none_mode = is_none_mode;
1802274fcceeSMax Reitz     s->backing_mode = backing_mode;
1803cdf3bc93SMax Reitz     s->zero_target = zero_target;
1804481debaaSMax Reitz     s->copy_mode = copy_mode;
18055bc361b8SFam Zheng     s->base = base;
18063f072a7fSMax Reitz     s->base_overlay = bdrv_find_overlay(bs, base);
1807eee13dfeSPaolo Bonzini     s->granularity = granularity;
180848ac0a4dSWen Congyang     s->buf_size = ROUND_UP(buf_size, granularity);
18090fc9f8eaSFam Zheng     s->unmap = unmap;
1810b49f7eadSWen Congyang     if (auto_complete) {
1811b49f7eadSWen Congyang         s->should_complete = true;
1812b49f7eadSWen Congyang     }
1813b812f671SPaolo Bonzini 
18140db6e54aSFam Zheng     s->dirty_bitmap = bdrv_create_dirty_bitmap(bs, granularity, NULL, errp);
1815b8afb520SFam Zheng     if (!s->dirty_bitmap) {
181688f9d1b3SKevin Wolf         goto fail;
1817b8afb520SFam Zheng     }
1818dbdf699cSVladimir Sementsov-Ogievskiy     if (s->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING) {
1819dbdf699cSVladimir Sementsov-Ogievskiy         bdrv_disable_dirty_bitmap(s->dirty_bitmap);
1820dbdf699cSVladimir Sementsov-Ogievskiy     }
182110f3cd15SAlberto Garcia 
182267b24427SAlberto Garcia     ret = block_job_add_bdrv(&s->common, "source", bs, 0,
182367b24427SAlberto Garcia                              BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE |
182467b24427SAlberto Garcia                              BLK_PERM_CONSISTENT_READ,
182567b24427SAlberto Garcia                              errp);
182667b24427SAlberto Garcia     if (ret < 0) {
182767b24427SAlberto Garcia         goto fail;
182867b24427SAlberto Garcia     }
182967b24427SAlberto Garcia 
18304ef85a9cSKevin Wolf     /* Required permissions are already taken with blk_new() */
183176d554e2SKevin Wolf     block_job_add_bdrv(&s->common, "target", target, 0, BLK_PERM_ALL,
183276d554e2SKevin Wolf                        &error_abort);
183376d554e2SKevin Wolf 
1834f3ede4b0SAlberto Garcia     /* In commit_active_start() all intermediate nodes disappear, so
1835f3ede4b0SAlberto Garcia      * any jobs in them must be blocked */
18364ef85a9cSKevin Wolf     if (target_is_backing) {
18373f072a7fSMax Reitz         BlockDriverState *iter, *filtered_target;
18383f072a7fSMax Reitz         uint64_t iter_shared_perms;
18393f072a7fSMax Reitz 
18403f072a7fSMax Reitz         /*
18413f072a7fSMax Reitz          * The topmost node with
18423f072a7fSMax Reitz          * bdrv_skip_filters(filtered_target) == bdrv_skip_filters(target)
18433f072a7fSMax Reitz          */
18443f072a7fSMax Reitz         filtered_target = bdrv_cow_bs(bdrv_find_overlay(bs, target));
18453f072a7fSMax Reitz 
18463f072a7fSMax Reitz         assert(bdrv_skip_filters(filtered_target) ==
18473f072a7fSMax Reitz                bdrv_skip_filters(target));
18483f072a7fSMax Reitz 
18493f072a7fSMax Reitz         /*
18503f072a7fSMax Reitz          * XXX BLK_PERM_WRITE needs to be allowed so we don't block
18514ef85a9cSKevin Wolf          * ourselves at s->base (if writes are blocked for a node, they are
18524ef85a9cSKevin Wolf          * also blocked for its backing file). The other options would be a
18533f072a7fSMax Reitz          * second filter driver above s->base (== target).
18543f072a7fSMax Reitz          */
18553f072a7fSMax Reitz         iter_shared_perms = BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE;
18563f072a7fSMax Reitz 
18573f072a7fSMax Reitz         for (iter = bdrv_filter_or_cow_bs(bs); iter != target;
18583f072a7fSMax Reitz              iter = bdrv_filter_or_cow_bs(iter))
18593f072a7fSMax Reitz         {
18603f072a7fSMax Reitz             if (iter == filtered_target) {
18613f072a7fSMax Reitz                 /*
18623f072a7fSMax Reitz                  * From here on, all nodes are filters on the base.
18633f072a7fSMax Reitz                  * This allows us to share BLK_PERM_CONSISTENT_READ.
18643f072a7fSMax Reitz                  */
18653f072a7fSMax Reitz                 iter_shared_perms |= BLK_PERM_CONSISTENT_READ;
18663f072a7fSMax Reitz             }
18673f072a7fSMax Reitz 
18684ef85a9cSKevin Wolf             ret = block_job_add_bdrv(&s->common, "intermediate node", iter, 0,
18693f072a7fSMax Reitz                                      iter_shared_perms, errp);
18704ef85a9cSKevin Wolf             if (ret < 0) {
18714ef85a9cSKevin Wolf                 goto fail;
18724ef85a9cSKevin Wolf             }
1873f3ede4b0SAlberto Garcia         }
1874ef53dc09SAlberto Garcia 
1875ef53dc09SAlberto Garcia         if (bdrv_freeze_backing_chain(mirror_top_bs, target, errp) < 0) {
1876ef53dc09SAlberto Garcia             goto fail;
1877ef53dc09SAlberto Garcia         }
1878f3ede4b0SAlberto Garcia     }
187910f3cd15SAlberto Garcia 
188012aa4082SMax Reitz     QTAILQ_INIT(&s->ops_in_flight);
188112aa4082SMax Reitz 
18825ccac6f1SJohn Snow     trace_mirror_start(bs, s, opaque);
1883da01ff7fSKevin Wolf     job_start(&s->common.job);
1884cc19f177SVladimir Sementsov-Ogievskiy 
1885cc19f177SVladimir Sementsov-Ogievskiy     return &s->common;
18864ef85a9cSKevin Wolf 
18874ef85a9cSKevin Wolf fail:
18884ef85a9cSKevin Wolf     if (s) {
18897a25fcd0SMax Reitz         /* Make sure this BDS does not go away until we have completed the graph
18907a25fcd0SMax Reitz          * changes below */
18917a25fcd0SMax Reitz         bdrv_ref(mirror_top_bs);
18927a25fcd0SMax Reitz 
18934ef85a9cSKevin Wolf         g_free(s->replaces);
18944ef85a9cSKevin Wolf         blk_unref(s->target);
1895429076e8SMax Reitz         bs_opaque->job = NULL;
1896e917e2cbSAlberto Garcia         if (s->dirty_bitmap) {
18975deb6cbdSVladimir Sementsov-Ogievskiy             bdrv_release_dirty_bitmap(s->dirty_bitmap);
1898e917e2cbSAlberto Garcia         }
18994ad35181SKevin Wolf         job_early_fail(&s->common.job);
19004ef85a9cSKevin Wolf     }
19014ef85a9cSKevin Wolf 
1902f94dc3b4SMax Reitz     bs_opaque->stop = true;
1903f94dc3b4SMax Reitz     bdrv_child_refresh_perms(mirror_top_bs, mirror_top_bs->backing,
1904c1cef672SFam Zheng                              &error_abort);
19053f072a7fSMax Reitz     bdrv_replace_node(mirror_top_bs, mirror_top_bs->backing->bs, &error_abort);
19067a25fcd0SMax Reitz 
19077a25fcd0SMax Reitz     bdrv_unref(mirror_top_bs);
1908cc19f177SVladimir Sementsov-Ogievskiy 
1909cc19f177SVladimir Sementsov-Ogievskiy     return NULL;
1910893f7ebaSPaolo Bonzini }
191103544a6eSFam Zheng 
191271aa9867SAlberto Garcia void mirror_start(const char *job_id, BlockDriverState *bs,
191371aa9867SAlberto Garcia                   BlockDriverState *target, const char *replaces,
1914a1999b33SJohn Snow                   int creation_flags, int64_t speed,
1915a1999b33SJohn Snow                   uint32_t granularity, int64_t buf_size,
1916274fcceeSMax Reitz                   MirrorSyncMode mode, BlockMirrorBackingMode backing_mode,
1917cdf3bc93SMax Reitz                   bool zero_target,
1918274fcceeSMax Reitz                   BlockdevOnError on_source_error,
191903544a6eSFam Zheng                   BlockdevOnError on_target_error,
1920481debaaSMax Reitz                   bool unmap, const char *filter_node_name,
1921481debaaSMax Reitz                   MirrorCopyMode copy_mode, Error **errp)
192203544a6eSFam Zheng {
192303544a6eSFam Zheng     bool is_none_mode;
192403544a6eSFam Zheng     BlockDriverState *base;
192503544a6eSFam Zheng 
1926b4ad82aaSEmanuele Giuseppe Esposito     GLOBAL_STATE_CODE();
1927b4ad82aaSEmanuele Giuseppe Esposito 
1928c8b56501SJohn Snow     if ((mode == MIRROR_SYNC_MODE_INCREMENTAL) ||
1929c8b56501SJohn Snow         (mode == MIRROR_SYNC_MODE_BITMAP)) {
1930c8b56501SJohn Snow         error_setg(errp, "Sync mode '%s' not supported",
1931c8b56501SJohn Snow                    MirrorSyncMode_str(mode));
1932d58d8453SJohn Snow         return;
1933d58d8453SJohn Snow     }
193403544a6eSFam Zheng     is_none_mode = mode == MIRROR_SYNC_MODE_NONE;
19353f072a7fSMax Reitz     base = mode == MIRROR_SYNC_MODE_TOP ? bdrv_backing_chain_next(bs) : NULL;
1936a1999b33SJohn Snow     mirror_start_job(job_id, bs, creation_flags, target, replaces,
1937cdf3bc93SMax Reitz                      speed, granularity, buf_size, backing_mode, zero_target,
193851ccfa2dSFam Zheng                      on_source_error, on_target_error, unmap, NULL, NULL,
19396cdbceb1SKevin Wolf                      &mirror_job_driver, is_none_mode, base, false,
1940481debaaSMax Reitz                      filter_node_name, true, copy_mode, errp);
194103544a6eSFam Zheng }
194203544a6eSFam Zheng 
1943cc19f177SVladimir Sementsov-Ogievskiy BlockJob *commit_active_start(const char *job_id, BlockDriverState *bs,
194447970dfbSJohn Snow                               BlockDriverState *base, int creation_flags,
194547970dfbSJohn Snow                               int64_t speed, BlockdevOnError on_error,
19460db832f4SKevin Wolf                               const char *filter_node_name,
194778bbd910SFam Zheng                               BlockCompletionFunc *cb, void *opaque,
194878bbd910SFam Zheng                               bool auto_complete, Error **errp)
194903544a6eSFam Zheng {
19501ba79388SAlberto Garcia     bool base_read_only;
1951eb5becc1SVladimir Sementsov-Ogievskiy     BlockJob *job;
19524da83585SJeff Cody 
1953b4ad82aaSEmanuele Giuseppe Esposito     GLOBAL_STATE_CODE();
1954b4ad82aaSEmanuele Giuseppe Esposito 
19551ba79388SAlberto Garcia     base_read_only = bdrv_is_read_only(base);
19564da83585SJeff Cody 
19571ba79388SAlberto Garcia     if (base_read_only) {
19581ba79388SAlberto Garcia         if (bdrv_reopen_set_read_only(base, false, errp) < 0) {
1959cc19f177SVladimir Sementsov-Ogievskiy             return NULL;
196020a63d2cSFam Zheng         }
19611ba79388SAlberto Garcia     }
19624da83585SJeff Cody 
1963eb5becc1SVladimir Sementsov-Ogievskiy     job = mirror_start_job(
1964cc19f177SVladimir Sementsov-Ogievskiy                      job_id, bs, creation_flags, base, NULL, speed, 0, 0,
1965cdf3bc93SMax Reitz                      MIRROR_LEAVE_BACKING_CHAIN, false,
196651ccfa2dSFam Zheng                      on_error, on_error, true, cb, opaque,
19676cdbceb1SKevin Wolf                      &commit_active_job_driver, false, base, auto_complete,
1968481debaaSMax Reitz                      filter_node_name, false, MIRROR_COPY_MODE_BACKGROUND,
1969eb5becc1SVladimir Sementsov-Ogievskiy                      errp);
1970eb5becc1SVladimir Sementsov-Ogievskiy     if (!job) {
19714da83585SJeff Cody         goto error_restore_flags;
19724da83585SJeff Cody     }
19734da83585SJeff Cody 
1974eb5becc1SVladimir Sementsov-Ogievskiy     return job;
19754da83585SJeff Cody 
19764da83585SJeff Cody error_restore_flags:
19774da83585SJeff Cody     /* ignore error and errp for bdrv_reopen, because we want to propagate
19784da83585SJeff Cody      * the original error */
19791ba79388SAlberto Garcia     if (base_read_only) {
19801ba79388SAlberto Garcia         bdrv_reopen_set_read_only(base, true, NULL);
19811ba79388SAlberto Garcia     }
1982cc19f177SVladimir Sementsov-Ogievskiy     return NULL;
198303544a6eSFam Zheng }
1984