1 #!/usr/bin/env python 2 # 3 # Tests for incremental drive-backup 4 # 5 # Copyright (C) 2015 John Snow for Red Hat, Inc. 6 # 7 # Based on 056. 8 # 9 # This program is free software; you can redistribute it and/or modify 10 # it under the terms of the GNU General Public License as published by 11 # the Free Software Foundation; either version 2 of the License, or 12 # (at your option) any later version. 13 # 14 # This program is distributed in the hope that it will be useful, 15 # but WITHOUT ANY WARRANTY; without even the implied warranty of 16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 # GNU General Public License for more details. 18 # 19 # You should have received a copy of the GNU General Public License 20 # along with this program. If not, see <http://www.gnu.org/licenses/>. 21 # 22 23 import os 24 import iotests 25 26 27 def io_write_patterns(img, patterns): 28 for pattern in patterns: 29 iotests.qemu_io('-c', 'write -P%s %s %s' % pattern, img) 30 31 32 def try_remove(img): 33 try: 34 os.remove(img) 35 except OSError: 36 pass 37 38 39 def transaction_action(action, **kwargs): 40 return { 41 'type': action, 42 'data': dict((k.replace('_', '-'), v) for k, v in kwargs.iteritems()) 43 } 44 45 46 def transaction_bitmap_clear(node, name, **kwargs): 47 return transaction_action('block-dirty-bitmap-clear', 48 node=node, name=name, **kwargs) 49 50 51 def transaction_drive_backup(device, target, **kwargs): 52 return transaction_action('drive-backup', device=device, target=target, 53 **kwargs) 54 55 56 class Bitmap: 57 def __init__(self, name, drive): 58 self.name = name 59 self.drive = drive 60 self.num = 0 61 self.backups = list() 62 63 def base_target(self): 64 return (self.drive['backup'], None) 65 66 def new_target(self, num=None): 67 if num is None: 68 num = self.num 69 self.num = num + 1 70 base = os.path.join(iotests.test_dir, 71 "%s.%s." % (self.drive['id'], self.name)) 72 suff = "%i.%s" % (num, self.drive['fmt']) 73 target = base + "inc" + suff 74 reference = base + "ref" + suff 75 self.backups.append((target, reference)) 76 return (target, reference) 77 78 def last_target(self): 79 if self.backups: 80 return self.backups[-1] 81 return self.base_target() 82 83 def del_target(self): 84 for image in self.backups.pop(): 85 try_remove(image) 86 self.num -= 1 87 88 def cleanup(self): 89 for backup in self.backups: 90 for image in backup: 91 try_remove(image) 92 93 94 class TestIncrementalBackupBase(iotests.QMPTestCase): 95 def __init__(self, *args): 96 super(TestIncrementalBackupBase, self).__init__(*args) 97 self.bitmaps = list() 98 self.files = list() 99 self.drives = list() 100 self.vm = iotests.VM() 101 self.err_img = os.path.join(iotests.test_dir, 'err.%s' % iotests.imgfmt) 102 103 104 def setUp(self): 105 # Create a base image with a distinctive patterning 106 drive0 = self.add_node('drive0') 107 self.img_create(drive0['file'], drive0['fmt']) 108 self.vm.add_drive(drive0['file']) 109 self.write_default_pattern(drive0['file']) 110 self.vm.launch() 111 112 113 def write_default_pattern(self, target): 114 io_write_patterns(target, (('0x41', 0, 512), 115 ('0xd5', '1M', '32k'), 116 ('0xdc', '32M', '124k'))) 117 118 119 def add_node(self, node_id, fmt=iotests.imgfmt, path=None, backup=None): 120 if path is None: 121 path = os.path.join(iotests.test_dir, '%s.%s' % (node_id, fmt)) 122 if backup is None: 123 backup = os.path.join(iotests.test_dir, 124 '%s.full.backup.%s' % (node_id, fmt)) 125 126 self.drives.append({ 127 'id': node_id, 128 'file': path, 129 'backup': backup, 130 'fmt': fmt }) 131 return self.drives[-1] 132 133 134 def img_create(self, img, fmt=iotests.imgfmt, size='64M', 135 parent=None, parentFormat=None, **kwargs): 136 optargs = [] 137 for k,v in kwargs.iteritems(): 138 optargs = optargs + ['-o', '%s=%s' % (k,v)] 139 args = ['create', '-f', fmt] + optargs + [img, size] 140 if parent: 141 if parentFormat is None: 142 parentFormat = fmt 143 args = args + ['-b', parent, '-F', parentFormat] 144 iotests.qemu_img(*args) 145 self.files.append(img) 146 147 148 def do_qmp_backup(self, error='Input/output error', **kwargs): 149 res = self.vm.qmp('drive-backup', **kwargs) 150 self.assert_qmp(res, 'return', {}) 151 return self.wait_qmp_backup(kwargs['device'], error) 152 153 154 def wait_qmp_backup(self, device, error='Input/output error'): 155 event = self.vm.event_wait(name="BLOCK_JOB_COMPLETED", 156 match={'data': {'device': device}}) 157 self.assertNotEqual(event, None) 158 159 try: 160 failure = self.dictpath(event, 'data/error') 161 except AssertionError: 162 # Backup succeeded. 163 self.assert_qmp(event, 'data/offset', event['data']['len']) 164 return True 165 else: 166 # Backup failed. 167 self.assert_qmp(event, 'data/error', error) 168 return False 169 170 171 def wait_qmp_backup_cancelled(self, device): 172 event = self.vm.event_wait(name='BLOCK_JOB_CANCELLED', 173 match={'data': {'device': device}}) 174 self.assertNotEqual(event, None) 175 176 177 def create_anchor_backup(self, drive=None): 178 if drive is None: 179 drive = self.drives[-1] 180 res = self.do_qmp_backup(device=drive['id'], sync='full', 181 format=drive['fmt'], target=drive['backup']) 182 self.assertTrue(res) 183 self.files.append(drive['backup']) 184 return drive['backup'] 185 186 187 def make_reference_backup(self, bitmap=None): 188 if bitmap is None: 189 bitmap = self.bitmaps[-1] 190 _, reference = bitmap.last_target() 191 res = self.do_qmp_backup(device=bitmap.drive['id'], sync='full', 192 format=bitmap.drive['fmt'], target=reference) 193 self.assertTrue(res) 194 195 196 def add_bitmap(self, name, drive, **kwargs): 197 bitmap = Bitmap(name, drive) 198 self.bitmaps.append(bitmap) 199 result = self.vm.qmp('block-dirty-bitmap-add', node=drive['id'], 200 name=bitmap.name, **kwargs) 201 self.assert_qmp(result, 'return', {}) 202 return bitmap 203 204 205 def prepare_backup(self, bitmap=None, parent=None): 206 if bitmap is None: 207 bitmap = self.bitmaps[-1] 208 if parent is None: 209 parent, _ = bitmap.last_target() 210 211 target, _ = bitmap.new_target() 212 self.img_create(target, bitmap.drive['fmt'], parent=parent) 213 return target 214 215 216 def create_incremental(self, bitmap=None, parent=None, 217 parentFormat=None, validate=True): 218 if bitmap is None: 219 bitmap = self.bitmaps[-1] 220 if parent is None: 221 parent, _ = bitmap.last_target() 222 223 target = self.prepare_backup(bitmap, parent) 224 res = self.do_qmp_backup(device=bitmap.drive['id'], 225 sync='incremental', bitmap=bitmap.name, 226 format=bitmap.drive['fmt'], target=target, 227 mode='existing') 228 if not res: 229 bitmap.del_target(); 230 self.assertFalse(validate) 231 else: 232 self.make_reference_backup(bitmap) 233 return res 234 235 236 def check_backups(self): 237 for bitmap in self.bitmaps: 238 for incremental, reference in bitmap.backups: 239 self.assertTrue(iotests.compare_images(incremental, reference)) 240 last = bitmap.last_target()[0] 241 self.assertTrue(iotests.compare_images(last, bitmap.drive['file'])) 242 243 244 def hmp_io_writes(self, drive, patterns): 245 for pattern in patterns: 246 self.vm.hmp_qemu_io(drive, 'write -P%s %s %s' % pattern) 247 self.vm.hmp_qemu_io(drive, 'flush') 248 249 250 def do_incremental_simple(self, **kwargs): 251 self.create_anchor_backup() 252 self.add_bitmap('bitmap0', self.drives[0], **kwargs) 253 254 # Sanity: Create a "hollow" incremental backup 255 self.create_incremental() 256 # Three writes: One complete overwrite, one new segment, 257 # and one partial overlap. 258 self.hmp_io_writes(self.drives[0]['id'], (('0xab', 0, 512), 259 ('0xfe', '16M', '256k'), 260 ('0x64', '32736k', '64k'))) 261 self.create_incremental() 262 # Three more writes, one of each kind, like above 263 self.hmp_io_writes(self.drives[0]['id'], (('0x9a', 0, 512), 264 ('0x55', '8M', '352k'), 265 ('0x78', '15872k', '1M'))) 266 self.create_incremental() 267 self.vm.shutdown() 268 self.check_backups() 269 270 271 def tearDown(self): 272 self.vm.shutdown() 273 for bitmap in self.bitmaps: 274 bitmap.cleanup() 275 for filename in self.files: 276 try_remove(filename) 277 278 279 280 class TestIncrementalBackup(TestIncrementalBackupBase): 281 def test_incremental_simple(self): 282 ''' 283 Test: Create and verify three incremental backups. 284 285 Create a bitmap and a full backup before VM execution begins, 286 then create a series of three incremental backups "during execution," 287 i.e.; after IO requests begin modifying the drive. 288 ''' 289 return self.do_incremental_simple() 290 291 292 def test_small_granularity(self): 293 ''' 294 Test: Create and verify backups made with a small granularity bitmap. 295 296 Perform the same test as test_incremental_simple, but with a granularity 297 of only 32KiB instead of the present default of 64KiB. 298 ''' 299 return self.do_incremental_simple(granularity=32768) 300 301 302 def test_large_granularity(self): 303 ''' 304 Test: Create and verify backups made with a large granularity bitmap. 305 306 Perform the same test as test_incremental_simple, but with a granularity 307 of 128KiB instead of the present default of 64KiB. 308 ''' 309 return self.do_incremental_simple(granularity=131072) 310 311 312 def test_larger_cluster_target(self): 313 ''' 314 Test: Create and verify backups made to a larger cluster size target. 315 316 With a default granularity of 64KiB, verify that backups made to a 317 larger cluster size target of 128KiB without a backing file works. 318 ''' 319 drive0 = self.drives[0] 320 321 # Create a cluster_size=128k full backup / "anchor" backup 322 self.img_create(drive0['backup'], cluster_size='128k') 323 self.assertTrue(self.do_qmp_backup(device=drive0['id'], sync='full', 324 format=drive0['fmt'], 325 target=drive0['backup'], 326 mode='existing')) 327 328 # Create bitmap and dirty it with some new writes. 329 # overwrite [32736, 32799] which will dirty bitmap clusters at 330 # 32M-64K and 32M. 32M+64K will be left undirtied. 331 bitmap0 = self.add_bitmap('bitmap0', drive0) 332 self.hmp_io_writes(drive0['id'], 333 (('0xab', 0, 512), 334 ('0xfe', '16M', '256k'), 335 ('0x64', '32736k', '64k'))) 336 337 338 # Prepare a cluster_size=128k backup target without a backing file. 339 (target, _) = bitmap0.new_target() 340 self.img_create(target, bitmap0.drive['fmt'], cluster_size='128k') 341 342 # Perform Incremental Backup 343 self.assertTrue(self.do_qmp_backup(device=bitmap0.drive['id'], 344 sync='incremental', 345 bitmap=bitmap0.name, 346 format=bitmap0.drive['fmt'], 347 target=target, 348 mode='existing')) 349 self.make_reference_backup(bitmap0) 350 351 # Add the backing file, then compare and exit. 352 iotests.qemu_img('rebase', '-f', drive0['fmt'], '-u', '-b', 353 drive0['backup'], '-F', drive0['fmt'], target) 354 self.vm.shutdown() 355 self.check_backups() 356 357 358 def test_incremental_transaction(self): 359 '''Test: Verify backups made from transactionally created bitmaps. 360 361 Create a bitmap "before" VM execution begins, then create a second 362 bitmap AFTER writes have already occurred. Use transactions to create 363 a full backup and synchronize both bitmaps to this backup. 364 Create an incremental backup through both bitmaps and verify that 365 both backups match the current drive0 image. 366 ''' 367 368 drive0 = self.drives[0] 369 bitmap0 = self.add_bitmap('bitmap0', drive0) 370 self.hmp_io_writes(drive0['id'], (('0xab', 0, 512), 371 ('0xfe', '16M', '256k'), 372 ('0x64', '32736k', '64k'))) 373 bitmap1 = self.add_bitmap('bitmap1', drive0) 374 375 result = self.vm.qmp('transaction', actions=[ 376 transaction_bitmap_clear(bitmap0.drive['id'], bitmap0.name), 377 transaction_bitmap_clear(bitmap1.drive['id'], bitmap1.name), 378 transaction_drive_backup(drive0['id'], drive0['backup'], 379 sync='full', format=drive0['fmt']) 380 ]) 381 self.assert_qmp(result, 'return', {}) 382 self.wait_until_completed(drive0['id']) 383 self.files.append(drive0['backup']) 384 385 self.hmp_io_writes(drive0['id'], (('0x9a', 0, 512), 386 ('0x55', '8M', '352k'), 387 ('0x78', '15872k', '1M'))) 388 # Both bitmaps should be correctly in sync. 389 self.create_incremental(bitmap0) 390 self.create_incremental(bitmap1) 391 self.vm.shutdown() 392 self.check_backups() 393 394 395 def test_transaction_failure(self): 396 '''Test: Verify backups made from a transaction that partially fails. 397 398 Add a second drive with its own unique pattern, and add a bitmap to each 399 drive. Use blkdebug to interfere with the backup on just one drive and 400 attempt to create a coherent incremental backup across both drives. 401 402 verify a failure in one but not both, then delete the failed stubs and 403 re-run the same transaction. 404 405 verify that both incrementals are created successfully. 406 ''' 407 408 # Create a second drive, with pattern: 409 drive1 = self.add_node('drive1') 410 self.img_create(drive1['file'], drive1['fmt']) 411 io_write_patterns(drive1['file'], (('0x14', 0, 512), 412 ('0x5d', '1M', '32k'), 413 ('0xcd', '32M', '124k'))) 414 415 # Create a blkdebug interface to this img as 'drive1' 416 result = self.vm.qmp('blockdev-add', options={ 417 'id': drive1['id'], 418 'driver': drive1['fmt'], 419 'file': { 420 'driver': 'blkdebug', 421 'image': { 422 'driver': 'file', 423 'filename': drive1['file'] 424 }, 425 'set-state': [{ 426 'event': 'flush_to_disk', 427 'state': 1, 428 'new_state': 2 429 }], 430 'inject-error': [{ 431 'event': 'read_aio', 432 'errno': 5, 433 'state': 2, 434 'immediately': False, 435 'once': True 436 }], 437 } 438 }) 439 self.assert_qmp(result, 'return', {}) 440 441 # Create bitmaps and full backups for both drives 442 drive0 = self.drives[0] 443 dr0bm0 = self.add_bitmap('bitmap0', drive0) 444 dr1bm0 = self.add_bitmap('bitmap0', drive1) 445 self.create_anchor_backup(drive0) 446 self.create_anchor_backup(drive1) 447 self.assert_no_active_block_jobs() 448 self.assertFalse(self.vm.get_qmp_events(wait=False)) 449 450 # Emulate some writes 451 self.hmp_io_writes(drive0['id'], (('0xab', 0, 512), 452 ('0xfe', '16M', '256k'), 453 ('0x64', '32736k', '64k'))) 454 self.hmp_io_writes(drive1['id'], (('0xba', 0, 512), 455 ('0xef', '16M', '256k'), 456 ('0x46', '32736k', '64k'))) 457 458 # Create incremental backup targets 459 target0 = self.prepare_backup(dr0bm0) 460 target1 = self.prepare_backup(dr1bm0) 461 462 # Ask for a new incremental backup per-each drive, 463 # expecting drive1's backup to fail: 464 transaction = [ 465 transaction_drive_backup(drive0['id'], target0, sync='incremental', 466 format=drive0['fmt'], mode='existing', 467 bitmap=dr0bm0.name), 468 transaction_drive_backup(drive1['id'], target1, sync='incremental', 469 format=drive1['fmt'], mode='existing', 470 bitmap=dr1bm0.name) 471 ] 472 result = self.vm.qmp('transaction', actions=transaction, 473 properties={'completion-mode': 'grouped'} ) 474 self.assert_qmp(result, 'return', {}) 475 476 # Observe that drive0's backup is cancelled and drive1 completes with 477 # an error. 478 self.wait_qmp_backup_cancelled(drive0['id']) 479 self.assertFalse(self.wait_qmp_backup(drive1['id'])) 480 error = self.vm.event_wait('BLOCK_JOB_ERROR') 481 self.assert_qmp(error, 'data', {'device': drive1['id'], 482 'action': 'report', 483 'operation': 'read'}) 484 self.assertFalse(self.vm.get_qmp_events(wait=False)) 485 self.assert_no_active_block_jobs() 486 487 # Delete drive0's successful target and eliminate our record of the 488 # unsuccessful drive1 target. Then re-run the same transaction. 489 dr0bm0.del_target() 490 dr1bm0.del_target() 491 target0 = self.prepare_backup(dr0bm0) 492 target1 = self.prepare_backup(dr1bm0) 493 494 # Re-run the exact same transaction. 495 result = self.vm.qmp('transaction', actions=transaction, 496 properties={'completion-mode':'grouped'}) 497 self.assert_qmp(result, 'return', {}) 498 499 # Both should complete successfully this time. 500 self.assertTrue(self.wait_qmp_backup(drive0['id'])) 501 self.assertTrue(self.wait_qmp_backup(drive1['id'])) 502 self.make_reference_backup(dr0bm0) 503 self.make_reference_backup(dr1bm0) 504 self.assertFalse(self.vm.get_qmp_events(wait=False)) 505 self.assert_no_active_block_jobs() 506 507 # And the images should of course validate. 508 self.vm.shutdown() 509 self.check_backups() 510 511 512 def test_sync_dirty_bitmap_missing(self): 513 self.assert_no_active_block_jobs() 514 self.files.append(self.err_img) 515 result = self.vm.qmp('drive-backup', device=self.drives[0]['id'], 516 sync='incremental', format=self.drives[0]['fmt'], 517 target=self.err_img) 518 self.assert_qmp(result, 'error/class', 'GenericError') 519 520 521 def test_sync_dirty_bitmap_not_found(self): 522 self.assert_no_active_block_jobs() 523 self.files.append(self.err_img) 524 result = self.vm.qmp('drive-backup', device=self.drives[0]['id'], 525 sync='incremental', bitmap='unknown', 526 format=self.drives[0]['fmt'], target=self.err_img) 527 self.assert_qmp(result, 'error/class', 'GenericError') 528 529 530 def test_sync_dirty_bitmap_bad_granularity(self): 531 ''' 532 Test: Test what happens if we provide an improper granularity. 533 534 The granularity must always be a power of 2. 535 ''' 536 self.assert_no_active_block_jobs() 537 self.assertRaises(AssertionError, self.add_bitmap, 538 'bitmap0', self.drives[0], 539 granularity=64000) 540 541 542 class TestIncrementalBackupBlkdebug(TestIncrementalBackupBase): 543 '''Incremental backup tests that utilize a BlkDebug filter on drive0.''' 544 545 def setUp(self): 546 drive0 = self.add_node('drive0') 547 self.img_create(drive0['file'], drive0['fmt']) 548 self.write_default_pattern(drive0['file']) 549 self.vm.launch() 550 551 def test_incremental_failure(self): 552 '''Test: Verify backups made after a failure are correct. 553 554 Simulate a failure during an incremental backup block job, 555 emulate additional writes, then create another incremental backup 556 afterwards and verify that the backup created is correct. 557 ''' 558 559 drive0 = self.drives[0] 560 result = self.vm.qmp('blockdev-add', options={ 561 'id': drive0['id'], 562 'driver': drive0['fmt'], 563 'file': { 564 'driver': 'blkdebug', 565 'image': { 566 'driver': 'file', 567 'filename': drive0['file'] 568 }, 569 'set-state': [{ 570 'event': 'flush_to_disk', 571 'state': 1, 572 'new_state': 2 573 }], 574 'inject-error': [{ 575 'event': 'read_aio', 576 'errno': 5, 577 'state': 2, 578 'immediately': False, 579 'once': True 580 }], 581 } 582 }) 583 self.assert_qmp(result, 'return', {}) 584 585 self.create_anchor_backup(drive0) 586 self.add_bitmap('bitmap0', drive0) 587 # Note: at this point, during a normal execution, 588 # Assume that the VM resumes and begins issuing IO requests here. 589 590 self.hmp_io_writes(drive0['id'], (('0xab', 0, 512), 591 ('0xfe', '16M', '256k'), 592 ('0x64', '32736k', '64k'))) 593 594 result = self.create_incremental(validate=False) 595 self.assertFalse(result) 596 self.hmp_io_writes(drive0['id'], (('0x9a', 0, 512), 597 ('0x55', '8M', '352k'), 598 ('0x78', '15872k', '1M'))) 599 self.create_incremental() 600 self.vm.shutdown() 601 self.check_backups() 602 603 604 if __name__ == '__main__': 605 iotests.main(supported_fmts=['qcow2']) 606