262 	struct data_vio *agent;  member
400 	VDO_ASSERT_LOG_ONLY(data_vio == data_vio->hash_lock->agent,  in assert_hash_lock_agent()
480 static void start_deduping(struct hash_lock *lock, struct data_vio *agent,
482 static void start_locking(struct hash_lock *lock, struct data_vio *agent);
483 static void start_writing(struct hash_lock *lock, struct data_vio *agent);
525 	struct data_vio *old_agent = lock->agent;  in retire_lock_agent()
528 	lock->agent = new_agent;  in retire_lock_agent()
548 	if ((lock->state != VDO_HASH_LOCK_WRITING) || !cancel_data_vio_compression(lock->agent))  in wait_on_hash_lock()
558 	data_vio->compression.lock_holder = lock->agent;  in wait_on_hash_lock()
581 static void start_bypassing(struct hash_lock *lock, struct data_vio *agent)  in start_bypassing()  argument
584 	exit_hash_lock(agent);  in start_bypassing()
596 	if (lock->agent == NULL) {  in vdo_clean_failed_hash_lock()
597 		lock->agent = data_vio;  in vdo_clean_failed_hash_lock()
598 	} else if (data_vio != lock->agent) {  in vdo_clean_failed_hash_lock()
617 	lock->agent = NULL;  in vdo_clean_failed_hash_lock()
631 	struct data_vio *agent = as_data_vio(completion);  in finish_unlocking()  local
632 	struct hash_lock *lock = agent->hash_lock;  in finish_unlocking()
634 	assert_hash_lock_agent(agent, __func__);  in finish_unlocking()
646 		start_writing(lock, agent);  in finish_unlocking()
665 		agent = retire_lock_agent(lock);  in finish_unlocking()
666 		start_locking(lock, agent);  in finish_unlocking()
674 	start_bypassing(lock, agent);  in finish_unlocking()
687 	struct data_vio *agent = as_data_vio(completion);  in unlock_duplicate_pbn()  local
688 	struct hash_lock *lock = agent->hash_lock;  in unlock_duplicate_pbn()
690 	assert_data_vio_in_duplicate_zone(agent);  in unlock_duplicate_pbn()
694 	vdo_release_physical_zone_pbn_lock(agent->duplicate.zone, agent->duplicate.pbn,  in unlock_duplicate_pbn()
701 	launch_data_vio_hash_zone_callback(agent, finish_unlocking);  in unlock_duplicate_pbn()
710 static void start_unlocking(struct hash_lock *lock, struct data_vio *agent)  in start_unlocking()  argument
713 	launch_data_vio_duplicate_zone_callback(agent, unlock_duplicate_pbn);  in start_unlocking()
724 static void process_update_result(struct data_vio *agent)  in process_update_result()  argument
726 	struct dedupe_context *context = agent->dedupe_context;  in process_update_result()
732 	agent->dedupe_context = NULL;  in process_update_result()
744 	struct data_vio *agent = as_data_vio(completion);  in finish_updating()  local
745 	struct hash_lock *lock = agent->hash_lock;  in finish_updating()
747 	assert_hash_lock_agent(agent, __func__);  in finish_updating()
749 	process_update_result(agent);  in finish_updating()
763 		start_deduping(lock, agent, true);  in finish_updating()
772 		start_unlocking(lock, agent);  in finish_updating()
780 	start_bypassing(lock, agent);  in finish_updating()
791 static void start_updating(struct hash_lock *lock, struct data_vio *agent)  in start_updating()  argument
798 	agent->last_async_operation = VIO_ASYNC_OP_UPDATE_DEDUPE_INDEX;  in start_updating()
799 	set_data_vio_hash_zone_callback(agent, finish_updating);  in start_updating()
800 	query_index(agent, UDS_UPDATE);  in start_updating()
816 	struct data_vio *agent = data_vio;  in finish_deduping()  local
818 	VDO_ASSERT_LOG_ONLY(lock->agent == NULL, "shouldn't have an agent in DEDUPING");  in finish_deduping()
829 	lock->agent = agent;  in finish_deduping()
838 		start_updating(lock, agent);  in finish_deduping()
845 		start_unlocking(lock, agent);  in finish_deduping()
961 	new_lock->agent = new_agent;  in fork_hash_lock()
1004 static void start_deduping(struct hash_lock *lock, struct data_vio *agent,  in start_deduping()  argument
1014 		VDO_ASSERT_LOG_ONLY(!vdo_is_state_compressed(agent->new_mapped.state),  in start_deduping()
1018 		transfer_allocation_lock(agent);  in start_deduping()
1029 	lock->agent = NULL;  in start_deduping()
1037 		launch_dedupe(lock, agent, true);  in start_deduping()
1038 		agent = NULL;  in start_deduping()
1049 		finish_deduping(lock, agent);  in start_deduping()
1075 	struct data_vio *agent = as_data_vio(completion);  in finish_verifying()  local
1076 	struct hash_lock *lock = agent->hash_lock;  in finish_verifying()
1078 	assert_hash_lock_agent(agent, __func__);  in finish_verifying()
1080 	lock->verified = agent->is_duplicate;  in finish_verifying()
1089 			increment_stat(&agent->hash_zone->statistics.dedupe_advice_valid);  in finish_verifying()
1091 			increment_stat(&agent->hash_zone->statistics.dedupe_advice_stale);  in finish_verifying()
1099 		agent->is_duplicate = false;  in finish_verifying()
1108 		start_deduping(lock, agent, false);  in finish_verifying()
1118 		start_unlocking(lock, agent);  in finish_verifying()
1136 	struct data_vio *agent = as_data_vio(completion);  in verify_callback()  local
1138 	agent->is_duplicate = blocks_equal(agent->vio.data, agent->scratch_block);  in verify_callback()
1139 	launch_data_vio_hash_zone_callback(agent, finish_verifying);  in verify_callback()
1144 	struct data_vio *agent = as_data_vio(completion);  in uncompress_and_verify()  local
1147 	result = uncompress_data_vio(agent, agent->duplicate.state,  in uncompress_and_verify()
1148 				     agent->scratch_block);  in uncompress_and_verify()
1154 	agent->is_duplicate = false;  in uncompress_and_verify()
1155 	launch_data_vio_hash_zone_callback(agent, finish_verifying);  in uncompress_and_verify()
1160 	struct data_vio *agent = vio_as_data_vio(bio->bi_private);  in verify_endio()  local
1165 		agent->is_duplicate = false;  in verify_endio()
1166 		launch_data_vio_hash_zone_callback(agent, finish_verifying);  in verify_endio()
1170 	if (vdo_is_state_compressed(agent->duplicate.state)) {  in verify_endio()
1171 		launch_data_vio_cpu_callback(agent, uncompress_and_verify,  in verify_endio()
1176 	launch_data_vio_cpu_callback(agent, verify_callback,  in verify_endio()
1191 static void start_verifying(struct hash_lock *lock, struct data_vio *agent)  in start_verifying()  argument
1194 	struct vio *vio = &agent->vio;  in start_verifying()
1195 	char *buffer = (vdo_is_state_compressed(agent->duplicate.state) ?  in start_verifying()
1196 			(char *) agent->compression.block :  in start_verifying()
1197 			agent->scratch_block);  in start_verifying()
1202 	agent->last_async_operation = VIO_ASYNC_OP_VERIFY_DUPLICATION;  in start_verifying()
1204 			       agent->duplicate.pbn);  in start_verifying()
1206 		set_data_vio_hash_zone_callback(agent, finish_verifying);  in start_verifying()
1207 		continue_data_vio_with_error(agent, result);  in start_verifying()
1211 	set_data_vio_bio_zone_callback(agent, vdo_submit_vio);  in start_verifying()
1224 	struct data_vio *agent = as_data_vio(completion);  in finish_locking()  local
1225 	struct hash_lock *lock = agent->hash_lock;  in finish_locking()
1227 	assert_hash_lock_agent(agent, __func__);  in finish_locking()
1229 	if (!agent->is_duplicate) {  in finish_locking()
1237 		increment_stat(&agent->hash_zone->statistics.dedupe_advice_stale);  in finish_locking()
1239 		start_writing(lock, agent);  in finish_locking()
1252 		start_verifying(lock, agent);  in finish_locking()
1262 		agent->is_duplicate = false;  in finish_locking()
1265 		start_unlocking(lock, agent);  in finish_locking()
1273 	start_deduping(lock, agent, false);  in finish_locking()
1276 static bool acquire_provisional_reference(struct data_vio *agent, struct pbn_lock *lock,  in acquire_provisional_reference()  argument
1280 	struct vdo_slab *slab = vdo_get_slab(depot, agent->duplicate.pbn);  in acquire_provisional_reference()
1281 	int result = vdo_acquire_provisional_reference(slab, agent->duplicate.pbn, lock);  in acquire_provisional_reference()
1288 	agent->is_duplicate = false;  in acquire_provisional_reference()
1289 	vdo_release_physical_zone_pbn_lock(agent->duplicate.zone,  in acquire_provisional_reference()
1290 					   agent->duplicate.pbn, lock);  in acquire_provisional_reference()
1291 	continue_data_vio_with_error(agent, result);  in acquire_provisional_reference()
1311 	struct data_vio *agent = as_data_vio(completion);  in lock_duplicate_pbn()  local
1312 	struct slab_depot *depot = vdo_from_data_vio(agent)->depot;  in lock_duplicate_pbn()
1313 	struct physical_zone *zone = agent->duplicate.zone;  in lock_duplicate_pbn()
1315 	assert_data_vio_in_duplicate_zone(agent);  in lock_duplicate_pbn()
1317 	set_data_vio_hash_zone_callback(agent, finish_locking);  in lock_duplicate_pbn()
1323 	increment_limit = vdo_get_increment_limit(depot, agent->duplicate.pbn);  in lock_duplicate_pbn()
1329 		agent->is_duplicate = false;  in lock_duplicate_pbn()
1330 		continue_data_vio(agent);  in lock_duplicate_pbn()
1334 	result = vdo_attempt_physical_zone_pbn_lock(zone, agent->duplicate.pbn,  in lock_duplicate_pbn()
1337 		continue_data_vio_with_error(agent, result);  in lock_duplicate_pbn()
1377 		agent->is_duplicate = false;  in lock_duplicate_pbn()
1378 		continue_data_vio(agent);  in lock_duplicate_pbn()
1383 		if (!acquire_provisional_reference(agent, lock, depot))  in lock_duplicate_pbn()
1397 	set_duplicate_lock(agent->hash_lock, lock);  in lock_duplicate_pbn()
1403 	continue_data_vio(agent);  in lock_duplicate_pbn()
1412 static void start_locking(struct hash_lock *lock, struct data_vio *agent)  in start_locking()  argument
1424 	agent->last_async_operation = VIO_ASYNC_OP_LOCK_DUPLICATE_PBN;  in start_locking()
1425 	launch_data_vio_duplicate_zone_callback(agent, lock_duplicate_pbn);  in start_locking()
1442 static void finish_writing(struct hash_lock *lock, struct data_vio *agent)  in finish_writing()  argument
1448 	lock->duplicate = agent->new_mapped;  in finish_writing()
1467 		start_deduping(lock, agent, true);  in finish_writing()
1481 		start_updating(lock, agent);  in finish_writing()
1487 		set_duplicate_location(agent, lock->duplicate);  in finish_writing()
1488 		start_unlocking(lock, agent);  in finish_writing()
1495 		start_bypassing(lock, agent);  in finish_writing()
1534 		vdo_waitq_enqueue_waiter(&lock->waiters, &lock->agent->waiter);  in select_writing_agent()
1535 		lock->agent = data_vio;  in select_writing_agent()
1538 		data_vio = lock->agent;  in select_writing_agent()
1555 static void start_writing(struct hash_lock *lock, struct data_vio *agent)  in start_writing()  argument
1563 	if (!data_vio_has_allocation(agent)) {  in start_writing()
1564 		agent = select_writing_agent(lock);  in start_writing()
1566 		if (!data_vio_has_allocation(agent)) {  in start_writing()
1574 			continue_data_vio_with_error(agent, VDO_NO_SPACE);  in start_writing()
1584 		cancel_data_vio_compression(agent);  in start_writing()
1590 	launch_compress_data_vio(agent);  in start_writing()
1643 static void process_query_result(struct data_vio *agent)  in process_query_result()  argument
1645 	struct dedupe_context *context = agent->dedupe_context;  in process_query_result()
1651 		agent->is_duplicate = decode_uds_advice(context);  in process_query_result()
1652 		agent->dedupe_context = NULL;  in process_query_result()
1665 	struct data_vio *agent = as_data_vio(completion);  in finish_querying()  local
1666 	struct hash_lock *lock = agent->hash_lock;  in finish_querying()
1668 	assert_hash_lock_agent(agent, __func__);  in finish_querying()
1670 	process_query_result(agent);  in finish_querying()
1672 	if (agent->is_duplicate) {  in finish_querying()
1673 		lock->duplicate = agent->duplicate;  in finish_querying()
1679 		start_locking(lock, agent);  in finish_querying()
1685 		lock->update_advice = !data_vio_has_allocation(agent);  in finish_querying()
1690 		start_writing(lock, agent);  in finish_querying()
1705 	lock->agent = data_vio;  in start_querying()
1745 		VDO_ASSERT_LOG_ONLY(data_vio == lock->agent,  in vdo_continue_hash_lock()
2757 		     vdo_waitq_num_waiters(&lock->waiters), lock->agent);  in dump_hash_lock()