18 #include "completion.h"
101 	struct vdo_completion *completion;  member
165 	struct vdo_page_completion *completion;  in page_completion_from_waiter()  local
170 	completion = container_of(waiter, struct vdo_page_completion, waiter);  in page_completion_from_waiter()
171 	vdo_assert_completion_type(&completion->completion, VDO_PAGE_COMPLETION);  in page_completion_from_waiter()
172 	return completion;  in page_completion_from_waiter()
200 		info->vio->completion.callback_thread_id = cache->zone->thread_id;  in initialize_info()
484  * complete_with_page() - Helper to complete the VDO Page Completion request successfully.
486  * @vdo_page_comp: The VDO page completion to complete.
499 		vdo_fail_completion(&vdo_page_comp->completion, VDO_BAD_PAGE);  in complete_with_page()
505 	vdo_finish_completion(&vdo_page_comp->completion);  in complete_with_page()
509  * complete_waiter_with_error() - Complete a page completion with an error code.
510  * @waiter: The page completion, as a waiter.
519 	vdo_fail_completion(&page_completion_from_waiter(waiter)->completion, *result);  in complete_waiter_with_error()
523  * complete_waiter_with_page() - Complete a page completion with a page.
524  * @waiter: The page completion, as a waiter.
539  * Upon completion the waitq will be empty.
552 	 * Increment the busy count once for each pending completion so that this page does not  in distribute_page_over_waitq()
596  * validate_completed_page() - Check that a page completion which is being freed to the cache
598  * @completion: The page completion to check.
603 static int __must_check validate_completed_page(struct vdo_page_completion *completion,  in validate_completed_page()  argument
608 	result = VDO_ASSERT(completion->ready, "VDO Page completion not ready");  in validate_completed_page()
612 	result = VDO_ASSERT(completion->info != NULL,  in validate_completed_page()
613 			    "VDO Page Completion must be complete");  in validate_completed_page()
617 	result = VDO_ASSERT(completion->info->pbn == completion->pbn,  in validate_completed_page()
618 			    "VDO Page Completion pbn must be consistent");  in validate_completed_page()
622 	result = VDO_ASSERT(is_valid(completion->info),  in validate_completed_page()
623 			    "VDO Page Completion page must be valid");  in validate_completed_page()
628 		result = VDO_ASSERT(completion->writable,  in validate_completed_page()
629 				    "VDO Page Completion must be writable");  in validate_completed_page()
664 validate_completed_page_or_enter_read_only_mode(struct vdo_page_completion *completion,  in validate_completed_page_or_enter_read_only_mode()  argument
667 	int result = validate_completed_page(completion, writable);  in validate_completed_page_or_enter_read_only_mode()
672 	enter_zone_read_only_mode(completion->info->cache->zone, result);  in validate_completed_page_or_enter_read_only_mode()
678  * @completion: The page read vio.
680 static void handle_load_error(struct vdo_completion *completion)  in handle_load_error()  argument
682 	int result = completion->result;  in handle_load_error()
683 	struct page_info *info = completion->parent;  in handle_load_error()
687 	vio_record_metadata_io_error(as_vio(completion));  in handle_load_error()
704  * @completion: The vio which has loaded the page. Its parent is the page_info.
706 static void page_is_loaded(struct vdo_completion *completion)  in page_is_loaded()  argument
708 	struct page_info *info = completion->parent;  in page_is_loaded()
725 		vdo_continue_completion(completion, result);  in page_is_loaded()
746  * @completion: The page load completion.
748 static void handle_rebuild_read_error(struct vdo_completion *completion)  in handle_rebuild_read_error()  argument
750 	struct page_info *info = completion->parent;  in handle_rebuild_read_error()
759 	vio_record_metadata_io_error(as_vio(completion));  in handle_rebuild_read_error()
762 	vdo_reset_completion(completion);  in handle_rebuild_read_error()
763 	page_is_loaded(completion);  in handle_rebuild_read_error()
769 	struct page_info *info = vio->completion.parent;  in load_cache_page_endio()
807 static void write_pages(struct vdo_completion *completion);
810 static void handle_flush_error(struct vdo_completion *completion)  in handle_flush_error()  argument
812 	struct page_info *info = completion->parent;  in handle_flush_error()
814 	vio_record_metadata_io_error(as_vio(completion));  in handle_flush_error()
815 	set_persistent_error(info->cache, "flush failed", completion->result);  in handle_flush_error()
816 	write_pages(completion);  in handle_flush_error()
822 	struct page_info *info = vio->completion.parent;  in flush_endio()
886  * @waiter: The page completion waiter to check.
891  * Return: true if the page completion is for the desired page number.
901  * allocate_free_page() - Allocate a free page to the first completion in the waiting queue, and
955  * If the selected page is not dirty, immediately allocates the page to the oldest completion
1019  * @completion: The page write vio.
1021 static void handle_page_write_error(struct vdo_completion *completion)  in handle_page_write_error()  argument
1023 	int result = completion->result;  in handle_page_write_error()
1024 	struct page_info *info = completion->parent;  in handle_page_write_error()
1027 	vio_record_metadata_io_error(as_vio(completion));  in handle_page_write_error()
1046 static void page_is_written_out(struct vdo_completion *completion);
1051 	struct page_info *info = vio->completion.parent;  in write_cache_page_endio()
1058  * @completion: The vio which wrote the page. Its parent is a page_info.
1060 static void page_is_written_out(struct vdo_completion *completion)  in page_is_written_out()  argument
1064 	struct page_info *info = completion->parent;  in page_is_written_out()
1129 			struct vdo_completion *completion = &info->vio->completion;  in write_pages()  local
1131 			vdo_reset_completion(completion);  in write_pages()
1132 			completion->callback = page_is_written_out;  in write_pages()
1133 			completion->error_handler = handle_page_write_error;  in write_pages()
1134 			vdo_fail_completion(completion, VDO_READ_ONLY);  in write_pages()
1152  * vdo_release_page_completion() - Release a VDO Page Completion.
1153  * @completion: The page completion to release.
1155  * The page referenced by this completion (if any) will no longer be held busy by this completion.
1159 void vdo_release_page_completion(struct vdo_completion *completion)  in vdo_release_page_completion()  argument
1162 	struct vdo_page_completion *page_completion = as_vdo_page_completion(completion);  in vdo_release_page_completion()
1165 	if (completion->result == VDO_SUCCESS) {  in vdo_release_page_completion()
1208  * vdo_get_page() - Initialize a page completion and get a block map page.
1219  * by the completion to be loaded from disk. When the callback is invoked, the page will be
1229 	struct vdo_completion *completion = &page_completion->completion;  in vdo_get_page()  local
1234 			    "New page completion was not already on a wait queue");  in vdo_get_page()
1242 	vdo_initialize_completion(completion, cache->vdo, VDO_PAGE_COMPLETION);  in vdo_get_page()
1243 	vdo_prepare_completion(completion, callback, error_handler,  in vdo_get_page()
1245 	completion->requeue = requeue;  in vdo_get_page()
1248 		vdo_fail_completion(completion, VDO_READ_ONLY);  in vdo_get_page()
1299  * @completion: The vdo_page_completion containing the page.
1301 void vdo_request_page_write(struct vdo_completion *completion)  in vdo_request_page_write()  argument
1304 	struct vdo_page_completion *vdo_page_comp = as_vdo_page_completion(completion);  in vdo_request_page_write()
1315  * vdo_get_cached_page() - Get the block map page from a page completion.
1316  * @completion: A vdo page completion whose callback has been called.
1321 int vdo_get_cached_page(struct vdo_completion *completion,  in vdo_get_cached_page()  argument
1327 	vpc = as_vdo_page_completion(completion);  in vdo_get_cached_page()
1572 static void finish_page_write(struct vdo_completion *completion)  in finish_page_write()  argument
1575 	struct vio *vio = as_vio(completion);  in finish_page_write()
1577 	struct tree_page *page = completion->parent;  in finish_page_write()
1618 static void handle_write_error(struct vdo_completion *completion)  in handle_write_error()  argument
1620 	int result = completion->result;  in handle_write_error()
1621 	struct vio *vio = as_vio(completion);  in handle_write_error()
1632 static void write_initialized_page(struct vdo_completion *completion)  in write_initialized_page()  argument
1634 	struct vio *vio = as_vio(completion);  in write_initialized_page()
1637 	struct tree_page *tree_page = completion->parent;  in write_initialized_page()
1669 	struct vdo_completion *completion = &vio->vio.completion;  in write_page()  local
1684 	completion->parent = tree_page;  in write_page()
1686 	completion->callback_thread_id = zone->thread_id;  in write_page()
1702 		write_initialized_page(completion);  in write_page()
1738 	data_vio->vio.completion.error_handler = handle_data_vio_error;  in finish_lookup()
1840 static void finish_block_map_page_load(struct vdo_completion *completion)  in finish_block_map_page_load()  argument
1846 	struct vio *vio = as_vio(completion);  in finish_block_map_page_load()
1848 	struct data_vio *data_vio = completion->parent;  in finish_block_map_page_load()
1868 static void handle_io_error(struct vdo_completion *completion)  in handle_io_error()  argument
1870 	int result = completion->result;  in handle_io_error()
1871 	struct vio *vio = as_vio(completion);  in handle_io_error()
1873 	struct data_vio *data_vio = completion->parent;  in handle_io_error()
1883 	struct data_vio *data_vio = vio->completion.parent;  in load_page_endio()
1896 	pooled->vio.completion.parent = data_vio;  in load_page()
1955 static void allocation_failure(struct vdo_completion *completion)  in allocation_failure()  argument
1957 	struct data_vio *data_vio = as_data_vio(completion);  in allocation_failure()
1959 	if (vdo_requeue_completion_if_needed(completion,  in allocation_failure()
1963 	abort_lookup(data_vio, completion->result, "allocation");  in allocation_failure()
2084 static void finish_block_map_allocation(struct vdo_completion *completion)  in finish_block_map_allocation()  argument
2090 	struct data_vio *data_vio = as_data_vio(completion);  in finish_block_map_allocation()
2145 static void release_block_map_write_lock(struct vdo_completion *completion)  in release_block_map_write_lock()  argument
2147 	struct data_vio *data_vio = as_data_vio(completion);  in release_block_map_write_lock()
2160 static void set_block_map_page_reference_count(struct vdo_completion *completion)  in set_block_map_page_reference_count()  argument
2162 	struct data_vio *data_vio = as_data_vio(completion);  in set_block_map_page_reference_count()
2166 	completion->callback = release_block_map_write_lock;  in set_block_map_page_reference_count()
2167 	vdo_modify_reference_count(completion, &data_vio->increment_updater);  in set_block_map_page_reference_count()
2170 static void journal_block_map_allocation(struct vdo_completion *completion)  in journal_block_map_allocation()  argument
2172 	struct data_vio *data_vio = as_data_vio(completion);  in journal_block_map_allocation()
2178 	vdo_add_recovery_journal_entry(completion->vdo->recovery_journal, data_vio);  in journal_block_map_allocation()
2181 static void allocate_block(struct vdo_completion *completion)  in allocate_block()  argument
2183 	struct data_vio *data_vio = as_data_vio(completion);  in allocate_block()
2523 	struct vdo_completion *completion = cursors->completion;  in finish_cursor()  local
2531 	vdo_finish_completion(completion);  in finish_cursor()
2538  * @completion: The VIO doing a read or write.
2540 static void continue_traversal(struct vdo_completion *completion)  in continue_traversal()  argument
2542 	vio_record_metadata_io_error(as_vio(completion));  in continue_traversal()
2543 	traverse(completion->parent);  in continue_traversal()
2548  * @completion: The VIO doing the read.
2550 static void finish_traversal_load(struct vdo_completion *completion)  in finish_traversal_load()  argument
2552 	struct cursor *cursor = completion->parent;  in finish_traversal_load()
2568 	struct cursor *cursor = vio->completion.parent;  in traversal_endio()
2618 									    cursor->parent->completion);  in traverse()
2658 	pooled->vio.completion.parent = cursor;  in launch_cursor()
2659 	pooled->vio.completion.callback_thread_id = cursor->parent->zone->thread_id;  in launch_cursor()
2701  * @completion: The completion to notify on each traversed PBN, and when traversal completes.
2704 			 struct vdo_completion *completion)  in vdo_traverse_forest()  argument
2713 		vdo_fail_completion(completion, result);  in vdo_traverse_forest()
2720 	cursors->completion = completion;  in vdo_traverse_forest()
3069 static void grow_forest(void *context, struct vdo_completion *completion)  in grow_forest()  argument
3072 	vdo_finish_completion(completion);  in grow_forest()
3093 /* Release the page completion and then continue the requester. */
3094 static inline void finish_processing_page(struct vdo_completion *completion, int result)  in finish_processing_page()  argument
3096 	struct vdo_completion *parent = completion->parent;  in finish_processing_page()
3098 	vdo_release_page_completion(completion);  in finish_processing_page()
3102 static void handle_page_error(struct vdo_completion *completion)  in handle_page_error()  argument
3104 	finish_processing_page(completion, completion->result);  in handle_page_error()
3120 		     modifiable, &data_vio->vio.completion,  in fetch_mapping_page()
3195 static void get_mapping_from_fetched_page(struct vdo_completion *completion)  in get_mapping_from_fetched_page()  argument
3198 	struct vdo_page_completion *vpc = as_vdo_page_completion(completion);  in get_mapping_from_fetched_page()
3201 	struct data_vio *data_vio = as_data_vio(completion->parent);  in get_mapping_from_fetched_page()
3204 	if (completion->result != VDO_SUCCESS) {  in get_mapping_from_fetched_page()
3205 		finish_processing_page(completion, completion->result);  in get_mapping_from_fetched_page()
3211 		finish_processing_page(completion, result);  in get_mapping_from_fetched_page()
3220 	finish_processing_page(completion, result);  in get_mapping_from_fetched_page()
3264 static void put_mapping_in_fetched_page(struct vdo_completion *completion)  in put_mapping_in_fetched_page()  argument
3266 	struct data_vio *data_vio = as_data_vio(completion->parent);  in put_mapping_in_fetched_page()
3272 	if (completion->result != VDO_SUCCESS) {  in put_mapping_in_fetched_page()
3273 		finish_processing_page(completion, completion->result);  in put_mapping_in_fetched_page()
3277 	vpc = as_vdo_page_completion(completion);  in put_mapping_in_fetched_page()
3280 		finish_processing_page(completion, result);  in put_mapping_in_fetched_page()
3292 	finish_processing_page(completion, VDO_SUCCESS);  in put_mapping_in_fetched_page()