176 struct tcmu_cmd {  struct
497 static void tcmu_cmd_free_data(struct tcmu_cmd *tcmu_cmd, uint32_t len)  in tcmu_cmd_free_data()  argument
499 	struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;  in tcmu_cmd_free_data()
503 		clear_bit(tcmu_cmd->dbi[i], udev->data_bitmap);  in tcmu_cmd_free_data()
507 				       struct tcmu_cmd *tcmu_cmd,  in tcmu_get_empty_block()  argument
546 	tcmu_cmd_set_dbi(tcmu_cmd, dbi);  in tcmu_get_empty_block()
555 				 struct tcmu_cmd *tcmu_cmd, int length)  in tcmu_get_empty_blocks()  argument
564 		dbi = tcmu_get_empty_block(udev, tcmu_cmd, dbi, blk_data_len,  in tcmu_get_empty_blocks()
572 static inline void tcmu_free_cmd(struct tcmu_cmd *tcmu_cmd)  in tcmu_free_cmd()  argument
574 	kfree(tcmu_cmd->dbi);  in tcmu_free_cmd()
575 	kmem_cache_free(tcmu_cmd_cache, tcmu_cmd);  in tcmu_free_cmd()
578 static inline void tcmu_cmd_set_block_cnts(struct tcmu_cmd *cmd)  in tcmu_cmd_set_block_cnts()
596 static int new_block_to_iov(struct tcmu_dev *udev, struct tcmu_cmd *cmd,  in new_block_to_iov()
622 static void tcmu_setup_iovs(struct tcmu_dev *udev, struct tcmu_cmd *cmd,  in tcmu_setup_iovs()
633 static struct tcmu_cmd *tcmu_alloc_cmd(struct se_cmd *se_cmd)  in tcmu_alloc_cmd()
637 	struct tcmu_cmd *tcmu_cmd;  in tcmu_alloc_cmd()  local
639 	tcmu_cmd = kmem_cache_zalloc(tcmu_cmd_cache, GFP_NOIO);  in tcmu_alloc_cmd()
640 	if (!tcmu_cmd)  in tcmu_alloc_cmd()
643 	INIT_LIST_HEAD(&tcmu_cmd->queue_entry);  in tcmu_alloc_cmd()
644 	tcmu_cmd->se_cmd = se_cmd;  in tcmu_alloc_cmd()
645 	tcmu_cmd->tcmu_dev = udev;  in tcmu_alloc_cmd()
647 	tcmu_cmd_set_block_cnts(tcmu_cmd);  in tcmu_alloc_cmd()
648 	tcmu_cmd->dbi = kcalloc(tcmu_cmd->dbi_cnt, sizeof(uint32_t),  in tcmu_alloc_cmd()
650 	if (!tcmu_cmd->dbi) {  in tcmu_alloc_cmd()
651 		kmem_cache_free(tcmu_cmd_cache, tcmu_cmd);  in tcmu_alloc_cmd()
655 	return tcmu_cmd;  in tcmu_alloc_cmd()
703 				  struct tcmu_cmd *tcmu_cmd, uint32_t direction,  in tcmu_copy_data()  argument
724 			dbi = new_block_to_iov(udev, tcmu_cmd, iov, dbi,  in tcmu_copy_data()
727 			dbi = tcmu_cmd_get_dbi(tcmu_cmd);  in tcmu_copy_data()
774 static void scatter_data_area(struct tcmu_dev *udev, struct tcmu_cmd *tcmu_cmd,  in scatter_data_area()  argument
777 	struct se_cmd *se_cmd = tcmu_cmd->se_cmd;  in scatter_data_area()
779 	tcmu_copy_data(udev, tcmu_cmd, TCMU_SG_TO_DATA_AREA, se_cmd->t_data_sg,  in scatter_data_area()
783 static void gather_data_area(struct tcmu_dev *udev, struct tcmu_cmd *tcmu_cmd,  in gather_data_area()  argument
786 	struct se_cmd *se_cmd = tcmu_cmd->se_cmd;  in gather_data_area()
799 		tcmu_cmd_set_dbi_cur(tcmu_cmd,  in gather_data_area()
800 				     tcmu_cmd->dbi_cnt - tcmu_cmd->dbi_bidi_cnt);  in gather_data_area()
806 	tcmu_copy_data(udev, tcmu_cmd, TCMU_DATA_AREA_TO_SG, data_sg,  in gather_data_area()
854 static int tcmu_alloc_data_space(struct tcmu_dev *udev, struct tcmu_cmd *cmd,  in tcmu_alloc_data_space()
900 static inline size_t tcmu_cmd_get_cmd_size(struct tcmu_cmd *tcmu_cmd,  in tcmu_cmd_get_cmd_size()  argument
903 	struct se_cmd *se_cmd = tcmu_cmd->se_cmd;  in tcmu_cmd_get_cmd_size()
915 static void tcmu_setup_cmd_timer(struct tcmu_cmd *tcmu_cmd, unsigned int tmo,  in tcmu_setup_cmd_timer()  argument
921 	tcmu_cmd->deadline = round_jiffies_up(jiffies + msecs_to_jiffies(tmo));  in tcmu_setup_cmd_timer()
923 		mod_timer(timer, tcmu_cmd->deadline);  in tcmu_setup_cmd_timer()
925 	pr_debug("Timeout set up for cmd %p, dev = %s, tmo = %lu\n", tcmu_cmd,  in tcmu_setup_cmd_timer()
926 		 tcmu_cmd->tcmu_dev->name, tmo / MSEC_PER_SEC);  in tcmu_setup_cmd_timer()
929 static int add_to_qfull_queue(struct tcmu_cmd *tcmu_cmd)  in add_to_qfull_queue()  argument
931 	struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;  in add_to_qfull_queue()
947 	tcmu_setup_cmd_timer(tcmu_cmd, tmo, &udev->qfull_timer);  in add_to_qfull_queue()
949 	list_add_tail(&tcmu_cmd->queue_entry, &udev->qfull_queue);  in add_to_qfull_queue()
951 		 tcmu_cmd, udev->name);  in add_to_qfull_queue()
1012 static int queue_cmd_ring(struct tcmu_cmd *tcmu_cmd, sense_reason_t *scsi_err)  in queue_cmd_ring()  argument
1014 	struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;  in queue_cmd_ring()
1015 	struct se_cmd *se_cmd = tcmu_cmd->se_cmd;  in queue_cmd_ring()
1025 	size_t data_length = (size_t)tcmu_cmd->dbi_cnt * blk_size;  in queue_cmd_ring()
1049 	iov_cnt = tcmu_alloc_data_space(udev, tcmu_cmd, &iov_bidi_cnt);  in queue_cmd_ring()
1058 	command_size = tcmu_cmd_get_cmd_size(tcmu_cmd, base_command_size);  in queue_cmd_ring()
1063 		tcmu_cmd_free_data(tcmu_cmd, tcmu_cmd->dbi_cur);  in queue_cmd_ring()
1075 	if (xa_alloc(&udev->commands, &cmd_id, tcmu_cmd, XA_LIMIT(1, 0xffff),  in queue_cmd_ring()
1079 		tcmu_cmd_free_data(tcmu_cmd, tcmu_cmd->dbi_cnt);  in queue_cmd_ring()
1083 	tcmu_cmd->cmd_id = cmd_id;  in queue_cmd_ring()
1085 	pr_debug("allocated cmd id %u for cmd %p dev %s\n", tcmu_cmd->cmd_id,  in queue_cmd_ring()
1086 		 tcmu_cmd, udev->name);  in queue_cmd_ring()
1095 	tcmu_cmd_reset_dbi_cur(tcmu_cmd);  in queue_cmd_ring()
1100 		scatter_data_area(udev, tcmu_cmd, &iov);  in queue_cmd_ring()
1102 		tcmu_setup_iovs(udev, tcmu_cmd, &iov, se_cmd->data_length);  in queue_cmd_ring()
1109 		tcmu_setup_iovs(udev, tcmu_cmd, &iov, tcmu_cmd->data_len_bidi);  in queue_cmd_ring()
1113 	tcmu_setup_cmd_timer(tcmu_cmd, udev->cmd_time_out, &udev->cmd_timer);  in queue_cmd_ring()
1115 	entry->hdr.cmd_id = tcmu_cmd->cmd_id;  in queue_cmd_ring()
1128 	list_add_tail(&tcmu_cmd->queue_entry, &udev->inflight_queue);  in queue_cmd_ring()
1136 	tcmu_cmd_free_data(tcmu_cmd, tcmu_cmd->dbi_cur);  in queue_cmd_ring()
1137 	tcmu_cmd_reset_dbi_cur(tcmu_cmd);  in queue_cmd_ring()
1140 	if (add_to_qfull_queue(tcmu_cmd)) {  in queue_cmd_ring()
1207 	struct tcmu_cmd *tcmu_cmd;  in tcmu_queue_cmd()  local
1211 	tcmu_cmd = tcmu_alloc_cmd(se_cmd);  in tcmu_queue_cmd()
1212 	if (!tcmu_cmd)  in tcmu_queue_cmd()
1217 		ret = queue_cmd_ring(tcmu_cmd, &scsi_ret);  in tcmu_queue_cmd()
1219 		tcmu_free_cmd(tcmu_cmd);  in tcmu_queue_cmd()
1221 		se_cmd->priv = tcmu_cmd;  in tcmu_queue_cmd()
1229 	struct tcmu_cmd *cmd;  in tcmu_set_next_deadline()
1232 		cmd = list_first_entry(queue, struct tcmu_cmd, queue_entry);  in tcmu_set_next_deadline()
1260 	struct tcmu_cmd *cmd;  in tcmu_tmr_notify()
1320 static bool tcmu_handle_completion(struct tcmu_cmd *cmd,  in tcmu_handle_completion()
1446 	struct tcmu_cmd *cmd;  in tcmu_handle_completions()
1518 static void tcmu_check_expired_ring_cmd(struct tcmu_cmd *cmd)  in tcmu_check_expired_ring_cmd()
1537 static void tcmu_check_expired_queue_cmd(struct tcmu_cmd *cmd)  in tcmu_check_expired_queue_cmd()
1652 static int tcmu_check_and_free_pending_cmd(struct tcmu_cmd *cmd)  in tcmu_check_and_free_pending_cmd()
1716 	struct tcmu_cmd *cmd;  in tcmu_dev_kref_release()
1752 	struct tcmu_cmd *tcmu_cmd, *tmp_cmd;  in run_qfull_queue()  local
1764 	list_for_each_entry_safe(tcmu_cmd, tmp_cmd, &cmds, queue_entry) {  in run_qfull_queue()
1765 		list_del_init(&tcmu_cmd->queue_entry);  in run_qfull_queue()
1768 			 tcmu_cmd, udev->name);  in run_qfull_queue()
1778 			tcmu_cmd->se_cmd->priv = NULL;  in run_qfull_queue()
1779 			target_complete_cmd(tcmu_cmd->se_cmd, SAM_STAT_BUSY);  in run_qfull_queue()
1780 			tcmu_free_cmd(tcmu_cmd);  in run_qfull_queue()
1784 		ret = queue_cmd_ring(tcmu_cmd, &scsi_ret);  in run_qfull_queue()
1787 				 tcmu_cmd, udev->name, scsi_ret);  in run_qfull_queue()
1792 			tcmu_cmd->se_cmd->priv = NULL;  in run_qfull_queue()
1793 			target_complete_cmd(tcmu_cmd->se_cmd,  in run_qfull_queue()
1795 			tcmu_free_cmd(tcmu_cmd);  in run_qfull_queue()
1963 	struct tcmu_cmd *cmd;  in tcmu_release()
2365 	struct tcmu_cmd *cmd;  in tcmu_reset_ring()
3088 	struct tcmu_cmd *cmd;  in tcmu_free_kept_buf_store()
3266 	struct tcmu_cmd *cmd, *tmp_cmd;  in check_timedout_devices()
3320 				sizeof(struct tcmu_cmd),  in tcmu_module_init()
3321 				__alignof__(struct tcmu_cmd),  in tcmu_module_init()