Lines Matching +full:single +full:- +full:tt
1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * iSCSI over TCP/IP Data-Path lib
7 * Copyright (C) 2005 - 2006 Mike Christie
9 * maintained by open-iscsi@googlegroups.com
42 MODULE_DESCRIPTION("iSCSI/TCP data-path");
59 &(_conn)->cls_conn->dev, \
80 * iscsi_tcp_segment_init_sg - init indicated scatterlist entry
93 segment->sg = sg; in iscsi_tcp_segment_init_sg()
94 segment->sg_offset = offset; in iscsi_tcp_segment_init_sg()
95 segment->size = min(sg->length - offset, in iscsi_tcp_segment_init_sg()
96 segment->total_size - segment->total_copied); in iscsi_tcp_segment_init_sg()
97 segment->data = NULL; in iscsi_tcp_segment_init_sg()
101 * iscsi_tcp_segment_map - map the current S/G page
113 if (segment->data != NULL || !segment->sg) in iscsi_tcp_segment_map()
116 sg = segment->sg; in iscsi_tcp_segment_map()
117 BUG_ON(segment->sg_mapped); in iscsi_tcp_segment_map()
118 BUG_ON(sg->length == 0); in iscsi_tcp_segment_map()
128 * coalescing neighboring slab objects into a single frag which in iscsi_tcp_segment_map()
135 segment->atomic_mapped = true; in iscsi_tcp_segment_map()
136 segment->sg_mapped = kmap_atomic(sg_page(sg)); in iscsi_tcp_segment_map()
138 segment->atomic_mapped = false; in iscsi_tcp_segment_map()
140 segment->sg_mapped = kmap(sg_page(sg)); in iscsi_tcp_segment_map()
143 segment->data = segment->sg_mapped + sg->offset + segment->sg_offset; in iscsi_tcp_segment_map()
148 if (segment->sg_mapped) { in iscsi_tcp_segment_unmap()
149 if (segment->atomic_mapped) in iscsi_tcp_segment_unmap()
150 kunmap_atomic(segment->sg_mapped); in iscsi_tcp_segment_unmap()
152 kunmap(sg_page(segment->sg)); in iscsi_tcp_segment_unmap()
153 segment->sg_mapped = NULL; in iscsi_tcp_segment_unmap()
154 segment->data = NULL; in iscsi_tcp_segment_unmap()
165 segment->data = digest; in iscsi_tcp_segment_splice_digest()
166 segment->digest_len = ISCSI_DIGEST_SIZE; in iscsi_tcp_segment_splice_digest()
167 segment->total_size += ISCSI_DIGEST_SIZE; in iscsi_tcp_segment_splice_digest()
168 segment->size = ISCSI_DIGEST_SIZE; in iscsi_tcp_segment_splice_digest()
169 segment->copied = 0; in iscsi_tcp_segment_splice_digest()
170 segment->sg = NULL; in iscsi_tcp_segment_splice_digest()
171 segment->hash = NULL; in iscsi_tcp_segment_splice_digest()
175 * iscsi_tcp_segment_done - check whether the segment is complete
188 * This function must be re-entrant.
197 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "copied %u %u size %u %s\n", in iscsi_tcp_segment_done()
198 segment->copied, copied, segment->size, in iscsi_tcp_segment_done()
200 if (segment->hash && copied) { in iscsi_tcp_segment_done()
207 if (!segment->data) { in iscsi_tcp_segment_done()
209 sg_set_page(&sg, sg_page(segment->sg), copied, in iscsi_tcp_segment_done()
210 segment->copied + segment->sg_offset + in iscsi_tcp_segment_done()
211 segment->sg->offset); in iscsi_tcp_segment_done()
213 sg_init_one(&sg, segment->data + segment->copied, in iscsi_tcp_segment_done()
215 ahash_request_set_crypt(segment->hash, &sg, NULL, copied); in iscsi_tcp_segment_done()
216 crypto_ahash_update(segment->hash); in iscsi_tcp_segment_done()
219 segment->copied += copied; in iscsi_tcp_segment_done()
220 if (segment->copied < segment->size) { in iscsi_tcp_segment_done()
225 segment->total_copied += segment->copied; in iscsi_tcp_segment_done()
226 segment->copied = 0; in iscsi_tcp_segment_done()
227 segment->size = 0; in iscsi_tcp_segment_done()
233 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "total copied %u total size %u\n", in iscsi_tcp_segment_done()
234 segment->total_copied, segment->total_size); in iscsi_tcp_segment_done()
235 if (segment->total_copied < segment->total_size) { in iscsi_tcp_segment_done()
237 iscsi_tcp_segment_init_sg(segment, sg_next(segment->sg), in iscsi_tcp_segment_done()
240 BUG_ON(segment->size == 0); in iscsi_tcp_segment_done()
245 if (!(tcp_conn->iscsi_conn->session->tt->caps & CAP_PADDING_OFFLOAD)) { in iscsi_tcp_segment_done()
246 pad = iscsi_padding(segment->total_copied); in iscsi_tcp_segment_done()
248 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, in iscsi_tcp_segment_done()
250 segment->total_size += pad; in iscsi_tcp_segment_done()
251 segment->size = pad; in iscsi_tcp_segment_done()
252 segment->data = segment->padbuf; in iscsi_tcp_segment_done()
261 if (segment->hash) { in iscsi_tcp_segment_done()
262 ahash_request_set_crypt(segment->hash, NULL, in iscsi_tcp_segment_done()
263 segment->digest, 0); in iscsi_tcp_segment_done()
264 crypto_ahash_final(segment->hash); in iscsi_tcp_segment_done()
266 recv ? segment->recv_digest : segment->digest); in iscsi_tcp_segment_done()
275 * iscsi_tcp_segment_recv - copy data to segment
300 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, in iscsi_tcp_segment_recv()
305 copy = min(len - copied, segment->size - segment->copied); in iscsi_tcp_segment_recv()
306 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "copying %d\n", copy); in iscsi_tcp_segment_recv()
307 memcpy(segment->data + segment->copied, ptr + copied, copy); in iscsi_tcp_segment_recv()
329 if (!segment->digest_len) in iscsi_tcp_dgst_verify()
332 if (memcmp(segment->recv_digest, segment->digest, in iscsi_tcp_dgst_verify()
333 segment->digest_len)) { in iscsi_tcp_dgst_verify()
334 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "digest mismatch\n"); in iscsi_tcp_dgst_verify()
349 segment->total_size = size; in __iscsi_segment_init()
350 segment->done = done; in __iscsi_segment_init()
353 segment->hash = hash; in __iscsi_segment_init()
364 segment->data = data; in iscsi_segment_init_linear()
365 segment->size = size; in iscsi_segment_init_linear()
381 if (offset < sg->length) { in iscsi_segment_seek_sg()
385 offset -= sg->length; in iscsi_segment_seek_sg()
393 * iscsi_tcp_hdr_recv_prep - prep segment for hdr reception
402 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, in iscsi_tcp_hdr_recv_prep()
403 "(%s)\n", tcp_conn->iscsi_conn->hdrdgst_en ? in iscsi_tcp_hdr_recv_prep()
405 iscsi_segment_init_linear(&tcp_conn->in.segment, in iscsi_tcp_hdr_recv_prep()
406 tcp_conn->in.hdr_buf, sizeof(struct iscsi_hdr), in iscsi_tcp_hdr_recv_prep()
418 struct iscsi_conn *conn = tcp_conn->iscsi_conn; in iscsi_tcp_data_recv_done()
424 rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, in iscsi_tcp_data_recv_done()
425 conn->data, tcp_conn->in.datalen); in iscsi_tcp_data_recv_done()
436 struct iscsi_conn *conn = tcp_conn->iscsi_conn; in iscsi_tcp_data_recv_prep()
439 if (conn->datadgst_en && in iscsi_tcp_data_recv_prep()
440 !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD)) in iscsi_tcp_data_recv_prep()
441 rx_hash = tcp_conn->rx_hash; in iscsi_tcp_data_recv_prep()
443 iscsi_segment_init_linear(&tcp_conn->in.segment, in iscsi_tcp_data_recv_prep()
444 conn->data, tcp_conn->in.datalen, in iscsi_tcp_data_recv_prep()
449 * iscsi_tcp_cleanup_task - free tcp_task resources
456 struct iscsi_tcp_task *tcp_task = task->dd_data; in iscsi_tcp_cleanup_task()
460 if (!task->sc) in iscsi_tcp_cleanup_task()
463 spin_lock_bh(&tcp_task->queue2pool); in iscsi_tcp_cleanup_task()
465 while (kfifo_out(&tcp_task->r2tqueue, (void*)&r2t, sizeof(void*))) { in iscsi_tcp_cleanup_task()
466 kfifo_in(&tcp_task->r2tpool.queue, (void*)&r2t, in iscsi_tcp_cleanup_task()
468 ISCSI_DBG_TCP(task->conn, "pending r2t dropped\n"); in iscsi_tcp_cleanup_task()
471 r2t = tcp_task->r2t; in iscsi_tcp_cleanup_task()
473 kfifo_in(&tcp_task->r2tpool.queue, (void*)&r2t, in iscsi_tcp_cleanup_task()
475 tcp_task->r2t = NULL; in iscsi_tcp_cleanup_task()
477 spin_unlock_bh(&tcp_task->queue2pool); in iscsi_tcp_cleanup_task()
482 * iscsi_tcp_data_in - SCSI Data-In Response processing
488 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; in iscsi_tcp_data_in()
489 struct iscsi_tcp_task *tcp_task = task->dd_data; in iscsi_tcp_data_in()
490 struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)tcp_conn->in.hdr; in iscsi_tcp_data_in()
491 int datasn = be32_to_cpu(rhdr->datasn); in iscsi_tcp_data_in()
492 unsigned total_in_length = task->sc->sdb.length; in iscsi_tcp_data_in()
498 if (!(rhdr->flags & ISCSI_FLAG_DATA_STATUS)) in iscsi_tcp_data_in()
499 iscsi_update_cmdsn(conn->session, (struct iscsi_nopin*)rhdr); in iscsi_tcp_data_in()
501 if (tcp_conn->in.datalen == 0) in iscsi_tcp_data_in()
504 if (tcp_task->exp_datasn != datasn) { in iscsi_tcp_data_in()
505 ISCSI_DBG_TCP(conn, "task->exp_datasn(%d) != rhdr->datasn(%d)" in iscsi_tcp_data_in()
506 "\n", tcp_task->exp_datasn, datasn); in iscsi_tcp_data_in()
510 tcp_task->exp_datasn++; in iscsi_tcp_data_in()
512 tcp_task->data_offset = be32_to_cpu(rhdr->offset); in iscsi_tcp_data_in()
513 if (tcp_task->data_offset + tcp_conn->in.datalen > total_in_length) { in iscsi_tcp_data_in()
515 "total_length_in(%d)\n", tcp_task->data_offset, in iscsi_tcp_data_in()
516 tcp_conn->in.datalen, total_in_length); in iscsi_tcp_data_in()
520 conn->datain_pdus_cnt++; in iscsi_tcp_data_in()
525 * iscsi_tcp_r2t_rsp - iSCSI R2T Response processing
531 struct iscsi_session *session = conn->session; in iscsi_tcp_r2t_rsp()
542 spin_lock(&session->back_lock); in iscsi_tcp_r2t_rsp()
543 task = iscsi_itt_to_ctask(conn, hdr->itt); in iscsi_tcp_r2t_rsp()
545 spin_unlock(&session->back_lock); in iscsi_tcp_r2t_rsp()
547 } else if (task->sc->sc_data_direction != DMA_TO_DEVICE) { in iscsi_tcp_r2t_rsp()
548 spin_unlock(&session->back_lock); in iscsi_tcp_r2t_rsp()
555 if (task->state != ISCSI_TASK_RUNNING) { in iscsi_tcp_r2t_rsp()
556 spin_unlock(&session->back_lock); in iscsi_tcp_r2t_rsp()
560 task->last_xfer = jiffies; in iscsi_tcp_r2t_rsp()
562 spin_unlock(&session->back_lock); in iscsi_tcp_r2t_rsp()
567 tcp_conn = conn->dd_data; in iscsi_tcp_r2t_rsp()
568 rhdr = (struct iscsi_r2t_rsp *)tcp_conn->in.hdr; in iscsi_tcp_r2t_rsp()
569 /* fill-in new R2T associated with the task */ in iscsi_tcp_r2t_rsp()
571 spin_unlock(&session->back_lock); in iscsi_tcp_r2t_rsp()
573 if (tcp_conn->in.datalen) { in iscsi_tcp_r2t_rsp()
576 tcp_conn->in.datalen); in iscsi_tcp_r2t_rsp()
581 tcp_task = task->dd_data; in iscsi_tcp_r2t_rsp()
582 r2tsn = be32_to_cpu(rhdr->r2tsn); in iscsi_tcp_r2t_rsp()
583 if (tcp_task->exp_datasn != r2tsn){ in iscsi_tcp_r2t_rsp()
584 ISCSI_DBG_TCP(conn, "task->exp_datasn(%d) != rhdr->r2tsn(%d)\n", in iscsi_tcp_r2t_rsp()
585 tcp_task->exp_datasn, r2tsn); in iscsi_tcp_r2t_rsp()
590 if (session->state != ISCSI_STATE_LOGGED_IN) { in iscsi_tcp_r2t_rsp()
593 task->itt); in iscsi_tcp_r2t_rsp()
598 data_length = be32_to_cpu(rhdr->data_length); in iscsi_tcp_r2t_rsp()
606 if (data_length > session->max_burst) in iscsi_tcp_r2t_rsp()
609 data_length, session->max_burst); in iscsi_tcp_r2t_rsp()
611 data_offset = be32_to_cpu(rhdr->data_offset); in iscsi_tcp_r2t_rsp()
612 if (data_offset + data_length > task->sc->sdb.length) { in iscsi_tcp_r2t_rsp()
616 data_offset, task->sc->sdb.length); in iscsi_tcp_r2t_rsp()
621 spin_lock(&tcp_task->pool2queue); in iscsi_tcp_r2t_rsp()
622 rc = kfifo_out(&tcp_task->r2tpool.queue, (void *)&r2t, sizeof(void *)); in iscsi_tcp_r2t_rsp()
627 spin_unlock(&tcp_task->pool2queue); in iscsi_tcp_r2t_rsp()
632 r2t->exp_statsn = rhdr->statsn; in iscsi_tcp_r2t_rsp()
633 r2t->data_length = data_length; in iscsi_tcp_r2t_rsp()
634 r2t->data_offset = data_offset; in iscsi_tcp_r2t_rsp()
636 r2t->ttt = rhdr->ttt; /* no flip */ in iscsi_tcp_r2t_rsp()
637 r2t->datasn = 0; in iscsi_tcp_r2t_rsp()
638 r2t->sent = 0; in iscsi_tcp_r2t_rsp()
640 tcp_task->exp_datasn = r2tsn + 1; in iscsi_tcp_r2t_rsp()
641 kfifo_in(&tcp_task->r2tqueue, (void*)&r2t, sizeof(void*)); in iscsi_tcp_r2t_rsp()
642 conn->r2t_pdus_cnt++; in iscsi_tcp_r2t_rsp()
643 spin_unlock(&tcp_task->pool2queue); in iscsi_tcp_r2t_rsp()
660 struct iscsi_conn *conn = tcp_conn->iscsi_conn; in iscsi_tcp_process_data_in()
661 struct iscsi_hdr *hdr = tcp_conn->in.hdr; in iscsi_tcp_process_data_in()
667 /* check for non-exceptional status */ in iscsi_tcp_process_data_in()
668 if (hdr->flags & ISCSI_FLAG_DATA_STATUS) { in iscsi_tcp_process_data_in()
669 rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0); in iscsi_tcp_process_data_in()
679 * iscsi_tcp_hdr_dissect - process PDU header
692 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; in iscsi_tcp_hdr_dissect()
696 tcp_conn->in.datalen = ntoh24(hdr->dlength); in iscsi_tcp_hdr_dissect()
697 if (tcp_conn->in.datalen > conn->max_recv_dlength) { in iscsi_tcp_hdr_dissect()
700 tcp_conn->in.datalen, conn->max_recv_dlength); in iscsi_tcp_hdr_dissect()
707 ahslen = hdr->hlength << 2; in iscsi_tcp_hdr_dissect()
709 opcode = hdr->opcode & ISCSI_OPCODE_MASK; in iscsi_tcp_hdr_dissect()
711 rc = iscsi_verify_itt(conn, hdr->itt); in iscsi_tcp_hdr_dissect()
716 opcode, ahslen, tcp_conn->in.datalen); in iscsi_tcp_hdr_dissect()
720 spin_lock(&conn->session->back_lock); in iscsi_tcp_hdr_dissect()
721 task = iscsi_itt_to_ctask(conn, hdr->itt); in iscsi_tcp_hdr_dissect()
727 spin_unlock(&conn->session->back_lock); in iscsi_tcp_hdr_dissect()
731 if (tcp_conn->in.datalen) { in iscsi_tcp_hdr_dissect()
732 struct iscsi_tcp_task *tcp_task = task->dd_data; in iscsi_tcp_hdr_dissect()
734 struct scsi_data_buffer *sdb = &task->sc->sdb; in iscsi_tcp_hdr_dissect()
737 * Setup copy of Data-In into the struct scsi_cmnd in iscsi_tcp_hdr_dissect()
742 * update the digest per-entry. in iscsi_tcp_hdr_dissect()
744 if (conn->datadgst_en && in iscsi_tcp_hdr_dissect()
745 !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD)) in iscsi_tcp_hdr_dissect()
746 rx_hash = tcp_conn->rx_hash; in iscsi_tcp_hdr_dissect()
750 tcp_task->data_offset, in iscsi_tcp_hdr_dissect()
751 tcp_conn->in.datalen); in iscsi_tcp_hdr_dissect()
752 task->last_xfer = jiffies; in iscsi_tcp_hdr_dissect()
753 rc = iscsi_segment_seek_sg(&tcp_conn->in.segment, in iscsi_tcp_hdr_dissect()
754 sdb->table.sgl, in iscsi_tcp_hdr_dissect()
755 sdb->table.nents, in iscsi_tcp_hdr_dissect()
756 tcp_task->data_offset, in iscsi_tcp_hdr_dissect()
757 tcp_conn->in.datalen, in iscsi_tcp_hdr_dissect()
760 spin_unlock(&conn->session->back_lock); in iscsi_tcp_hdr_dissect()
764 spin_unlock(&conn->session->back_lock); in iscsi_tcp_hdr_dissect()
767 if (tcp_conn->in.datalen) { in iscsi_tcp_hdr_dissect()
789 if (ISCSI_DEF_MAX_RECV_SEG_LEN < tcp_conn->in.datalen) { in iscsi_tcp_hdr_dissect()
794 tcp_conn->in.datalen, in iscsi_tcp_hdr_dissect()
803 if (tcp_conn->in.datalen) { in iscsi_tcp_hdr_dissect()
821 if (tcp_conn->in.datalen) in iscsi_tcp_hdr_dissect()
830 * iscsi_tcp_hdr_recv_done - process PDU header
842 struct iscsi_conn *conn = tcp_conn->iscsi_conn; in iscsi_tcp_hdr_recv_done()
849 hdr = (struct iscsi_hdr *) tcp_conn->in.hdr_buf; in iscsi_tcp_hdr_recv_done()
850 if (segment->copied == sizeof(struct iscsi_hdr) && hdr->hlength) { in iscsi_tcp_hdr_recv_done()
851 /* Bump the header length - the caller will in iscsi_tcp_hdr_recv_done()
854 unsigned int ahslen = hdr->hlength << 2; in iscsi_tcp_hdr_recv_done()
857 if (sizeof(*hdr) + ahslen > sizeof(tcp_conn->in.hdr_buf)) in iscsi_tcp_hdr_recv_done()
860 segment->total_size += ahslen; in iscsi_tcp_hdr_recv_done()
861 segment->size += ahslen; in iscsi_tcp_hdr_recv_done()
868 if (conn->hdrdgst_en && in iscsi_tcp_hdr_recv_done()
869 !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD)) { in iscsi_tcp_hdr_recv_done()
870 if (segment->digest_len == 0) { in iscsi_tcp_hdr_recv_done()
877 segment->recv_digest); in iscsi_tcp_hdr_recv_done()
881 iscsi_tcp_dgst_header(tcp_conn->rx_hash, hdr, in iscsi_tcp_hdr_recv_done()
882 segment->total_copied - ISCSI_DIGEST_SIZE, in iscsi_tcp_hdr_recv_done()
883 segment->digest); in iscsi_tcp_hdr_recv_done()
889 tcp_conn->in.hdr = hdr; in iscsi_tcp_hdr_recv_done()
894 * iscsi_tcp_recv_segment_is_hdr - tests if we are reading in a header
902 return tcp_conn->in.segment.done == iscsi_tcp_hdr_recv_done; in iscsi_tcp_recv_segment_is_hdr()
907 * iscsi_tcp_recv_skb - Process skb
920 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; in iscsi_tcp_recv_skb()
921 struct iscsi_segment *segment = &tcp_conn->in.segment; in iscsi_tcp_recv_skb()
926 ISCSI_DBG_TCP(conn, "in %d bytes\n", skb->len - offset); in iscsi_tcp_recv_skb()
932 conn->last_recv = jiffies; in iscsi_tcp_recv_skb()
934 if (unlikely(test_bit(ISCSI_CONN_FLAG_SUSPEND_RX, &conn->flags))) { in iscsi_tcp_recv_skb()
941 segment->total_copied = segment->total_size; in iscsi_tcp_recv_skb()
945 skb_prepare_seq_read(skb, offset, skb->len, &seq); in iscsi_tcp_recv_skb()
957 BUG_ON(segment->copied >= segment->size); in iscsi_tcp_recv_skb()
965 if (segment->total_copied >= segment->total_size) { in iscsi_tcp_recv_skb()
974 rc = segment->done(tcp_conn, segment); in iscsi_tcp_recv_skb()
984 conn->rxdata_octets += consumed; in iscsi_tcp_recv_skb()
990 * iscsi_tcp_task_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
995 struct iscsi_tcp_task *tcp_task = task->dd_data; in iscsi_tcp_task_init()
996 struct iscsi_conn *conn = task->conn; in iscsi_tcp_task_init()
997 struct scsi_cmnd *sc = task->sc; in iscsi_tcp_task_init()
1005 ISCSI_DBG_TCP(conn, "mtask deq [itt 0x%x]\n", task->itt); in iscsi_tcp_task_init()
1007 return conn->session->tt->init_pdu(task, 0, task->data_count); in iscsi_tcp_task_init()
1010 BUG_ON(kfifo_len(&tcp_task->r2tqueue)); in iscsi_tcp_task_init()
1011 tcp_task->exp_datasn = 0; in iscsi_tcp_task_init()
1015 task->itt, task->imm_count, task->unsol_r2t.data_length); in iscsi_tcp_task_init()
1017 err = conn->session->tt->init_pdu(task, 0, task->imm_count); in iscsi_tcp_task_init()
1020 task->imm_count = 0; in iscsi_tcp_task_init()
1027 struct iscsi_tcp_task *tcp_task = task->dd_data; in iscsi_tcp_get_curr_r2t()
1031 r2t = &task->unsol_r2t; in iscsi_tcp_get_curr_r2t()
1033 spin_lock_bh(&tcp_task->queue2pool); in iscsi_tcp_get_curr_r2t()
1034 if (tcp_task->r2t) { in iscsi_tcp_get_curr_r2t()
1035 r2t = tcp_task->r2t; in iscsi_tcp_get_curr_r2t()
1037 if (r2t->data_length <= r2t->sent) { in iscsi_tcp_get_curr_r2t()
1038 ISCSI_DBG_TCP(task->conn, in iscsi_tcp_get_curr_r2t()
1040 kfifo_in(&tcp_task->r2tpool.queue, in iscsi_tcp_get_curr_r2t()
1041 (void *)&tcp_task->r2t, in iscsi_tcp_get_curr_r2t()
1043 tcp_task->r2t = r2t = NULL; in iscsi_tcp_get_curr_r2t()
1048 if (kfifo_out(&tcp_task->r2tqueue, in iscsi_tcp_get_curr_r2t()
1049 (void *)&tcp_task->r2t, sizeof(void *)) != in iscsi_tcp_get_curr_r2t()
1053 r2t = tcp_task->r2t; in iscsi_tcp_get_curr_r2t()
1055 spin_unlock_bh(&tcp_task->queue2pool); in iscsi_tcp_get_curr_r2t()
1062 * iscsi_tcp_task_xmit - xmit normal PDU task
1066 * -EAGAIN if there's still data in the queue, or != 0 for any other kind
1071 struct iscsi_conn *conn = task->conn; in iscsi_tcp_task_xmit()
1072 struct iscsi_session *session = conn->session; in iscsi_tcp_task_xmit()
1078 rc = session->tt->xmit_pdu(task); in iscsi_tcp_task_xmit()
1083 if (!task->sc) { in iscsi_tcp_task_xmit()
1084 if (task->hdr->itt == RESERVED_ITT) in iscsi_tcp_task_xmit()
1090 if (task->sc->sc_data_direction != DMA_TO_DEVICE) in iscsi_tcp_task_xmit()
1100 rc = conn->session->tt->alloc_pdu(task, ISCSI_OP_SCSI_DATA_OUT); in iscsi_tcp_task_xmit()
1103 iscsi_prep_data_out_pdu(task, r2t, (struct iscsi_data *) task->hdr); in iscsi_tcp_task_xmit()
1106 r2t, r2t->datasn - 1, task->hdr->itt, in iscsi_tcp_task_xmit()
1107 r2t->data_offset + r2t->sent, r2t->data_count); in iscsi_tcp_task_xmit()
1109 rc = conn->session->tt->init_pdu(task, r2t->data_offset + r2t->sent, in iscsi_tcp_task_xmit()
1110 r2t->data_count); in iscsi_tcp_task_xmit()
1116 r2t->sent += r2t->data_count; in iscsi_tcp_task_xmit()
1134 conn = cls_conn->dd_data; in iscsi_tcp_conn_setup()
1139 conn->max_recv_dlength = ISCSI_DEF_MAX_RECV_SEG_LEN; in iscsi_tcp_conn_setup()
1141 tcp_conn = conn->dd_data; in iscsi_tcp_conn_setup()
1142 tcp_conn->iscsi_conn = conn; in iscsi_tcp_conn_setup()
1143 tcp_conn->dd_data = conn->dd_data + sizeof(*tcp_conn); in iscsi_tcp_conn_setup()
1160 * initialize per-task: R2T pool and xmit queue in iscsi_tcp_r2tpool_alloc()
1162 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) { in iscsi_tcp_r2tpool_alloc()
1163 struct iscsi_task *task = session->cmds[cmd_i]; in iscsi_tcp_r2tpool_alloc()
1164 struct iscsi_tcp_task *tcp_task = task->dd_data; in iscsi_tcp_r2tpool_alloc()
1167 * pre-allocated x2 as much r2ts to handle race when in iscsi_tcp_r2tpool_alloc()
1173 if (iscsi_pool_init(&tcp_task->r2tpool, in iscsi_tcp_r2tpool_alloc()
1174 session->max_r2t * 2, NULL, in iscsi_tcp_r2tpool_alloc()
1180 if (kfifo_alloc(&tcp_task->r2tqueue, in iscsi_tcp_r2tpool_alloc()
1181 session->max_r2t * 4 * sizeof(void*), GFP_KERNEL)) { in iscsi_tcp_r2tpool_alloc()
1182 iscsi_pool_free(&tcp_task->r2tpool); in iscsi_tcp_r2tpool_alloc()
1185 spin_lock_init(&tcp_task->pool2queue); in iscsi_tcp_r2tpool_alloc()
1186 spin_lock_init(&tcp_task->queue2pool); in iscsi_tcp_r2tpool_alloc()
1193 struct iscsi_task *task = session->cmds[i]; in iscsi_tcp_r2tpool_alloc()
1194 struct iscsi_tcp_task *tcp_task = task->dd_data; in iscsi_tcp_r2tpool_alloc()
1196 kfifo_free(&tcp_task->r2tqueue); in iscsi_tcp_r2tpool_alloc()
1197 iscsi_pool_free(&tcp_task->r2tpool); in iscsi_tcp_r2tpool_alloc()
1199 return -ENOMEM; in iscsi_tcp_r2tpool_alloc()
1207 for (i = 0; i < session->cmds_max; i++) { in iscsi_tcp_r2tpool_free()
1208 struct iscsi_task *task = session->cmds[i]; in iscsi_tcp_r2tpool_free()
1209 struct iscsi_tcp_task *tcp_task = task->dd_data; in iscsi_tcp_r2tpool_free()
1211 kfifo_free(&tcp_task->r2tqueue); in iscsi_tcp_r2tpool_free()
1212 iscsi_pool_free(&tcp_task->r2tpool); in iscsi_tcp_r2tpool_free()
1219 struct iscsi_session *session = conn->session; in iscsi_tcp_set_max_r2t()
1223 if (session->max_r2t == r2ts) in iscsi_tcp_set_max_r2t()
1227 return -EINVAL; in iscsi_tcp_set_max_r2t()
1229 session->max_r2t = r2ts; in iscsi_tcp_set_max_r2t()
1238 struct iscsi_conn *conn = cls_conn->dd_data; in iscsi_tcp_conn_get_stats()
1240 stats->txdata_octets = conn->txdata_octets; in iscsi_tcp_conn_get_stats()
1241 stats->rxdata_octets = conn->rxdata_octets; in iscsi_tcp_conn_get_stats()
1242 stats->scsicmd_pdus = conn->scsicmd_pdus_cnt; in iscsi_tcp_conn_get_stats()
1243 stats->dataout_pdus = conn->dataout_pdus_cnt; in iscsi_tcp_conn_get_stats()
1244 stats->scsirsp_pdus = conn->scsirsp_pdus_cnt; in iscsi_tcp_conn_get_stats()
1245 stats->datain_pdus = conn->datain_pdus_cnt; in iscsi_tcp_conn_get_stats()
1246 stats->r2t_pdus = conn->r2t_pdus_cnt; in iscsi_tcp_conn_get_stats()
1247 stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt; in iscsi_tcp_conn_get_stats()
1248 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt; in iscsi_tcp_conn_get_stats()