Lines Matching +full:single +full:- +full:tt
1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * iSCSI over TCP/IP Data-Path lib
7 * Copyright (C) 2005 - 2006 Mike Christie
9 * maintained by open-iscsi@googlegroups.com
42 MODULE_DESCRIPTION("iSCSI/TCP data-path");
59 &(_conn)->cls_conn->dev, \
80 * iscsi_tcp_segment_init_sg - init indicated scatterlist entry
93 segment->sg = sg; in iscsi_tcp_segment_init_sg()
94 segment->sg_offset = offset; in iscsi_tcp_segment_init_sg()
95 segment->size = min(sg->length - offset, in iscsi_tcp_segment_init_sg()
96 segment->total_size - segment->total_copied); in iscsi_tcp_segment_init_sg()
97 segment->data = NULL; in iscsi_tcp_segment_init_sg()
101 * iscsi_tcp_segment_map - map the current S/G page
113 if (segment->data != NULL || !segment->sg) in iscsi_tcp_segment_map()
116 sg = segment->sg; in iscsi_tcp_segment_map()
117 BUG_ON(segment->sg_mapped); in iscsi_tcp_segment_map()
118 BUG_ON(sg->length == 0); in iscsi_tcp_segment_map()
128 * coalescing neighboring slab objects into a single frag which in iscsi_tcp_segment_map()
135 segment->atomic_mapped = true; in iscsi_tcp_segment_map()
136 segment->sg_mapped = kmap_atomic(sg_page(sg)); in iscsi_tcp_segment_map()
138 segment->atomic_mapped = false; in iscsi_tcp_segment_map()
140 segment->sg_mapped = kmap(sg_page(sg)); in iscsi_tcp_segment_map()
143 segment->data = segment->sg_mapped + sg->offset + segment->sg_offset; in iscsi_tcp_segment_map()
148 if (segment->sg_mapped) { in iscsi_tcp_segment_unmap()
149 if (segment->atomic_mapped) in iscsi_tcp_segment_unmap()
150 kunmap_atomic(segment->sg_mapped); in iscsi_tcp_segment_unmap()
152 kunmap(sg_page(segment->sg)); in iscsi_tcp_segment_unmap()
153 segment->sg_mapped = NULL; in iscsi_tcp_segment_unmap()
154 segment->data = NULL; in iscsi_tcp_segment_unmap()
165 segment->data = digest; in iscsi_tcp_segment_splice_digest()
166 segment->digest_len = ISCSI_DIGEST_SIZE; in iscsi_tcp_segment_splice_digest()
167 segment->total_size += ISCSI_DIGEST_SIZE; in iscsi_tcp_segment_splice_digest()
168 segment->size = ISCSI_DIGEST_SIZE; in iscsi_tcp_segment_splice_digest()
169 segment->copied = 0; in iscsi_tcp_segment_splice_digest()
170 segment->sg = NULL; in iscsi_tcp_segment_splice_digest()
171 segment->crcp = NULL; in iscsi_tcp_segment_splice_digest()
175 * iscsi_tcp_segment_done - check whether the segment is complete
188 * This function must be re-entrant.
196 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "copied %u %u size %u %s\n", in iscsi_tcp_segment_done()
197 segment->copied, copied, segment->size, in iscsi_tcp_segment_done()
199 if (segment->crcp && copied) { in iscsi_tcp_segment_done()
200 if (segment->data) { in iscsi_tcp_segment_done()
201 *segment->crcp = crc32c(*segment->crcp, in iscsi_tcp_segment_done()
202 segment->data + segment->copied, in iscsi_tcp_segment_done()
207 data = kmap_local_page(sg_page(segment->sg)); in iscsi_tcp_segment_done()
208 *segment->crcp = crc32c(*segment->crcp, in iscsi_tcp_segment_done()
209 data + segment->copied + in iscsi_tcp_segment_done()
210 segment->sg_offset + in iscsi_tcp_segment_done()
211 segment->sg->offset, in iscsi_tcp_segment_done()
217 segment->copied += copied; in iscsi_tcp_segment_done()
218 if (segment->copied < segment->size) { in iscsi_tcp_segment_done()
223 segment->total_copied += segment->copied; in iscsi_tcp_segment_done()
224 segment->copied = 0; in iscsi_tcp_segment_done()
225 segment->size = 0; in iscsi_tcp_segment_done()
231 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "total copied %u total size %u\n", in iscsi_tcp_segment_done()
232 segment->total_copied, segment->total_size); in iscsi_tcp_segment_done()
233 if (segment->total_copied < segment->total_size) { in iscsi_tcp_segment_done()
235 iscsi_tcp_segment_init_sg(segment, sg_next(segment->sg), in iscsi_tcp_segment_done()
238 BUG_ON(segment->size == 0); in iscsi_tcp_segment_done()
243 if (!(tcp_conn->iscsi_conn->session->tt->caps & CAP_PADDING_OFFLOAD)) { in iscsi_tcp_segment_done()
244 pad = iscsi_padding(segment->total_copied); in iscsi_tcp_segment_done()
246 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, in iscsi_tcp_segment_done()
248 segment->total_size += pad; in iscsi_tcp_segment_done()
249 segment->size = pad; in iscsi_tcp_segment_done()
250 segment->data = segment->padbuf; in iscsi_tcp_segment_done()
259 if (segment->crcp) { in iscsi_tcp_segment_done()
260 put_unaligned_le32(~*segment->crcp, segment->digest); in iscsi_tcp_segment_done()
262 recv ? segment->recv_digest : segment->digest); in iscsi_tcp_segment_done()
271 * iscsi_tcp_segment_recv - copy data to segment
295 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, in iscsi_tcp_segment_recv()
300 copy = min(len - copied, segment->size - segment->copied); in iscsi_tcp_segment_recv()
301 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "copying %d\n", copy); in iscsi_tcp_segment_recv()
302 memcpy(segment->data + segment->copied, ptr + copied, copy); in iscsi_tcp_segment_recv()
320 if (!segment->digest_len) in iscsi_tcp_dgst_verify()
323 if (memcmp(segment->recv_digest, segment->digest, in iscsi_tcp_dgst_verify()
324 segment->digest_len)) { in iscsi_tcp_dgst_verify()
325 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "digest mismatch\n"); in iscsi_tcp_dgst_verify()
340 segment->total_size = size; in __iscsi_segment_init()
341 segment->done = done; in __iscsi_segment_init()
344 segment->crcp = crcp; in __iscsi_segment_init()
354 segment->data = data; in iscsi_segment_init_linear()
355 segment->size = size; in iscsi_segment_init_linear()
370 if (offset < sg->length) { in iscsi_segment_seek_sg()
374 offset -= sg->length; in iscsi_segment_seek_sg()
382 * iscsi_tcp_hdr_recv_prep - prep segment for hdr reception
391 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, in iscsi_tcp_hdr_recv_prep()
392 "(%s)\n", tcp_conn->iscsi_conn->hdrdgst_en ? in iscsi_tcp_hdr_recv_prep()
394 iscsi_segment_init_linear(&tcp_conn->in.segment, in iscsi_tcp_hdr_recv_prep()
395 tcp_conn->in.hdr_buf, sizeof(struct iscsi_hdr), in iscsi_tcp_hdr_recv_prep()
407 struct iscsi_conn *conn = tcp_conn->iscsi_conn; in iscsi_tcp_data_recv_done()
413 rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, in iscsi_tcp_data_recv_done()
414 conn->data, tcp_conn->in.datalen); in iscsi_tcp_data_recv_done()
425 struct iscsi_conn *conn = tcp_conn->iscsi_conn; in iscsi_tcp_data_recv_prep()
428 if (conn->datadgst_en && in iscsi_tcp_data_recv_prep()
429 !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD)) in iscsi_tcp_data_recv_prep()
430 rx_crcp = tcp_conn->rx_crcp; in iscsi_tcp_data_recv_prep()
432 iscsi_segment_init_linear(&tcp_conn->in.segment, in iscsi_tcp_data_recv_prep()
433 conn->data, tcp_conn->in.datalen, in iscsi_tcp_data_recv_prep()
438 * iscsi_tcp_cleanup_task - free tcp_task resources
445 struct iscsi_tcp_task *tcp_task = task->dd_data; in iscsi_tcp_cleanup_task()
449 if (!task->sc) in iscsi_tcp_cleanup_task()
452 spin_lock_bh(&tcp_task->queue2pool); in iscsi_tcp_cleanup_task()
454 while (kfifo_out(&tcp_task->r2tqueue, (void*)&r2t, sizeof(void*))) { in iscsi_tcp_cleanup_task()
455 kfifo_in(&tcp_task->r2tpool.queue, (void*)&r2t, in iscsi_tcp_cleanup_task()
457 ISCSI_DBG_TCP(task->conn, "pending r2t dropped\n"); in iscsi_tcp_cleanup_task()
460 r2t = tcp_task->r2t; in iscsi_tcp_cleanup_task()
462 kfifo_in(&tcp_task->r2tpool.queue, (void*)&r2t, in iscsi_tcp_cleanup_task()
464 tcp_task->r2t = NULL; in iscsi_tcp_cleanup_task()
466 spin_unlock_bh(&tcp_task->queue2pool); in iscsi_tcp_cleanup_task()
471 * iscsi_tcp_data_in - SCSI Data-In Response processing
477 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; in iscsi_tcp_data_in()
478 struct iscsi_tcp_task *tcp_task = task->dd_data; in iscsi_tcp_data_in()
479 struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)tcp_conn->in.hdr; in iscsi_tcp_data_in()
480 int datasn = be32_to_cpu(rhdr->datasn); in iscsi_tcp_data_in()
481 unsigned total_in_length = task->sc->sdb.length; in iscsi_tcp_data_in()
487 if (!(rhdr->flags & ISCSI_FLAG_DATA_STATUS)) in iscsi_tcp_data_in()
488 iscsi_update_cmdsn(conn->session, (struct iscsi_nopin*)rhdr); in iscsi_tcp_data_in()
490 if (tcp_conn->in.datalen == 0) in iscsi_tcp_data_in()
493 if (tcp_task->exp_datasn != datasn) { in iscsi_tcp_data_in()
494 ISCSI_DBG_TCP(conn, "task->exp_datasn(%d) != rhdr->datasn(%d)" in iscsi_tcp_data_in()
495 "\n", tcp_task->exp_datasn, datasn); in iscsi_tcp_data_in()
499 tcp_task->exp_datasn++; in iscsi_tcp_data_in()
501 tcp_task->data_offset = be32_to_cpu(rhdr->offset); in iscsi_tcp_data_in()
502 if (tcp_task->data_offset + tcp_conn->in.datalen > total_in_length) { in iscsi_tcp_data_in()
504 "total_length_in(%d)\n", tcp_task->data_offset, in iscsi_tcp_data_in()
505 tcp_conn->in.datalen, total_in_length); in iscsi_tcp_data_in()
509 conn->datain_pdus_cnt++; in iscsi_tcp_data_in()
514 * iscsi_tcp_r2t_rsp - iSCSI R2T Response processing
520 struct iscsi_session *session = conn->session; in iscsi_tcp_r2t_rsp()
531 spin_lock(&session->back_lock); in iscsi_tcp_r2t_rsp()
532 task = iscsi_itt_to_ctask(conn, hdr->itt); in iscsi_tcp_r2t_rsp()
534 spin_unlock(&session->back_lock); in iscsi_tcp_r2t_rsp()
536 } else if (task->sc->sc_data_direction != DMA_TO_DEVICE) { in iscsi_tcp_r2t_rsp()
537 spin_unlock(&session->back_lock); in iscsi_tcp_r2t_rsp()
544 if (task->state != ISCSI_TASK_RUNNING) { in iscsi_tcp_r2t_rsp()
545 spin_unlock(&session->back_lock); in iscsi_tcp_r2t_rsp()
549 task->last_xfer = jiffies; in iscsi_tcp_r2t_rsp()
551 spin_unlock(&session->back_lock); in iscsi_tcp_r2t_rsp()
556 tcp_conn = conn->dd_data; in iscsi_tcp_r2t_rsp()
557 rhdr = (struct iscsi_r2t_rsp *)tcp_conn->in.hdr; in iscsi_tcp_r2t_rsp()
558 /* fill-in new R2T associated with the task */ in iscsi_tcp_r2t_rsp()
560 spin_unlock(&session->back_lock); in iscsi_tcp_r2t_rsp()
562 if (tcp_conn->in.datalen) { in iscsi_tcp_r2t_rsp()
565 tcp_conn->in.datalen); in iscsi_tcp_r2t_rsp()
570 tcp_task = task->dd_data; in iscsi_tcp_r2t_rsp()
571 r2tsn = be32_to_cpu(rhdr->r2tsn); in iscsi_tcp_r2t_rsp()
572 if (tcp_task->exp_datasn != r2tsn){ in iscsi_tcp_r2t_rsp()
573 ISCSI_DBG_TCP(conn, "task->exp_datasn(%d) != rhdr->r2tsn(%d)\n", in iscsi_tcp_r2t_rsp()
574 tcp_task->exp_datasn, r2tsn); in iscsi_tcp_r2t_rsp()
579 if (session->state != ISCSI_STATE_LOGGED_IN) { in iscsi_tcp_r2t_rsp()
582 task->itt); in iscsi_tcp_r2t_rsp()
587 data_length = be32_to_cpu(rhdr->data_length); in iscsi_tcp_r2t_rsp()
595 if (data_length > session->max_burst) in iscsi_tcp_r2t_rsp()
598 data_length, session->max_burst); in iscsi_tcp_r2t_rsp()
600 data_offset = be32_to_cpu(rhdr->data_offset); in iscsi_tcp_r2t_rsp()
601 if (data_offset + data_length > task->sc->sdb.length) { in iscsi_tcp_r2t_rsp()
605 data_offset, task->sc->sdb.length); in iscsi_tcp_r2t_rsp()
610 spin_lock(&tcp_task->pool2queue); in iscsi_tcp_r2t_rsp()
611 rc = kfifo_out(&tcp_task->r2tpool.queue, (void *)&r2t, sizeof(void *)); in iscsi_tcp_r2t_rsp()
616 spin_unlock(&tcp_task->pool2queue); in iscsi_tcp_r2t_rsp()
621 r2t->exp_statsn = rhdr->statsn; in iscsi_tcp_r2t_rsp()
622 r2t->data_length = data_length; in iscsi_tcp_r2t_rsp()
623 r2t->data_offset = data_offset; in iscsi_tcp_r2t_rsp()
625 r2t->ttt = rhdr->ttt; /* no flip */ in iscsi_tcp_r2t_rsp()
626 r2t->datasn = 0; in iscsi_tcp_r2t_rsp()
627 r2t->sent = 0; in iscsi_tcp_r2t_rsp()
629 tcp_task->exp_datasn = r2tsn + 1; in iscsi_tcp_r2t_rsp()
630 kfifo_in(&tcp_task->r2tqueue, (void*)&r2t, sizeof(void*)); in iscsi_tcp_r2t_rsp()
631 conn->r2t_pdus_cnt++; in iscsi_tcp_r2t_rsp()
632 spin_unlock(&tcp_task->pool2queue); in iscsi_tcp_r2t_rsp()
649 struct iscsi_conn *conn = tcp_conn->iscsi_conn; in iscsi_tcp_process_data_in()
650 struct iscsi_hdr *hdr = tcp_conn->in.hdr; in iscsi_tcp_process_data_in()
656 /* check for non-exceptional status */ in iscsi_tcp_process_data_in()
657 if (hdr->flags & ISCSI_FLAG_DATA_STATUS) { in iscsi_tcp_process_data_in()
658 rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0); in iscsi_tcp_process_data_in()
668 * iscsi_tcp_hdr_dissect - process PDU header
681 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; in iscsi_tcp_hdr_dissect()
685 tcp_conn->in.datalen = ntoh24(hdr->dlength); in iscsi_tcp_hdr_dissect()
686 if (tcp_conn->in.datalen > conn->max_recv_dlength) { in iscsi_tcp_hdr_dissect()
689 tcp_conn->in.datalen, conn->max_recv_dlength); in iscsi_tcp_hdr_dissect()
696 ahslen = hdr->hlength << 2; in iscsi_tcp_hdr_dissect()
698 opcode = hdr->opcode & ISCSI_OPCODE_MASK; in iscsi_tcp_hdr_dissect()
700 rc = iscsi_verify_itt(conn, hdr->itt); in iscsi_tcp_hdr_dissect()
705 opcode, ahslen, tcp_conn->in.datalen); in iscsi_tcp_hdr_dissect()
709 spin_lock(&conn->session->back_lock); in iscsi_tcp_hdr_dissect()
710 task = iscsi_itt_to_ctask(conn, hdr->itt); in iscsi_tcp_hdr_dissect()
716 spin_unlock(&conn->session->back_lock); in iscsi_tcp_hdr_dissect()
720 if (tcp_conn->in.datalen) { in iscsi_tcp_hdr_dissect()
721 struct iscsi_tcp_task *tcp_task = task->dd_data; in iscsi_tcp_hdr_dissect()
723 struct scsi_data_buffer *sdb = &task->sc->sdb; in iscsi_tcp_hdr_dissect()
726 * Setup copy of Data-In into the struct scsi_cmnd in iscsi_tcp_hdr_dissect()
731 * update the digest per-entry. in iscsi_tcp_hdr_dissect()
733 if (conn->datadgst_en && in iscsi_tcp_hdr_dissect()
734 !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD)) in iscsi_tcp_hdr_dissect()
735 rx_crcp = tcp_conn->rx_crcp; in iscsi_tcp_hdr_dissect()
739 tcp_task->data_offset, in iscsi_tcp_hdr_dissect()
740 tcp_conn->in.datalen); in iscsi_tcp_hdr_dissect()
741 task->last_xfer = jiffies; in iscsi_tcp_hdr_dissect()
742 rc = iscsi_segment_seek_sg(&tcp_conn->in.segment, in iscsi_tcp_hdr_dissect()
743 sdb->table.sgl, in iscsi_tcp_hdr_dissect()
744 sdb->table.nents, in iscsi_tcp_hdr_dissect()
745 tcp_task->data_offset, in iscsi_tcp_hdr_dissect()
746 tcp_conn->in.datalen, in iscsi_tcp_hdr_dissect()
749 spin_unlock(&conn->session->back_lock); in iscsi_tcp_hdr_dissect()
753 spin_unlock(&conn->session->back_lock); in iscsi_tcp_hdr_dissect()
756 if (tcp_conn->in.datalen) { in iscsi_tcp_hdr_dissect()
778 if (ISCSI_DEF_MAX_RECV_SEG_LEN < tcp_conn->in.datalen) { in iscsi_tcp_hdr_dissect()
783 tcp_conn->in.datalen, in iscsi_tcp_hdr_dissect()
792 if (tcp_conn->in.datalen) { in iscsi_tcp_hdr_dissect()
810 if (tcp_conn->in.datalen) in iscsi_tcp_hdr_dissect()
819 * iscsi_tcp_hdr_recv_done - process PDU header
831 struct iscsi_conn *conn = tcp_conn->iscsi_conn; in iscsi_tcp_hdr_recv_done()
838 hdr = (struct iscsi_hdr *) tcp_conn->in.hdr_buf; in iscsi_tcp_hdr_recv_done()
839 if (segment->copied == sizeof(struct iscsi_hdr) && hdr->hlength) { in iscsi_tcp_hdr_recv_done()
840 /* Bump the header length - the caller will in iscsi_tcp_hdr_recv_done()
843 unsigned int ahslen = hdr->hlength << 2; in iscsi_tcp_hdr_recv_done()
846 if (sizeof(*hdr) + ahslen > sizeof(tcp_conn->in.hdr_buf)) in iscsi_tcp_hdr_recv_done()
849 segment->total_size += ahslen; in iscsi_tcp_hdr_recv_done()
850 segment->size += ahslen; in iscsi_tcp_hdr_recv_done()
857 if (conn->hdrdgst_en && in iscsi_tcp_hdr_recv_done()
858 !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD)) { in iscsi_tcp_hdr_recv_done()
859 if (segment->digest_len == 0) { in iscsi_tcp_hdr_recv_done()
866 segment->recv_digest); in iscsi_tcp_hdr_recv_done()
871 segment->total_copied - ISCSI_DIGEST_SIZE, in iscsi_tcp_hdr_recv_done()
872 segment->digest); in iscsi_tcp_hdr_recv_done()
878 tcp_conn->in.hdr = hdr; in iscsi_tcp_hdr_recv_done()
883 * iscsi_tcp_recv_segment_is_hdr - tests if we are reading in a header
891 return tcp_conn->in.segment.done == iscsi_tcp_hdr_recv_done; in iscsi_tcp_recv_segment_is_hdr()
896 * iscsi_tcp_recv_skb - Process skb
909 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; in iscsi_tcp_recv_skb()
910 struct iscsi_segment *segment = &tcp_conn->in.segment; in iscsi_tcp_recv_skb()
915 ISCSI_DBG_TCP(conn, "in %d bytes\n", skb->len - offset); in iscsi_tcp_recv_skb()
921 conn->last_recv = jiffies; in iscsi_tcp_recv_skb()
923 if (unlikely(test_bit(ISCSI_CONN_FLAG_SUSPEND_RX, &conn->flags))) { in iscsi_tcp_recv_skb()
930 segment->total_copied = segment->total_size; in iscsi_tcp_recv_skb()
934 skb_prepare_seq_read(skb, offset, skb->len, &seq); in iscsi_tcp_recv_skb()
946 BUG_ON(segment->copied >= segment->size); in iscsi_tcp_recv_skb()
954 if (segment->total_copied >= segment->total_size) { in iscsi_tcp_recv_skb()
963 rc = segment->done(tcp_conn, segment); in iscsi_tcp_recv_skb()
973 conn->rxdata_octets += consumed; in iscsi_tcp_recv_skb()
979 * iscsi_tcp_task_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
984 struct iscsi_tcp_task *tcp_task = task->dd_data; in iscsi_tcp_task_init()
985 struct iscsi_conn *conn = task->conn; in iscsi_tcp_task_init()
986 struct scsi_cmnd *sc = task->sc; in iscsi_tcp_task_init()
994 ISCSI_DBG_TCP(conn, "mtask deq [itt 0x%x]\n", task->itt); in iscsi_tcp_task_init()
996 return conn->session->tt->init_pdu(task, 0, task->data_count); in iscsi_tcp_task_init()
999 BUG_ON(kfifo_len(&tcp_task->r2tqueue)); in iscsi_tcp_task_init()
1000 tcp_task->exp_datasn = 0; in iscsi_tcp_task_init()
1004 task->itt, task->imm_count, task->unsol_r2t.data_length); in iscsi_tcp_task_init()
1006 err = conn->session->tt->init_pdu(task, 0, task->imm_count); in iscsi_tcp_task_init()
1009 task->imm_count = 0; in iscsi_tcp_task_init()
1016 struct iscsi_tcp_task *tcp_task = task->dd_data; in iscsi_tcp_get_curr_r2t()
1020 r2t = &task->unsol_r2t; in iscsi_tcp_get_curr_r2t()
1022 spin_lock_bh(&tcp_task->queue2pool); in iscsi_tcp_get_curr_r2t()
1023 if (tcp_task->r2t) { in iscsi_tcp_get_curr_r2t()
1024 r2t = tcp_task->r2t; in iscsi_tcp_get_curr_r2t()
1026 if (r2t->data_length <= r2t->sent) { in iscsi_tcp_get_curr_r2t()
1027 ISCSI_DBG_TCP(task->conn, in iscsi_tcp_get_curr_r2t()
1029 kfifo_in(&tcp_task->r2tpool.queue, in iscsi_tcp_get_curr_r2t()
1030 (void *)&tcp_task->r2t, in iscsi_tcp_get_curr_r2t()
1032 tcp_task->r2t = r2t = NULL; in iscsi_tcp_get_curr_r2t()
1037 if (kfifo_out(&tcp_task->r2tqueue, in iscsi_tcp_get_curr_r2t()
1038 (void *)&tcp_task->r2t, sizeof(void *)) != in iscsi_tcp_get_curr_r2t()
1042 r2t = tcp_task->r2t; in iscsi_tcp_get_curr_r2t()
1044 spin_unlock_bh(&tcp_task->queue2pool); in iscsi_tcp_get_curr_r2t()
1051 * iscsi_tcp_task_xmit - xmit normal PDU task
1055 * -EAGAIN if there's still data in the queue, or != 0 for any other kind
1060 struct iscsi_conn *conn = task->conn; in iscsi_tcp_task_xmit()
1061 struct iscsi_session *session = conn->session; in iscsi_tcp_task_xmit()
1067 rc = session->tt->xmit_pdu(task); in iscsi_tcp_task_xmit()
1072 if (!task->sc) { in iscsi_tcp_task_xmit()
1073 if (task->hdr->itt == RESERVED_ITT) in iscsi_tcp_task_xmit()
1079 if (task->sc->sc_data_direction != DMA_TO_DEVICE) in iscsi_tcp_task_xmit()
1089 rc = conn->session->tt->alloc_pdu(task, ISCSI_OP_SCSI_DATA_OUT); in iscsi_tcp_task_xmit()
1092 iscsi_prep_data_out_pdu(task, r2t, (struct iscsi_data *) task->hdr); in iscsi_tcp_task_xmit()
1095 r2t, r2t->datasn - 1, task->hdr->itt, in iscsi_tcp_task_xmit()
1096 r2t->data_offset + r2t->sent, r2t->data_count); in iscsi_tcp_task_xmit()
1098 rc = conn->session->tt->init_pdu(task, r2t->data_offset + r2t->sent, in iscsi_tcp_task_xmit()
1099 r2t->data_count); in iscsi_tcp_task_xmit()
1105 r2t->sent += r2t->data_count; in iscsi_tcp_task_xmit()
1123 conn = cls_conn->dd_data; in iscsi_tcp_conn_setup()
1128 conn->max_recv_dlength = ISCSI_DEF_MAX_RECV_SEG_LEN; in iscsi_tcp_conn_setup()
1130 tcp_conn = conn->dd_data; in iscsi_tcp_conn_setup()
1131 tcp_conn->iscsi_conn = conn; in iscsi_tcp_conn_setup()
1132 tcp_conn->dd_data = conn->dd_data + sizeof(*tcp_conn); in iscsi_tcp_conn_setup()
1149 * initialize per-task: R2T pool and xmit queue in iscsi_tcp_r2tpool_alloc()
1151 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) { in iscsi_tcp_r2tpool_alloc()
1152 struct iscsi_task *task = session->cmds[cmd_i]; in iscsi_tcp_r2tpool_alloc()
1153 struct iscsi_tcp_task *tcp_task = task->dd_data; in iscsi_tcp_r2tpool_alloc()
1156 * pre-allocated x2 as much r2ts to handle race when in iscsi_tcp_r2tpool_alloc()
1162 if (iscsi_pool_init(&tcp_task->r2tpool, in iscsi_tcp_r2tpool_alloc()
1163 session->max_r2t * 2, NULL, in iscsi_tcp_r2tpool_alloc()
1169 if (kfifo_alloc(&tcp_task->r2tqueue, in iscsi_tcp_r2tpool_alloc()
1170 session->max_r2t * 4 * sizeof(void*), GFP_KERNEL)) { in iscsi_tcp_r2tpool_alloc()
1171 iscsi_pool_free(&tcp_task->r2tpool); in iscsi_tcp_r2tpool_alloc()
1174 spin_lock_init(&tcp_task->pool2queue); in iscsi_tcp_r2tpool_alloc()
1175 spin_lock_init(&tcp_task->queue2pool); in iscsi_tcp_r2tpool_alloc()
1182 struct iscsi_task *task = session->cmds[i]; in iscsi_tcp_r2tpool_alloc()
1183 struct iscsi_tcp_task *tcp_task = task->dd_data; in iscsi_tcp_r2tpool_alloc()
1185 kfifo_free(&tcp_task->r2tqueue); in iscsi_tcp_r2tpool_alloc()
1186 iscsi_pool_free(&tcp_task->r2tpool); in iscsi_tcp_r2tpool_alloc()
1188 return -ENOMEM; in iscsi_tcp_r2tpool_alloc()
1196 for (i = 0; i < session->cmds_max; i++) { in iscsi_tcp_r2tpool_free()
1197 struct iscsi_task *task = session->cmds[i]; in iscsi_tcp_r2tpool_free()
1198 struct iscsi_tcp_task *tcp_task = task->dd_data; in iscsi_tcp_r2tpool_free()
1200 kfifo_free(&tcp_task->r2tqueue); in iscsi_tcp_r2tpool_free()
1201 iscsi_pool_free(&tcp_task->r2tpool); in iscsi_tcp_r2tpool_free()
1208 struct iscsi_session *session = conn->session; in iscsi_tcp_set_max_r2t()
1212 if (session->max_r2t == r2ts) in iscsi_tcp_set_max_r2t()
1216 return -EINVAL; in iscsi_tcp_set_max_r2t()
1218 session->max_r2t = r2ts; in iscsi_tcp_set_max_r2t()
1227 struct iscsi_conn *conn = cls_conn->dd_data; in iscsi_tcp_conn_get_stats()
1229 stats->txdata_octets = conn->txdata_octets; in iscsi_tcp_conn_get_stats()
1230 stats->rxdata_octets = conn->rxdata_octets; in iscsi_tcp_conn_get_stats()
1231 stats->scsicmd_pdus = conn->scsicmd_pdus_cnt; in iscsi_tcp_conn_get_stats()
1232 stats->dataout_pdus = conn->dataout_pdus_cnt; in iscsi_tcp_conn_get_stats()
1233 stats->scsirsp_pdus = conn->scsirsp_pdus_cnt; in iscsi_tcp_conn_get_stats()
1234 stats->datain_pdus = conn->datain_pdus_cnt; in iscsi_tcp_conn_get_stats()
1235 stats->r2t_pdus = conn->r2t_pdus_cnt; in iscsi_tcp_conn_get_stats()
1236 stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt; in iscsi_tcp_conn_get_stats()
1237 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt; in iscsi_tcp_conn_get_stats()