64 				   struct iscsi_segment *segment);
81  * @segment: the buffer object
85  * This function sets up the segment so that subsequent
90 iscsi_tcp_segment_init_sg(struct iscsi_segment *segment,  in iscsi_tcp_segment_init_sg()  argument
93 	segment->sg = sg;  in iscsi_tcp_segment_init_sg()
94 	segment->sg_offset = offset;  in iscsi_tcp_segment_init_sg()
95 	segment->size = min(sg->length - offset,  in iscsi_tcp_segment_init_sg()
96 			    segment->total_size - segment->total_copied);  in iscsi_tcp_segment_init_sg()
97 	segment->data = NULL;  in iscsi_tcp_segment_init_sg()
102  * @segment: iscsi_segment
109 static void iscsi_tcp_segment_map(struct iscsi_segment *segment, int recv)  in iscsi_tcp_segment_map()  argument
113 	if (segment->data != NULL || !segment->sg)  in iscsi_tcp_segment_map()
116 	sg = segment->sg;  in iscsi_tcp_segment_map()
117 	BUG_ON(segment->sg_mapped);  in iscsi_tcp_segment_map()
135 		segment->atomic_mapped = true;  in iscsi_tcp_segment_map()
136 		segment->sg_mapped = kmap_atomic(sg_page(sg));  in iscsi_tcp_segment_map()
138 		segment->atomic_mapped = false;  in iscsi_tcp_segment_map()
140 		segment->sg_mapped = kmap(sg_page(sg));  in iscsi_tcp_segment_map()
143 	segment->data = segment->sg_mapped + sg->offset + segment->sg_offset;  in iscsi_tcp_segment_map()
146 void iscsi_tcp_segment_unmap(struct iscsi_segment *segment)  in iscsi_tcp_segment_unmap()  argument
148 	if (segment->sg_mapped) {  in iscsi_tcp_segment_unmap()
149 		if (segment->atomic_mapped)  in iscsi_tcp_segment_unmap()
150 			kunmap_atomic(segment->sg_mapped);  in iscsi_tcp_segment_unmap()
152 			kunmap(sg_page(segment->sg));  in iscsi_tcp_segment_unmap()
153 		segment->sg_mapped = NULL;  in iscsi_tcp_segment_unmap()
154 		segment->data = NULL;  in iscsi_tcp_segment_unmap()
163 iscsi_tcp_segment_splice_digest(struct iscsi_segment *segment, void *digest)  in iscsi_tcp_segment_splice_digest()  argument
165 	segment->data = digest;  in iscsi_tcp_segment_splice_digest()
166 	segment->digest_len = ISCSI_DIGEST_SIZE;  in iscsi_tcp_segment_splice_digest()
167 	segment->total_size += ISCSI_DIGEST_SIZE;  in iscsi_tcp_segment_splice_digest()
168 	segment->size = ISCSI_DIGEST_SIZE;  in iscsi_tcp_segment_splice_digest()
169 	segment->copied = 0;  in iscsi_tcp_segment_splice_digest()
170 	segment->sg = NULL;  in iscsi_tcp_segment_splice_digest()
171 	segment->crcp = NULL;  in iscsi_tcp_segment_splice_digest()
175  * iscsi_tcp_segment_done - check whether the segment is complete
177  * @segment: iscsi segment to check
181  * Check if we're done receiving this segment. If the receive
191 			   struct iscsi_segment *segment, int recv,  in iscsi_tcp_segment_done()  argument
197 		      segment->copied, copied, segment->size,  in iscsi_tcp_segment_done()
199 	if (segment->crcp && copied) {  in iscsi_tcp_segment_done()
200 		if (segment->data) {  in iscsi_tcp_segment_done()
201 			*segment->crcp = crc32c(*segment->crcp,  in iscsi_tcp_segment_done()
202 						segment->data + segment->copied,  in iscsi_tcp_segment_done()
207 			data = kmap_local_page(sg_page(segment->sg));  in iscsi_tcp_segment_done()
208 			*segment->crcp = crc32c(*segment->crcp,  in iscsi_tcp_segment_done()
209 						data + segment->copied +  in iscsi_tcp_segment_done()
210 						segment->sg_offset +  in iscsi_tcp_segment_done()
211 						segment->sg->offset,  in iscsi_tcp_segment_done()
217 	segment->copied += copied;  in iscsi_tcp_segment_done()
218 	if (segment->copied < segment->size) {  in iscsi_tcp_segment_done()
219 		iscsi_tcp_segment_map(segment, recv);  in iscsi_tcp_segment_done()
223 	segment->total_copied += segment->copied;  in iscsi_tcp_segment_done()
224 	segment->copied = 0;  in iscsi_tcp_segment_done()
225 	segment->size = 0;  in iscsi_tcp_segment_done()
228 	iscsi_tcp_segment_unmap(segment);  in iscsi_tcp_segment_done()
232 		      segment->total_copied, segment->total_size);  in iscsi_tcp_segment_done()
233 	if (segment->total_copied < segment->total_size) {  in iscsi_tcp_segment_done()
235 		iscsi_tcp_segment_init_sg(segment, sg_next(segment->sg),  in iscsi_tcp_segment_done()
237 		iscsi_tcp_segment_map(segment, recv);  in iscsi_tcp_segment_done()
238 		BUG_ON(segment->size == 0);  in iscsi_tcp_segment_done()
244 		pad = iscsi_padding(segment->total_copied);  in iscsi_tcp_segment_done()
248 			segment->total_size += pad;  in iscsi_tcp_segment_done()
249 			segment->size = pad;  in iscsi_tcp_segment_done()
250 			segment->data = segment->padbuf;  in iscsi_tcp_segment_done()
259 	if (segment->crcp) {  in iscsi_tcp_segment_done()
260 		put_unaligned_le32(~*segment->crcp, segment->digest);  in iscsi_tcp_segment_done()
261 		iscsi_tcp_segment_splice_digest(segment,  in iscsi_tcp_segment_done()
262 				 recv ? segment->recv_digest : segment->digest);  in iscsi_tcp_segment_done()
271  * iscsi_tcp_segment_recv - copy data to segment
273  * @segment: the buffer to copy to
288 		       struct iscsi_segment *segment, const void *ptr,  in iscsi_tcp_segment_recv()  argument
293 	while (!iscsi_tcp_segment_done(tcp_conn, segment, 1, copy)) {  in iscsi_tcp_segment_recv()
300 		copy = min(len - copied, segment->size - segment->copied);  in iscsi_tcp_segment_recv()
302 		memcpy(segment->data + segment->copied, ptr + copied, copy);  in iscsi_tcp_segment_recv()
318 		      struct iscsi_segment *segment)  in iscsi_tcp_dgst_verify()  argument
320 	if (!segment->digest_len)  in iscsi_tcp_dgst_verify()
323 	if (memcmp(segment->recv_digest, segment->digest,  in iscsi_tcp_dgst_verify()
324 		   segment->digest_len)) {  in iscsi_tcp_dgst_verify()
333  * Helper function to set up segment buffer
336 __iscsi_segment_init(struct iscsi_segment *segment, size_t size,  in __iscsi_segment_init()  argument
339 	memset(segment, 0, sizeof(*segment));  in __iscsi_segment_init()
340 	segment->total_size = size;  in __iscsi_segment_init()
341 	segment->done = done;  in __iscsi_segment_init()
344 		segment->crcp = crcp;  in __iscsi_segment_init()
350 iscsi_segment_init_linear(struct iscsi_segment *segment, void *data,  in iscsi_segment_init_linear()  argument
353 	__iscsi_segment_init(segment, size, done, crcp);  in iscsi_segment_init_linear()
354 	segment->data = data;  in iscsi_segment_init_linear()
355 	segment->size = size;  in iscsi_segment_init_linear()
360 iscsi_segment_seek_sg(struct iscsi_segment *segment,  in iscsi_segment_seek_sg()  argument
368 	__iscsi_segment_init(segment, size, done, crcp);  in iscsi_segment_seek_sg()
371 			iscsi_tcp_segment_init_sg(segment, sg, offset);  in iscsi_segment_seek_sg()
382  * iscsi_tcp_hdr_recv_prep - prep segment for hdr reception
394 	iscsi_segment_init_linear(&tcp_conn->in.segment,  in iscsi_tcp_hdr_recv_prep()
405 			 struct iscsi_segment *segment)  in iscsi_tcp_data_recv_done()  argument
410 	if (!iscsi_tcp_dgst_verify(tcp_conn, segment))  in iscsi_tcp_data_recv_done()
432 	iscsi_segment_init_linear(&tcp_conn->in.segment,  in iscsi_tcp_data_recv_prep()
647 			  struct iscsi_segment *segment)  in iscsi_tcp_process_data_in()  argument
653 	if (!iscsi_tcp_dgst_verify(tcp_conn, segment))  in iscsi_tcp_process_data_in()
742 			rc = iscsi_segment_seek_sg(&tcp_conn->in.segment,  in iscsi_tcp_hdr_dissect()
821  * @segment: the buffer segment being processed
829 			struct iscsi_segment *segment)  in iscsi_tcp_hdr_recv_done()  argument
839 	if (segment->copied == sizeof(struct iscsi_hdr) && hdr->hlength) {  in iscsi_tcp_hdr_recv_done()
849 		segment->total_size += ahslen;  in iscsi_tcp_hdr_recv_done()
850 		segment->size += ahslen;  in iscsi_tcp_hdr_recv_done()
859 		if (segment->digest_len == 0) {  in iscsi_tcp_hdr_recv_done()
862 			 * splice it in so we can increment the skb/segment  in iscsi_tcp_hdr_recv_done()
863 			 * counters in preparation for the data segment.  in iscsi_tcp_hdr_recv_done()
865 			iscsi_tcp_segment_splice_digest(segment,  in iscsi_tcp_hdr_recv_done()
866 							segment->recv_digest);  in iscsi_tcp_hdr_recv_done()
871 				      segment->total_copied - ISCSI_DIGEST_SIZE,  in iscsi_tcp_hdr_recv_done()
872 				      segment->digest);  in iscsi_tcp_hdr_recv_done()
874 		if (!iscsi_tcp_dgst_verify(tcp_conn, segment))  in iscsi_tcp_hdr_recv_done()
891 	return tcp_conn->in.segment.done == iscsi_tcp_hdr_recv_done;  in iscsi_tcp_recv_segment_is_hdr()
898  * @skb: network buffer with header and/or data segment
910 	struct iscsi_segment *segment = &tcp_conn->in.segment;  in iscsi_tcp_recv_skb()  local
930 		segment->total_copied = segment->total_size;  in iscsi_tcp_recv_skb()
946 		BUG_ON(segment->copied >= segment->size);  in iscsi_tcp_recv_skb()
950 		rc = iscsi_tcp_segment_recv(tcp_conn, segment, ptr, avail);  in iscsi_tcp_recv_skb()
954 		if (segment->total_copied >= segment->total_size) {  in iscsi_tcp_recv_skb()
962 	ISCSI_DBG_TCP(conn, "segment done\n");  in iscsi_tcp_recv_skb()
963 	rc = segment->done(tcp_conn, segment);  in iscsi_tcp_recv_skb()
970 	/* The done() functions sets up the next segment. */  in iscsi_tcp_recv_skb()