Lines Matching +full:no +full:- +full:memory +full:- +full:wc
3 * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved.
15 * - Redistributions of source code must retain the above
19 * - Redistributions in binary form must reproduce the above
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
43 /* Register user buffer memory and initialize passive rdma
45 * task->data[ISER_DIR_IN].data_len, Protection size
46 * os stored in task->prot[ISER_DIR_IN].data_len
51 struct iscsi_iser_task *iser_task = task->dd_data; in iser_prepare_read_cmd()
54 struct iser_ctrl *hdr = &iser_task->desc.iser_header; in iser_prepare_read_cmd()
64 iser_err("Failed to set up Data-IN RDMA\n"); in iser_prepare_read_cmd()
67 mem_reg = &iser_task->rdma_reg[ISER_DIR_IN]; in iser_prepare_read_cmd()
69 hdr->flags |= ISER_RSV; in iser_prepare_read_cmd()
70 hdr->read_stag = cpu_to_be32(mem_reg->rkey); in iser_prepare_read_cmd()
71 hdr->read_va = cpu_to_be64(mem_reg->sge.addr); in iser_prepare_read_cmd()
74 task->itt, mem_reg->rkey, in iser_prepare_read_cmd()
75 (unsigned long long)mem_reg->sge.addr); in iser_prepare_read_cmd()
84 /* Register user buffer memory and initialize passive rdma
86 * task->data[ISER_DIR_OUT].data_len, Protection size
87 * is stored at task->prot[ISER_DIR_OUT].data_len
92 struct iscsi_iser_task *iser_task = task->dd_data; in iser_prepare_write_cmd()
95 struct iser_ctrl *hdr = &iser_task->desc.iser_header; in iser_prepare_write_cmd()
96 struct iser_data_buf *buf_out = &iser_task->data[ISER_DIR_OUT]; in iser_prepare_write_cmd()
97 struct ib_sge *tx_dsg = &iser_task->desc.tx_sg[1]; in iser_prepare_write_cmd()
106 buf_out->data_len == imm_sz); in iser_prepare_write_cmd()
112 mem_reg = &iser_task->rdma_reg[ISER_DIR_OUT]; in iser_prepare_write_cmd()
115 hdr->flags |= ISER_WSV; in iser_prepare_write_cmd()
116 if (buf_out->data_len > imm_sz) { in iser_prepare_write_cmd()
117 hdr->write_stag = cpu_to_be32(mem_reg->rkey); in iser_prepare_write_cmd()
118 hdr->write_va = cpu_to_be64(mem_reg->sge.addr + unsol_sz); in iser_prepare_write_cmd()
122 task->itt, mem_reg->rkey, in iser_prepare_write_cmd()
123 (unsigned long long)mem_reg->sge.addr, unsol_sz); in iser_prepare_write_cmd()
128 task->itt, imm_sz); in iser_prepare_write_cmd()
129 tx_dsg->addr = mem_reg->sge.addr; in iser_prepare_write_cmd()
130 tx_dsg->length = imm_sz; in iser_prepare_write_cmd()
131 tx_dsg->lkey = mem_reg->sge.lkey; in iser_prepare_write_cmd()
132 iser_task->desc.num_sge = 2; in iser_prepare_write_cmd()
145 void (*done)(struct ib_cq *cq, struct ib_wc *wc)) in iser_create_send_desc() argument
147 struct iser_device *device = iser_conn->ib_conn.device; in iser_create_send_desc()
149 tx_desc->type = type; in iser_create_send_desc()
150 tx_desc->cqe.done = done; in iser_create_send_desc()
152 ib_dma_sync_single_for_cpu(device->ib_device, in iser_create_send_desc()
153 tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE); in iser_create_send_desc()
155 memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl)); in iser_create_send_desc()
156 tx_desc->iser_header.flags = ISER_VER; in iser_create_send_desc()
157 tx_desc->num_sge = 1; in iser_create_send_desc()
162 struct iser_device *device = iser_conn->ib_conn.device; in iser_free_login_buf()
163 struct iser_login_desc *desc = &iser_conn->login_desc; in iser_free_login_buf()
165 if (!desc->req) in iser_free_login_buf()
168 ib_dma_unmap_single(device->ib_device, desc->req_dma, in iser_free_login_buf()
171 ib_dma_unmap_single(device->ib_device, desc->rsp_dma, in iser_free_login_buf()
174 kfree(desc->req); in iser_free_login_buf()
175 kfree(desc->rsp); in iser_free_login_buf()
178 desc->req = NULL; in iser_free_login_buf()
179 desc->rsp = NULL; in iser_free_login_buf()
184 struct iser_device *device = iser_conn->ib_conn.device; in iser_alloc_login_buf()
185 struct iser_login_desc *desc = &iser_conn->login_desc; in iser_alloc_login_buf()
187 desc->req = kmalloc(ISCSI_DEF_MAX_RECV_SEG_LEN, GFP_KERNEL); in iser_alloc_login_buf()
188 if (!desc->req) in iser_alloc_login_buf()
189 return -ENOMEM; in iser_alloc_login_buf()
191 desc->req_dma = ib_dma_map_single(device->ib_device, desc->req, in iser_alloc_login_buf()
194 if (ib_dma_mapping_error(device->ib_device, in iser_alloc_login_buf()
195 desc->req_dma)) in iser_alloc_login_buf()
198 desc->rsp = kmalloc(ISER_RX_LOGIN_SIZE, GFP_KERNEL); in iser_alloc_login_buf()
199 if (!desc->rsp) in iser_alloc_login_buf()
202 desc->rsp_dma = ib_dma_map_single(device->ib_device, desc->rsp, in iser_alloc_login_buf()
205 if (ib_dma_mapping_error(device->ib_device, in iser_alloc_login_buf()
206 desc->rsp_dma)) in iser_alloc_login_buf()
212 kfree(desc->rsp); in iser_alloc_login_buf()
214 ib_dma_unmap_single(device->ib_device, desc->req_dma, in iser_alloc_login_buf()
218 kfree(desc->req); in iser_alloc_login_buf()
220 return -ENOMEM; in iser_alloc_login_buf()
230 struct ib_conn *ib_conn = &iser_conn->ib_conn; in iser_alloc_rx_descriptors()
231 struct iser_device *device = ib_conn->device; in iser_alloc_rx_descriptors()
233 iser_conn->qp_max_recv_dtos = session->cmds_max; in iser_alloc_rx_descriptors()
235 if (iser_alloc_fastreg_pool(ib_conn, session->scsi_cmds_max, in iser_alloc_rx_descriptors()
236 iser_conn->pages_per_mr)) in iser_alloc_rx_descriptors()
242 iser_conn->num_rx_descs = session->cmds_max; in iser_alloc_rx_descriptors()
243 iser_conn->rx_descs = kmalloc_array(iser_conn->num_rx_descs, in iser_alloc_rx_descriptors()
246 if (!iser_conn->rx_descs) in iser_alloc_rx_descriptors()
249 rx_desc = iser_conn->rx_descs; in iser_alloc_rx_descriptors()
251 for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++) { in iser_alloc_rx_descriptors()
252 dma_addr = ib_dma_map_single(device->ib_device, (void *)rx_desc, in iser_alloc_rx_descriptors()
254 if (ib_dma_mapping_error(device->ib_device, dma_addr)) in iser_alloc_rx_descriptors()
257 rx_desc->dma_addr = dma_addr; in iser_alloc_rx_descriptors()
258 rx_desc->cqe.done = iser_task_rsp; in iser_alloc_rx_descriptors()
259 rx_sg = &rx_desc->rx_sg; in iser_alloc_rx_descriptors()
260 rx_sg->addr = rx_desc->dma_addr; in iser_alloc_rx_descriptors()
261 rx_sg->length = ISER_RX_PAYLOAD_SIZE; in iser_alloc_rx_descriptors()
262 rx_sg->lkey = device->pd->local_dma_lkey; in iser_alloc_rx_descriptors()
268 rx_desc = iser_conn->rx_descs; in iser_alloc_rx_descriptors()
270 ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, in iser_alloc_rx_descriptors()
272 kfree(iser_conn->rx_descs); in iser_alloc_rx_descriptors()
273 iser_conn->rx_descs = NULL; in iser_alloc_rx_descriptors()
280 return -ENOMEM; in iser_alloc_rx_descriptors()
287 struct ib_conn *ib_conn = &iser_conn->ib_conn; in iser_free_rx_descriptors()
288 struct iser_device *device = ib_conn->device; in iser_free_rx_descriptors()
292 rx_desc = iser_conn->rx_descs; in iser_free_rx_descriptors()
293 for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++) in iser_free_rx_descriptors()
294 ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, in iser_free_rx_descriptors()
296 kfree(iser_conn->rx_descs); in iser_free_rx_descriptors()
298 iser_conn->rx_descs = NULL; in iser_free_rx_descriptors()
305 struct iser_conn *iser_conn = conn->dd_data; in iser_post_rx_bufs()
306 struct iscsi_session *session = conn->session; in iser_post_rx_bufs()
310 iser_dbg("req op %x flags %x\n", req->opcode, req->flags); in iser_post_rx_bufs()
311 /* check if this is the last login - going to full feature phase */ in iser_post_rx_bufs()
312 if ((req->flags & ISCSI_FULL_FEATURE_PHASE) != ISCSI_FULL_FEATURE_PHASE) in iser_post_rx_bufs()
315 if (session->discovery_sess) { in iser_post_rx_bufs()
316 iser_info("Discovery session, re-using login RX buffer\n"); in iser_post_rx_bufs()
321 iser_conn->qp_max_recv_dtos - 1); in iser_post_rx_bufs()
328 for (i = 1; i < iser_conn->qp_max_recv_dtos; i++) { in iser_post_rx_bufs()
329 err = iser_post_recvm(iser_conn, &iser_conn->rx_descs[i]); in iser_post_rx_bufs()
338 * iser_send_command - send command PDU
344 struct iser_conn *iser_conn = conn->dd_data; in iser_send_command()
345 struct iscsi_iser_task *iser_task = task->dd_data; in iser_send_command()
349 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)task->hdr; in iser_send_command()
350 struct scsi_cmnd *sc = task->sc; in iser_send_command()
351 struct iser_tx_desc *tx_desc = &iser_task->desc; in iser_send_command()
353 edtl = ntohl(hdr->data_length); in iser_send_command()
359 if (hdr->flags & ISCSI_FLAG_CMD_READ) { in iser_send_command()
360 data_buf = &iser_task->data[ISER_DIR_IN]; in iser_send_command()
361 prot_buf = &iser_task->prot[ISER_DIR_IN]; in iser_send_command()
363 data_buf = &iser_task->data[ISER_DIR_OUT]; in iser_send_command()
364 prot_buf = &iser_task->prot[ISER_DIR_OUT]; in iser_send_command()
368 data_buf->sg = scsi_sglist(sc); in iser_send_command()
369 data_buf->size = scsi_sg_count(sc); in iser_send_command()
371 data_buf->data_len = scsi_bufflen(sc); in iser_send_command()
374 prot_buf->sg = scsi_prot_sglist(sc); in iser_send_command()
375 prot_buf->size = scsi_prot_sg_count(sc); in iser_send_command()
376 prot_buf->data_len = (data_buf->data_len >> in iser_send_command()
377 ilog2(sc->device->sector_size)) * 8; in iser_send_command()
380 if (hdr->flags & ISCSI_FLAG_CMD_READ) { in iser_send_command()
385 if (hdr->flags & ISCSI_FLAG_CMD_WRITE) { in iser_send_command()
387 task->imm_count, in iser_send_command()
388 task->imm_count + in iser_send_command()
389 task->unsol_r2t.data_length, in iser_send_command()
395 iser_task->status = ISER_TASK_STATUS_STARTED; in iser_send_command()
397 err = iser_post_send(&iser_conn->ib_conn, tx_desc); in iser_send_command()
402 iser_err("conn %p failed task->itt %d err %d\n",conn, task->itt, err); in iser_send_command()
407 * iser_send_data_out - send data out PDU
415 struct iser_conn *iser_conn = conn->dd_data; in iser_send_data_out()
416 struct iscsi_iser_task *iser_task = task->dd_data; in iser_send_data_out()
425 itt = (__force uint32_t)hdr->itt; in iser_send_data_out()
426 data_seg_len = ntoh24(hdr->dlength); in iser_send_data_out()
427 buf_offset = ntohl(hdr->offset); in iser_send_data_out()
434 return -ENOMEM; in iser_send_data_out()
436 tx_desc->type = ISCSI_TX_DATAOUT; in iser_send_data_out()
437 tx_desc->cqe.done = iser_dataout_comp; in iser_send_data_out()
438 tx_desc->iser_header.flags = ISER_VER; in iser_send_data_out()
439 memcpy(&tx_desc->iscsi_header, hdr, sizeof(struct iscsi_hdr)); in iser_send_data_out()
446 mem_reg = &iser_task->rdma_reg[ISER_DIR_OUT]; in iser_send_data_out()
447 tx_dsg = &tx_desc->tx_sg[1]; in iser_send_data_out()
448 tx_dsg->addr = mem_reg->sge.addr + buf_offset; in iser_send_data_out()
449 tx_dsg->length = data_seg_len; in iser_send_data_out()
450 tx_dsg->lkey = mem_reg->sge.lkey; in iser_send_data_out()
451 tx_desc->num_sge = 2; in iser_send_data_out()
453 if (buf_offset + data_seg_len > iser_task->data[ISER_DIR_OUT].data_len) { in iser_send_data_out()
454 iser_err("Offset:%ld & DSL:%ld in Data-Out inconsistent with total len:%ld, itt:%d\n", in iser_send_data_out()
456 iser_task->data[ISER_DIR_OUT].data_len, itt); in iser_send_data_out()
457 err = -EINVAL; in iser_send_data_out()
460 iser_dbg("data-out itt: %d, offset: %ld, sz: %ld\n", in iser_send_data_out()
463 err = iser_post_send(&iser_conn->ib_conn, tx_desc); in iser_send_data_out()
475 struct iser_conn *iser_conn = conn->dd_data; in iser_send_control()
476 struct iscsi_iser_task *iser_task = task->dd_data; in iser_send_control()
477 struct iser_tx_desc *mdesc = &iser_task->desc; in iser_send_control()
486 device = iser_conn->ib_conn.device; in iser_send_control()
488 data_seg_len = ntoh24(task->hdr->dlength); in iser_send_control()
491 struct iser_login_desc *desc = &iser_conn->login_desc; in iser_send_control()
492 struct ib_sge *tx_dsg = &mdesc->tx_sg[1]; in iser_send_control()
494 if (task != conn->login_task) { in iser_send_control()
499 ib_dma_sync_single_for_cpu(device->ib_device, desc->req_dma, in iser_send_control()
500 task->data_count, DMA_TO_DEVICE); in iser_send_control()
502 memcpy(desc->req, task->data, task->data_count); in iser_send_control()
504 ib_dma_sync_single_for_device(device->ib_device, desc->req_dma, in iser_send_control()
505 task->data_count, DMA_TO_DEVICE); in iser_send_control()
507 tx_dsg->addr = desc->req_dma; in iser_send_control()
508 tx_dsg->length = task->data_count; in iser_send_control()
509 tx_dsg->lkey = device->pd->local_dma_lkey; in iser_send_control()
510 mdesc->num_sge = 2; in iser_send_control()
513 if (task == conn->login_task) { in iser_send_control()
515 task->hdr->opcode, data_seg_len); in iser_send_control()
519 err = iser_post_rx_bufs(conn, task->hdr); in iser_send_control()
524 err = iser_post_send(&iser_conn->ib_conn, mdesc); in iser_send_control()
533 void iser_login_rsp(struct ib_cq *cq, struct ib_wc *wc) in iser_login_rsp() argument
535 struct ib_conn *ib_conn = wc->qp->qp_context; in iser_login_rsp()
537 struct iser_login_desc *desc = iser_login(wc->wr_cqe); in iser_login_rsp()
543 if (unlikely(wc->status != IB_WC_SUCCESS)) { in iser_login_rsp()
544 iser_err_comp(wc, "login_rsp"); in iser_login_rsp()
548 ib_dma_sync_single_for_cpu(ib_conn->device->ib_device, in iser_login_rsp()
549 desc->rsp_dma, ISER_RX_LOGIN_SIZE, in iser_login_rsp()
552 hdr = desc->rsp + sizeof(struct iser_ctrl); in iser_login_rsp()
553 data = desc->rsp + ISER_HEADERS_LEN; in iser_login_rsp()
554 length = wc->byte_len - ISER_HEADERS_LEN; in iser_login_rsp()
555 full_feature_phase = ((hdr->flags & ISCSI_FULL_FEATURE_PHASE) == in iser_login_rsp()
557 (hdr->flags & ISCSI_FLAG_CMD_FINAL); in iser_login_rsp()
559 iser_dbg("op 0x%x itt 0x%x dlen %d\n", hdr->opcode, in iser_login_rsp()
560 hdr->itt, length); in iser_login_rsp()
562 iscsi_iser_recv(iser_conn->iscsi_conn, hdr, data, length); in iser_login_rsp()
564 ib_dma_sync_single_for_device(ib_conn->device->ib_device, in iser_login_rsp()
565 desc->rsp_dma, ISER_RX_LOGIN_SIZE, in iser_login_rsp()
569 iser_conn->iscsi_conn->session->discovery_sess) in iser_login_rsp()
573 iser_post_recvm(iser_conn, iser_conn->rx_descs); in iser_login_rsp()
578 if (unlikely((!desc->sig_protected && rkey != desc->rsc.mr->rkey) || in iser_inv_desc()
579 (desc->sig_protected && rkey != desc->rsc.sig_mr->rkey))) { in iser_inv_desc()
581 return -EINVAL; in iser_inv_desc()
584 if (desc->sig_protected) in iser_inv_desc()
585 desc->rsc.sig_mr->need_inval = false; in iser_inv_desc()
587 desc->rsc.mr->need_inval = false; in iser_inv_desc()
592 static int iser_check_remote_inv(struct iser_conn *iser_conn, struct ib_wc *wc, in iser_check_remote_inv() argument
595 if (wc->wc_flags & IB_WC_WITH_INVALIDATE) { in iser_check_remote_inv()
597 u32 rkey = wc->ex.invalidate_rkey; in iser_check_remote_inv()
602 if (unlikely(!iser_conn->snd_w_inv)) { in iser_check_remote_inv()
605 return -EPROTO; in iser_check_remote_inv()
608 task = iscsi_itt_to_ctask(iser_conn->iscsi_conn, hdr->itt); in iser_check_remote_inv()
610 struct iscsi_iser_task *iser_task = task->dd_data; in iser_check_remote_inv()
613 if (iser_task->dir[ISER_DIR_IN]) { in iser_check_remote_inv()
614 desc = iser_task->rdma_reg[ISER_DIR_IN].desc; in iser_check_remote_inv()
616 return -EINVAL; in iser_check_remote_inv()
619 if (iser_task->dir[ISER_DIR_OUT]) { in iser_check_remote_inv()
620 desc = iser_task->rdma_reg[ISER_DIR_OUT].desc; in iser_check_remote_inv()
622 return -EINVAL; in iser_check_remote_inv()
625 iser_err("failed to get task for itt=%d\n", hdr->itt); in iser_check_remote_inv()
626 return -EINVAL; in iser_check_remote_inv()
634 void iser_task_rsp(struct ib_cq *cq, struct ib_wc *wc) in iser_task_rsp() argument
636 struct ib_conn *ib_conn = wc->qp->qp_context; in iser_task_rsp()
638 struct iser_rx_desc *desc = iser_rx(wc->wr_cqe); in iser_task_rsp()
642 if (unlikely(wc->status != IB_WC_SUCCESS)) { in iser_task_rsp()
643 iser_err_comp(wc, "task_rsp"); in iser_task_rsp()
647 ib_dma_sync_single_for_cpu(ib_conn->device->ib_device, in iser_task_rsp()
648 desc->dma_addr, ISER_RX_PAYLOAD_SIZE, in iser_task_rsp()
651 hdr = &desc->iscsi_header; in iser_task_rsp()
652 length = wc->byte_len - ISER_HEADERS_LEN; in iser_task_rsp()
654 iser_dbg("op 0x%x itt 0x%x dlen %d\n", hdr->opcode, in iser_task_rsp()
655 hdr->itt, length); in iser_task_rsp()
657 if (iser_check_remote_inv(iser_conn, wc, hdr)) { in iser_task_rsp()
658 iscsi_conn_failure(iser_conn->iscsi_conn, in iser_task_rsp()
663 iscsi_iser_recv(iser_conn->iscsi_conn, hdr, desc->data, length); in iser_task_rsp()
665 ib_dma_sync_single_for_device(ib_conn->device->ib_device, in iser_task_rsp()
666 desc->dma_addr, ISER_RX_PAYLOAD_SIZE, in iser_task_rsp()
674 void iser_cmd_comp(struct ib_cq *cq, struct ib_wc *wc) in iser_cmd_comp() argument
676 if (unlikely(wc->status != IB_WC_SUCCESS)) in iser_cmd_comp()
677 iser_err_comp(wc, "command"); in iser_cmd_comp()
680 void iser_ctrl_comp(struct ib_cq *cq, struct ib_wc *wc) in iser_ctrl_comp() argument
682 struct iser_tx_desc *desc = iser_tx(wc->wr_cqe); in iser_ctrl_comp()
685 if (unlikely(wc->status != IB_WC_SUCCESS)) { in iser_ctrl_comp()
686 iser_err_comp(wc, "control"); in iser_ctrl_comp()
691 task = (void *)desc - sizeof(struct iscsi_task); in iser_ctrl_comp()
692 if (task->hdr->itt == RESERVED_ITT) in iser_ctrl_comp()
696 void iser_dataout_comp(struct ib_cq *cq, struct ib_wc *wc) in iser_dataout_comp() argument
698 struct iser_tx_desc *desc = iser_tx(wc->wr_cqe); in iser_dataout_comp()
699 struct ib_conn *ib_conn = wc->qp->qp_context; in iser_dataout_comp()
700 struct iser_device *device = ib_conn->device; in iser_dataout_comp()
702 if (unlikely(wc->status != IB_WC_SUCCESS)) in iser_dataout_comp()
703 iser_err_comp(wc, "dataout"); in iser_dataout_comp()
705 ib_dma_unmap_single(device->ib_device, desc->dma_addr, in iser_dataout_comp()
713 iser_task->status = ISER_TASK_STATUS_INIT; in iser_task_rdma_init()
715 iser_task->dir[ISER_DIR_IN] = 0; in iser_task_rdma_init()
716 iser_task->dir[ISER_DIR_OUT] = 0; in iser_task_rdma_init()
718 iser_task->data[ISER_DIR_IN].data_len = 0; in iser_task_rdma_init()
719 iser_task->data[ISER_DIR_OUT].data_len = 0; in iser_task_rdma_init()
721 iser_task->prot[ISER_DIR_IN].data_len = 0; in iser_task_rdma_init()
722 iser_task->prot[ISER_DIR_OUT].data_len = 0; in iser_task_rdma_init()
724 iser_task->prot[ISER_DIR_IN].dma_nents = 0; in iser_task_rdma_init()
725 iser_task->prot[ISER_DIR_OUT].dma_nents = 0; in iser_task_rdma_init()
727 memset(&iser_task->rdma_reg[ISER_DIR_IN], 0, in iser_task_rdma_init()
729 memset(&iser_task->rdma_reg[ISER_DIR_OUT], 0, in iser_task_rdma_init()
736 if (iser_task->dir[ISER_DIR_IN]) { in iser_task_rdma_finalize()
742 if (iser_task->dir[ISER_DIR_OUT]) { in iser_task_rdma_finalize()