16  * OR NON-INFRINGEMENT, ARE DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH
22  * linux-drivers@broadcom.com
37 #include <linux/bsg-lib.h>
40 #include <scsi/libiscsi.h>
67 		"memory that can be allocated. Range is 16 - 128");
71 beiscsi_##_name##_disp(struct device *dev,\
74 	struct Scsi_Host *shost = class_to_shost(dev);\
77 			phba->attr_##_name);\
88 			    phba->attr_##_name, val); \
89 		phba->attr_##_name = val;\
95 		    "range allowed is ["#_minval" - "#_maxval"]\n", val);\
96 		return -EINVAL;\
101 beiscsi_##_name##_store(struct device *dev,\
105 	struct Scsi_Host  *shost = class_to_shost(dev);\
109 		return -EINVAL;\
111 		return -EINVAL;\
115 		return -EINVAL;\
123 		phba->attr_##_name = val;\
129 		    "range allowed is ["#_minval" - "#_maxval"]\n", val);\
130 	phba->attr_##_name = _defval;\
131 	return -EINVAL;\
221 	struct iscsi_task *abrt_task = iscsi_cmd(sc)->task;  in beiscsi_eh_abort()
231 	cls_session = starget_to_session(scsi_target(sc->device));  in beiscsi_eh_abort()
232 	session = cls_session->dd_data;  in beiscsi_eh_abort()
236 	spin_lock_bh(&session->back_lock);  in beiscsi_eh_abort()
237 	if (!abrt_task || !abrt_task->sc) {  in beiscsi_eh_abort()
238 		spin_unlock_bh(&session->back_lock);  in beiscsi_eh_abort()
243 		spin_unlock(&session->back_lock);  in beiscsi_eh_abort()
249 	abrt_io_task = abrt_task->dd_data;  in beiscsi_eh_abort()
250 	conn = abrt_task->conn;  in beiscsi_eh_abort()
251 	beiscsi_conn = conn->dd_data;  in beiscsi_eh_abort()
252 	phba = beiscsi_conn->phba;  in beiscsi_eh_abort()
256 			      abrt_io_task->pwrb_handle->pwrb, 1);  in beiscsi_eh_abort()
259 			      abrt_io_task->pwrb_handle->pwrb, 1);  in beiscsi_eh_abort()
261 	inv_tbl.cid = beiscsi_conn->beiscsi_conn_cid;  in beiscsi_eh_abort()
262 	inv_tbl.icd = abrt_io_task->psgl_handle->sgl_index;  in beiscsi_eh_abort()
263 	spin_unlock_bh(&session->back_lock);  in beiscsi_eh_abort()
293 	cls_session = starget_to_session(scsi_target(sc->device));  in beiscsi_eh_device_reset()
294 	session = cls_session->dd_data;  in beiscsi_eh_device_reset()
296 	spin_lock_bh(&session->frwd_lock);  in beiscsi_eh_device_reset()
297 	if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN) {  in beiscsi_eh_device_reset()
298 		spin_unlock_bh(&session->frwd_lock);  in beiscsi_eh_device_reset()
302 	conn = session->leadconn;  in beiscsi_eh_device_reset()
303 	beiscsi_conn = conn->dd_data;  in beiscsi_eh_device_reset()
304 	phba = beiscsi_conn->phba;  in beiscsi_eh_device_reset()
308 		spin_unlock_bh(&session->frwd_lock);  in beiscsi_eh_device_reset()
315 	spin_lock(&session->back_lock);  in beiscsi_eh_device_reset()
316 	for (i = 0; i < conn->session->cmds_max; i++) {  in beiscsi_eh_device_reset()
317 		task = conn->session->cmds[i];  in beiscsi_eh_device_reset()
318 		if (!task->sc)  in beiscsi_eh_device_reset()
321 		if (sc->device->lun != task->sc->device->lun)  in beiscsi_eh_device_reset()
336 			 * completing in libiscsi. Just ignore it here. When we  in beiscsi_eh_device_reset()
342 		io_task = task->dd_data;  in beiscsi_eh_device_reset()
346 				      io_task->pwrb_handle->pwrb, 1);  in beiscsi_eh_device_reset()
349 				      io_task->pwrb_handle->pwrb, 1);  in beiscsi_eh_device_reset()
352 		inv_tbl->tbl[nents].cid = beiscsi_conn->beiscsi_conn_cid;  in beiscsi_eh_device_reset()
353 		inv_tbl->tbl[nents].icd = io_task->psgl_handle->sgl_index;  in beiscsi_eh_device_reset()
354 		inv_tbl->task[nents] = task;  in beiscsi_eh_device_reset()
357 	spin_unlock(&session->back_lock);  in beiscsi_eh_device_reset()
358 	spin_unlock_bh(&session->frwd_lock);  in beiscsi_eh_device_reset()
371 	if (beiscsi_mgmt_invalidate_icds(phba, &inv_tbl->tbl[0], nents)) {  in beiscsi_eh_device_reset()
374 			    beiscsi_conn->beiscsi_conn_cid);  in beiscsi_eh_device_reset()
380 		iscsi_put_task(inv_tbl->task[i]);  in beiscsi_eh_device_reset()
388 /*------------------- PCI Driver operations and data ----------------- */
403 	.name = "Emulex 10Gbe open-iscsi Initiator Driver",
415 	.this_id = -1,
433 		dev_err(&pcidev->dev,  in beiscsi_hba_alloc()
434 			"beiscsi_hba_alloc - iscsi_host_alloc failed\n");  in beiscsi_hba_alloc()
437 	shost->max_id = BE2_MAX_SESSIONS - 1;  in beiscsi_hba_alloc()
438 	shost->max_channel = 0;  in beiscsi_hba_alloc()
439 	shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;  in beiscsi_hba_alloc()
440 	shost->max_lun = BEISCSI_NUM_MAX_LUN;  in beiscsi_hba_alloc()
441 	shost->transportt = beiscsi_scsi_transport;  in beiscsi_hba_alloc()
444 	phba->shost = shost;  in beiscsi_hba_alloc()
445 	phba->pcidev = pci_dev_get(pcidev);  in beiscsi_hba_alloc()
447 	phba->interface_handle = 0xFFFFFFFF;  in beiscsi_hba_alloc()
454 	if (phba->csr_va) {  in beiscsi_unmap_pci_function()
455 		iounmap(phba->csr_va);  in beiscsi_unmap_pci_function()
456 		phba->csr_va = NULL;  in beiscsi_unmap_pci_function()
458 	if (phba->db_va) {  in beiscsi_unmap_pci_function()
459 		iounmap(phba->db_va);  in beiscsi_unmap_pci_function()
460 		phba->db_va = NULL;  in beiscsi_unmap_pci_function()
462 	if (phba->pci_va) {  in beiscsi_unmap_pci_function()
463 		iounmap(phba->pci_va);  in beiscsi_unmap_pci_function()
464 		phba->pci_va = NULL;  in beiscsi_unmap_pci_function()
477 		return -ENOMEM;  in beiscsi_map_pci_bars()
478 	phba->ctrl.csr = addr;  in beiscsi_map_pci_bars()
479 	phba->csr_va = addr;  in beiscsi_map_pci_bars()
484 	phba->ctrl.db = addr;  in beiscsi_map_pci_bars()
485 	phba->db_va = addr;  in beiscsi_map_pci_bars()
487 	if (phba->generation == BE_GEN2)  in beiscsi_map_pci_bars()
497 	phba->ctrl.pcicfg = addr;  in beiscsi_map_pci_bars()
498 	phba->pci_va = addr;  in beiscsi_map_pci_bars()
503 	return -ENOMEM;  in beiscsi_map_pci_bars()
512 		dev_err(&pcidev->dev,  in beiscsi_enable_pci()
513 			"beiscsi_enable_pci - enable device failed\n");  in beiscsi_enable_pci()
519 		dev_err(&pcidev->dev,  in beiscsi_enable_pci()
520 				"beiscsi_enable_pci - request region failed\n");  in beiscsi_enable_pci()
525 	ret = dma_set_mask_and_coherent(&pcidev->dev, DMA_BIT_MASK(64));  in beiscsi_enable_pci()
527 		ret = dma_set_mask_and_coherent(&pcidev->dev, DMA_BIT_MASK(32));  in beiscsi_enable_pci()
529 			dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");  in beiscsi_enable_pci()
545 	struct be_ctrl_info *ctrl = &phba->ctrl;  in be_ctrl_init()
546 	struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;  in be_ctrl_init()
547 	struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;  in be_ctrl_init()
550 	ctrl->pdev = pdev;  in be_ctrl_init()
554 	mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;  in be_ctrl_init()
555 	mbox_mem_alloc->va = dma_alloc_coherent(&pdev->dev,  in be_ctrl_init()
556 			mbox_mem_alloc->size, &mbox_mem_alloc->dma, GFP_KERNEL);  in be_ctrl_init()
557 	if (!mbox_mem_alloc->va) {  in be_ctrl_init()
559 		return -ENOMEM;  in be_ctrl_init()
562 	mbox_mem_align->size = sizeof(struct be_mcc_mailbox);  in be_ctrl_init()
563 	mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);  in be_ctrl_init()
564 	mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);  in be_ctrl_init()
565 	memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));  in be_ctrl_init()
566 	mutex_init(&ctrl->mbox_lock);  in be_ctrl_init()
567 	spin_lock_init(&phba->ctrl.mcc_lock);  in be_ctrl_init()
573  * beiscsi_get_params()- Set the config paramters
592 		if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {  in beiscsi_get_params()
593 			icd_start = phba->fw_config.iscsi_icd_start[ulp_num];  in beiscsi_get_params()
594 			icd_count = phba->fw_config.iscsi_icd_count[ulp_num];  in beiscsi_get_params()
599 			align_mask = (icd_post_per_page - 1);  in beiscsi_get_params()
606 				phba->fw_config.  in beiscsi_get_params()
615 				icd_count_unavailable = ((icd_start_align -  in beiscsi_get_params()
617 							 (icd_count -  in beiscsi_get_params()
621 			phba->fw_config.iscsi_icd_count[ulp_num] = (icd_count -  in beiscsi_get_params()
629 					phba->fw_config.  in beiscsi_get_params()
631 					phba->fw_config.  in beiscsi_get_params()
638 	total_icd_count = phba->fw_config.iscsi_icd_count[ulp_num];  in beiscsi_get_params()
639 	phba->params.ios_per_ctrl = (total_icd_count -  in beiscsi_get_params()
642 	phba->params.cxns_per_ctrl = total_cid_count;  in beiscsi_get_params()
643 	phba->params.icds_per_ctrl = total_icd_count;  in beiscsi_get_params()
644 	phba->params.num_sge_per_io = BE2_SGE;  in beiscsi_get_params()
645 	phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;  in beiscsi_get_params()
646 	phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;  in beiscsi_get_params()
647 	phba->params.num_eq_entries = 1024;  in beiscsi_get_params()
648 	phba->params.num_cq_entries = 1024;  in beiscsi_get_params()
649 	phba->params.wrbs_per_cxn = 256;  in beiscsi_get_params()
675 	iowrite32(val, phba->db_va + DB_EQ_OFFSET);  in hwi_ring_eq_db()
679  * be_isr_mcc - The isr routine of the driver.
693 	eq = &pbe_eq->q;  in be_isr_mcc()
694 	phba =  pbe_eq->phba;  in be_isr_mcc()
695 	mcc = &phba->ctrl.mcc_obj.cq;  in be_isr_mcc()
699 	while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]  in be_isr_mcc()
701 		if (((eqe->dw[offsetof(struct amap_eq_entry,  in be_isr_mcc()
703 		     EQE_RESID_MASK) >> 16) == mcc->id) {  in be_isr_mcc()
712 		queue_work(phba->wq, &pbe_eq->mcc_work);  in be_isr_mcc()
713 		hwi_ring_eq_db(phba, eq->id, 1,	mcc_events, 1, 1);  in be_isr_mcc()
719  * be_isr_msix - The isr routine of the driver.
730 	eq = &pbe_eq->q;  in be_isr_msix()
732 	phba = pbe_eq->phba;  in be_isr_msix()
734 	hwi_ring_eq_db(phba, eq->id, 1,	0, 0, 1);  in be_isr_msix()
735 	irq_poll_sched(&pbe_eq->iopoll);  in be_isr_msix()
741  * be_isr - The isr routine of the driver.
759 	ctrl = &phba->ctrl;  in be_isr()
760 	isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +  in be_isr()
761 		       (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));  in be_isr()
765 	phwi_ctrlr = phba->phwi_ctrlr;  in be_isr()
766 	phwi_context = phwi_ctrlr->phwi_ctxt;  in be_isr()
767 	pbe_eq = &phwi_context->be_eq[0];  in be_isr()
769 	eq = &phwi_context->be_eq[0].q;  in be_isr()
770 	mcc = &phba->ctrl.mcc_obj.cq;  in be_isr()
775 	while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]  in be_isr()
777 		if (((eqe->dw[offsetof(struct amap_eq_entry,  in be_isr()
778 		      resource_id) / 32] & EQE_RESID_MASK) >> 16) == mcc->id)  in be_isr()
792 		queue_work(phba->wq, &pbe_eq->mcc_work);  in be_isr()
797 		irq_poll_sched(&pbe_eq->iopoll);  in be_isr()
798 	hwi_ring_eq_db(phba, eq->id, 0, (io_events + mcc_events), rearm, 1);  in be_isr()
807 	if (!phba->pcidev->msix_enabled) {  in beiscsi_free_irqs()
808 		if (phba->pcidev->irq)  in beiscsi_free_irqs()
809 			free_irq(phba->pcidev->irq, phba);  in beiscsi_free_irqs()
813 	phwi_context = phba->phwi_ctrlr->phwi_ctxt;  in beiscsi_free_irqs()
814 	for (i = 0; i <= phba->num_cpus; i++) {  in beiscsi_free_irqs()
815 		free_irq(pci_irq_vector(phba->pcidev, i),  in beiscsi_free_irqs()
816 			 &phwi_context->be_eq[i]);  in beiscsi_free_irqs()
817 		kfree(phba->msi_name[i]);  in beiscsi_free_irqs()
823 	struct pci_dev *pcidev = phba->pcidev;  in beiscsi_init_irqs()
828 	phwi_ctrlr = phba->phwi_ctrlr;  in beiscsi_init_irqs()
829 	phwi_context = phwi_ctrlr->phwi_ctxt;  in beiscsi_init_irqs()
831 	if (pcidev->msix_enabled) {  in beiscsi_init_irqs()
832 		for (i = 0; i < phba->num_cpus; i++) {  in beiscsi_init_irqs()
833 			phba->msi_name[i] = kasprintf(GFP_KERNEL,  in beiscsi_init_irqs()
835 						      phba->shost->host_no, i);  in beiscsi_init_irqs()
836 			if (!phba->msi_name[i]) {  in beiscsi_init_irqs()
837 				ret = -ENOMEM;  in beiscsi_init_irqs()
842 					  be_isr_msix, 0, phba->msi_name[i],  in beiscsi_init_irqs()
843 					  &phwi_context->be_eq[i]);  in beiscsi_init_irqs()
846 					    "BM_%d : %s-Failed to register msix for i = %d\n",  in beiscsi_init_irqs()
848 				kfree(phba->msi_name[i]);  in beiscsi_init_irqs()
852 		phba->msi_name[i] = kasprintf(GFP_KERNEL, "beiscsi_mcc_%02x",  in beiscsi_init_irqs()
853 					      phba->shost->host_no);  in beiscsi_init_irqs()
854 		if (!phba->msi_name[i]) {  in beiscsi_init_irqs()
855 			ret = -ENOMEM;  in beiscsi_init_irqs()
859 				  phba->msi_name[i], &phwi_context->be_eq[i]);  in beiscsi_init_irqs()
862 				    "BM_%d : %s-Failed to register beiscsi_msix_mcc\n",  in beiscsi_init_irqs()
864 			kfree(phba->msi_name[i]);  in beiscsi_init_irqs()
869 		ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,  in beiscsi_init_irqs()
873 				    "BM_%d : %s-Failed to register irq\n",  in beiscsi_init_irqs()
880 	for (j = i - 1; j >= 0; j--) {  in beiscsi_init_irqs()
881 		free_irq(pci_irq_vector(pcidev, i), &phwi_context->be_eq[j]);  in beiscsi_init_irqs()
882 		kfree(phba->msi_name[j]);  in beiscsi_init_irqs()
906 	iowrite32(val, phba->db_va + DB_CQ_OFFSET);  in hwi_ring_cq_db()
914 	spin_lock_irqsave(&phba->io_sgl_lock, flags);  in alloc_io_sgl_handle()
915 	if (phba->io_sgl_hndl_avbl) {  in alloc_io_sgl_handle()
919 			    phba->io_sgl_alloc_index);  in alloc_io_sgl_handle()
921 		psgl_handle = phba->io_sgl_hndl_base[phba->  in alloc_io_sgl_handle()
923 		phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;  in alloc_io_sgl_handle()
924 		phba->io_sgl_hndl_avbl--;  in alloc_io_sgl_handle()
925 		if (phba->io_sgl_alloc_index == (phba->params.  in alloc_io_sgl_handle()
926 						 ios_per_ctrl - 1))  in alloc_io_sgl_handle()
927 			phba->io_sgl_alloc_index = 0;  in alloc_io_sgl_handle()
929 			phba->io_sgl_alloc_index++;  in alloc_io_sgl_handle()
932 	spin_unlock_irqrestore(&phba->io_sgl_lock, flags);  in alloc_io_sgl_handle()
941 	spin_lock_irqsave(&phba->io_sgl_lock, flags);  in free_io_sgl_handle()
944 		    phba->io_sgl_free_index);  in free_io_sgl_handle()
946 	if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {  in free_io_sgl_handle()
953 			    phba->io_sgl_free_index,  in free_io_sgl_handle()
954 			    phba->io_sgl_hndl_base[phba->io_sgl_free_index]);  in free_io_sgl_handle()
955 		spin_unlock_irqrestore(&phba->io_sgl_lock, flags);  in free_io_sgl_handle()
958 	phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;  in free_io_sgl_handle()
959 	phba->io_sgl_hndl_avbl++;  in free_io_sgl_handle()
960 	if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))  in free_io_sgl_handle()
961 		phba->io_sgl_free_index = 0;  in free_io_sgl_handle()
963 		phba->io_sgl_free_index++;  in free_io_sgl_handle()
964 	spin_unlock_irqrestore(&phba->io_sgl_lock, flags);  in free_io_sgl_handle()
974 	spin_lock_irqsave(&pwrb_context->wrb_lock, flags);  in beiscsi_get_wrb_handle()
975 	if (!pwrb_context->wrb_handles_available) {  in beiscsi_get_wrb_handle()
976 		spin_unlock_irqrestore(&pwrb_context->wrb_lock, flags);  in beiscsi_get_wrb_handle()
979 	pwrb_handle = pwrb_context->pwrb_handle_base[pwrb_context->alloc_index];  in beiscsi_get_wrb_handle()
980 	pwrb_context->wrb_handles_available--;  in beiscsi_get_wrb_handle()
981 	if (pwrb_context->alloc_index == (wrbs_per_cxn - 1))  in beiscsi_get_wrb_handle()
982 		pwrb_context->alloc_index = 0;  in beiscsi_get_wrb_handle()
984 		pwrb_context->alloc_index++;  in beiscsi_get_wrb_handle()
985 	spin_unlock_irqrestore(&pwrb_context->wrb_lock, flags);  in beiscsi_get_wrb_handle()
988 		memset(pwrb_handle->pwrb, 0, sizeof(*pwrb_handle->pwrb));  in beiscsi_get_wrb_handle()
994  * alloc_wrb_handle - To allocate a wrb handle
1008 	phwi_ctrlr = phba->phwi_ctrlr;  in alloc_wrb_handle()
1009 	pwrb_context = &phwi_ctrlr->wrb_context[cri_index];  in alloc_wrb_handle()
1012 	return beiscsi_get_wrb_handle(pwrb_context, phba->params.wrbs_per_cxn);  in alloc_wrb_handle()
1022 	spin_lock_irqsave(&pwrb_context->wrb_lock, flags);  in beiscsi_put_wrb_handle()
1023 	pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;  in beiscsi_put_wrb_handle()
1024 	pwrb_context->wrb_handles_available++;  in beiscsi_put_wrb_handle()
1025 	if (pwrb_context->free_index == (wrbs_per_cxn - 1))  in beiscsi_put_wrb_handle()
1026 		pwrb_context->free_index = 0;  in beiscsi_put_wrb_handle()
1028 		pwrb_context->free_index++;  in beiscsi_put_wrb_handle()
1029 	pwrb_handle->pio_handle = NULL;  in beiscsi_put_wrb_handle()
1030 	spin_unlock_irqrestore(&pwrb_context->wrb_lock, flags);  in beiscsi_put_wrb_handle()
1034  * free_wrb_handle - To free the wrb handle back to pool
1047 			       phba->params.wrbs_per_cxn);  in free_wrb_handle()
1052 		    pwrb_handle, pwrb_context->free_index,  in free_wrb_handle()
1053 		    pwrb_context->wrb_handles_available);  in free_wrb_handle()
1061 	spin_lock_irqsave(&phba->mgmt_sgl_lock, flags);  in alloc_mgmt_sgl_handle()
1062 	if (phba->eh_sgl_hndl_avbl) {  in alloc_mgmt_sgl_handle()
1063 		psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];  in alloc_mgmt_sgl_handle()
1064 		phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;  in alloc_mgmt_sgl_handle()
1067 			    phba->eh_sgl_alloc_index,  in alloc_mgmt_sgl_handle()
1068 			    phba->eh_sgl_alloc_index);  in alloc_mgmt_sgl_handle()
1070 		phba->eh_sgl_hndl_avbl--;  in alloc_mgmt_sgl_handle()
1071 		if (phba->eh_sgl_alloc_index ==  in alloc_mgmt_sgl_handle()
1072 		    (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -  in alloc_mgmt_sgl_handle()
1074 			phba->eh_sgl_alloc_index = 0;  in alloc_mgmt_sgl_handle()
1076 			phba->eh_sgl_alloc_index++;  in alloc_mgmt_sgl_handle()
1079 	spin_unlock_irqrestore(&phba->mgmt_sgl_lock, flags);  in alloc_mgmt_sgl_handle()
1088 	spin_lock_irqsave(&phba->mgmt_sgl_lock, flags);  in free_mgmt_sgl_handle()
1092 		    phba->eh_sgl_free_index);  in free_mgmt_sgl_handle()
1094 	if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {  in free_mgmt_sgl_handle()
1102 			    phba->eh_sgl_free_index);  in free_mgmt_sgl_handle()
1103 		spin_unlock_irqrestore(&phba->mgmt_sgl_lock, flags);  in free_mgmt_sgl_handle()
1106 	phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;  in free_mgmt_sgl_handle()
1107 	phba->eh_sgl_hndl_avbl++;  in free_mgmt_sgl_handle()
1108 	if (phba->eh_sgl_free_index ==  in free_mgmt_sgl_handle()
1109 	    (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))  in free_mgmt_sgl_handle()
1110 		phba->eh_sgl_free_index = 0;  in free_mgmt_sgl_handle()
1112 		phba->eh_sgl_free_index++;  in free_mgmt_sgl_handle()
1113 	spin_unlock_irqrestore(&phba->mgmt_sgl_lock, flags);  in free_mgmt_sgl_handle()
1121 	struct beiscsi_io_task *io_task = task->dd_data;  in be_complete_io()
1123 				(struct be_status_bhs *)io_task->cmd_bhs;  in be_complete_io()
1124 	struct iscsi_conn *conn = beiscsi_conn->conn;  in be_complete_io()
1129 	exp_cmdsn = csol_cqe->exp_cmdsn;  in be_complete_io()
1130 	max_cmdsn = (csol_cqe->exp_cmdsn +  in be_complete_io()
1131 		     csol_cqe->cmd_wnd - 1);  in be_complete_io()
1132 	rsp = csol_cqe->i_resp;  in be_complete_io()
1133 	status = csol_cqe->i_sts;  in be_complete_io()
1134 	flags = csol_cqe->i_flags;  in be_complete_io()
1135 	resid = csol_cqe->res_cnt;  in be_complete_io()
1137 	if (!task->sc) {  in be_complete_io()
1138 		if (io_task->scsi_cmnd) {  in be_complete_io()
1139 			scsi_dma_unmap(io_task->scsi_cmnd);  in be_complete_io()
1140 			io_task->scsi_cmnd = NULL;  in be_complete_io()
1145 	task->sc->result = (DID_OK << 16) | status;  in be_complete_io()
1147 		task->sc->result = DID_ERROR << 16;  in be_complete_io()
1154 			task->sc->result = DID_ERROR << 16;  in be_complete_io()
1157 			scsi_set_resid(task->sc, resid);  in be_complete_io()
1158 			if (!status && (scsi_bufflen(task->sc) - resid <  in be_complete_io()
1159 			    task->sc->underflow))  in be_complete_io()
1160 				task->sc->result = DID_ERROR << 16;  in be_complete_io()
1166 		unsigned short *slen = (unsigned short *)sts_bhs->sense_info;  in be_complete_io()
1168 		sense = sts_bhs->sense_info + sizeof(unsigned short);  in be_complete_io()
1170 		memcpy(task->sc->sense_buffer, sense,  in be_complete_io()
1174 	if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ)  in be_complete_io()
1175 		conn->rxdata_octets += resid;  in be_complete_io()
1177 	if (io_task->scsi_cmnd) {  in be_complete_io()
1178 		scsi_dma_unmap(io_task->scsi_cmnd);  in be_complete_io()
1179 		io_task->scsi_cmnd = NULL;  in be_complete_io()
1190 	struct beiscsi_io_task *io_task = task->dd_data;  in be_complete_logout()
1191 	struct iscsi_conn *conn = beiscsi_conn->conn;  in be_complete_logout()
1193 	hdr = (struct iscsi_logout_rsp *)task->hdr;  in be_complete_logout()
1194 	hdr->opcode = ISCSI_OP_LOGOUT_RSP;  in be_complete_logout()
1195 	hdr->t2wait = 5;  in be_complete_logout()
1196 	hdr->t2retain = 0;  in be_complete_logout()
1197 	hdr->flags = csol_cqe->i_flags;  in be_complete_logout()
1198 	hdr->response = csol_cqe->i_resp;  in be_complete_logout()
1199 	hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn);  in be_complete_logout()
1200 	hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn +  in be_complete_logout()
1201 				     csol_cqe->cmd_wnd - 1);  in be_complete_logout()
1203 	hdr->dlength[0] = 0;  in be_complete_logout()
1204 	hdr->dlength[1] = 0;  in be_complete_logout()
1205 	hdr->dlength[2] = 0;  in be_complete_logout()
1206 	hdr->hlength = 0;  in be_complete_logout()
1207 	hdr->itt = io_task->libiscsi_itt;  in be_complete_logout()
1217 	struct iscsi_conn *conn = beiscsi_conn->conn;  in be_complete_tmf()
1218 	struct beiscsi_io_task *io_task = task->dd_data;  in be_complete_tmf()
1220 	hdr = (struct iscsi_tm_rsp *)task->hdr;  in be_complete_tmf()
1221 	hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;  in be_complete_tmf()
1222 	hdr->flags = csol_cqe->i_flags;  in be_complete_tmf()
1223 	hdr->response = csol_cqe->i_resp;  in be_complete_tmf()
1224 	hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn);  in be_complete_tmf()
1225 	hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn +  in be_complete_tmf()
1226 				     csol_cqe->cmd_wnd - 1);  in be_complete_tmf()
1228 	hdr->itt = io_task->libiscsi_itt;  in be_complete_tmf()
1243 	phwi_ctrlr = phba->phwi_ctrlr;  in hwi_complete_drvr_msgs()
1257 	pwrb_context = &phwi_ctrlr->wrb_context[cri_index];  in hwi_complete_drvr_msgs()
1258 	pwrb_handle = pwrb_context->pwrb_handle_basestd[wrb_index];  in hwi_complete_drvr_msgs()
1259 	session = beiscsi_conn->conn->session;  in hwi_complete_drvr_msgs()
1260 	spin_lock_bh(&session->back_lock);  in hwi_complete_drvr_msgs()
1261 	task = pwrb_handle->pio_handle;  in hwi_complete_drvr_msgs()
1264 	spin_unlock_bh(&session->back_lock);  in hwi_complete_drvr_msgs()
1273 	struct iscsi_conn *conn = beiscsi_conn->conn;  in be_complete_nopin_resp()
1274 	struct beiscsi_io_task *io_task = task->dd_data;  in be_complete_nopin_resp()
1276 	hdr = (struct iscsi_nopin *)task->hdr;  in be_complete_nopin_resp()
1277 	hdr->flags = csol_cqe->i_flags;  in be_complete_nopin_resp()
1278 	hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn);  in be_complete_nopin_resp()
1279 	hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn +  in be_complete_nopin_resp()
1280 				     csol_cqe->cmd_wnd - 1);  in be_complete_nopin_resp()
1282 	hdr->opcode = ISCSI_OP_NOOP_IN;  in be_complete_nopin_resp()
1283 	hdr->itt = io_task->libiscsi_itt;  in be_complete_nopin_resp()
1292 		csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe,  in adapter_get_sol_cqe()
1294 		csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe,  in adapter_get_sol_cqe()
1296 		csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe,  in adapter_get_sol_cqe()
1298 		csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe,  in adapter_get_sol_cqe()
1300 		csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe,  in adapter_get_sol_cqe()
1302 		csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe,  in adapter_get_sol_cqe()
1304 		csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe,  in adapter_get_sol_cqe()
1306 		csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe,  in adapter_get_sol_cqe()
1308 		csol_cqe->i_flags = AMAP_GET_BITS(struct amap_sol_cqe,  in adapter_get_sol_cqe()
1311 		csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe_v2,  in adapter_get_sol_cqe()
1313 		csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe_v2,  in adapter_get_sol_cqe()
1315 		csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe_v2,  in adapter_get_sol_cqe()
1317 		csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe_v2,  in adapter_get_sol_cqe()
1319 		csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2,  in adapter_get_sol_cqe()
1321 		csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe_v2,  in adapter_get_sol_cqe()
1325 			csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2,  in adapter_get_sol_cqe()
1328 			csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe_v2,  in adapter_get_sol_cqe()
1332 			csol_cqe->i_flags = ISCSI_FLAG_CMD_UNDERFLOW;  in adapter_get_sol_cqe()
1336 			csol_cqe->i_flags |= ISCSI_FLAG_CMD_OVERFLOW;  in adapter_get_sol_cqe()
1344 	struct iscsi_conn *conn = beiscsi_conn->conn;  in hwi_complete_cmd()
1345 	struct iscsi_session *session = conn->session;  in hwi_complete_cmd()
1354 	phwi_ctrlr = phba->phwi_ctrlr;  in hwi_complete_cmd()
1360 	pwrb_context = &phwi_ctrlr->wrb_context[cri_index];  in hwi_complete_cmd()
1362 	pwrb_handle = pwrb_context->pwrb_handle_basestd[  in hwi_complete_cmd()
1365 	spin_lock_bh(&session->back_lock);  in hwi_complete_cmd()
1366 	task = pwrb_handle->pio_handle;  in hwi_complete_cmd()
1368 		spin_unlock_bh(&session->back_lock);  in hwi_complete_cmd()
1371 	type = ((struct beiscsi_io_task *)task->dd_data)->wrb_type;  in hwi_complete_cmd()
1376 		if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==  in hwi_complete_cmd()
1384 		if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)  in hwi_complete_cmd()
1394 			    " %s- Solicited path\n", __func__);  in hwi_complete_cmd()
1411 	spin_unlock_bh(&session->back_lock);  in hwi_complete_cmd()
1416  * a. Unsolicited NOP-In (target initiated NOP-In)
1427 	struct beiscsi_hba *phba = beiscsi_conn->phba;  in beiscsi_complete_pdu()
1428 	struct iscsi_conn *conn = beiscsi_conn->conn;  in beiscsi_complete_pdu()
1451 		task = conn->login_task;  in beiscsi_complete_pdu()
1452 		io_task = task->dd_data;  in beiscsi_complete_pdu()
1454 		login_hdr->itt = io_task->libiscsi_itt;  in beiscsi_complete_pdu()
1471 	pasync_handle->is_final = 0;  in beiscsi_hdl_put_handle()
1472 	pasync_handle->buffer_len = 0;  in beiscsi_hdl_put_handle()
1473 	pasync_handle->in_use = 0;  in beiscsi_hdl_put_handle()
1474 	list_del_init(&pasync_handle->link);  in beiscsi_hdl_put_handle()
1485 	plist  = &pasync_ctx->async_entry[cri].wq.list;  in beiscsi_hdl_purge_handles()
1489 	INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wq.list);  in beiscsi_hdl_purge_handles()
1490 	pasync_ctx->async_entry[cri].wq.hdr_len = 0;  in beiscsi_hdl_purge_handles()
1491 	pasync_ctx->async_entry[cri].wq.bytes_received = 0;  in beiscsi_hdl_purge_handles()
1492 	pasync_ctx->async_entry[cri].wq.bytes_needed = 0;  in beiscsi_hdl_purge_handles()
1501 	struct beiscsi_hba *phba = beiscsi_conn->phba;  in beiscsi_hdl_get_handle()
1508 	cid = beiscsi_conn->beiscsi_conn_cid;  in beiscsi_hdl_get_handle()
1514 	 * - index in CQ entry gives the vertical index  in beiscsi_hdl_get_handle()
1515 	 * - address in CQ entry is the offset where the DMA last ended  in beiscsi_hdl_get_handle()
1516 	 * - final - no more notifications for this PDU  in beiscsi_hdl_get_handle()
1540 	phys_addr.u.a32.address_lo -= dpl;  in beiscsi_hdl_get_handle()
1547 		pasync_handle = pasync_ctx->async_entry[ci].header;  in beiscsi_hdl_get_handle()
1554 		pasync_handle = pasync_ctx->async_entry[ci].data;  in beiscsi_hdl_get_handle()
1561 	if (pasync_handle->pa.u.a64.address != phys_addr.u.a64.address ||  in beiscsi_hdl_get_handle()
1562 	    pasync_handle->index != ci) {  in beiscsi_hdl_get_handle()
1563 		/* driver bug - if ci does not match async handle index */  in beiscsi_hdl_get_handle()
1566 …	    "BM_%d : cid %u async PDU handle mismatch - addr in %cQE %llx at %u:addr in CQE %llx ci %u\n",  in beiscsi_hdl_get_handle()
1567 			    cid, pasync_handle->is_header ? 'H' : 'D',  in beiscsi_hdl_get_handle()
1568 			    pasync_handle->pa.u.a64.address,  in beiscsi_hdl_get_handle()
1569 			    pasync_handle->index,  in beiscsi_hdl_get_handle()
1571 		/* FW has stale address - attempt continuing by dropping */  in beiscsi_hdl_get_handle()
1583 	if (pasync_handle->in_use || !list_empty(&pasync_handle->link)) {  in beiscsi_hdl_get_handle()
1585 			    "BM_%d : cid %d async PDU handle in use - code %d ci %d addr %llx\n",  in beiscsi_hdl_get_handle()
1590 	list_del_init(&pasync_handle->link);  in beiscsi_hdl_get_handle()
1595 	pasync_handle->cri = cri;  in beiscsi_hdl_get_handle()
1596 	pasync_handle->is_final = final;  in beiscsi_hdl_get_handle()
1597 	pasync_handle->buffer_len = dpl;  in beiscsi_hdl_get_handle()
1598 	pasync_handle->in_use = 1;  in beiscsi_hdl_get_handle()
1608 	struct iscsi_session *session = beiscsi_conn->conn->session;  in beiscsi_hdl_fwd_pdu()
1610 	struct beiscsi_hba *phba = beiscsi_conn->phba;  in beiscsi_hdl_fwd_pdu()
1615 	plist = &pasync_ctx->async_entry[cri].wq.list;  in beiscsi_hdl_fwd_pdu()
1621 			phdr = pasync_handle->pbuffer;  in beiscsi_hdl_fwd_pdu()
1626 			pdata = pasync_handle->pbuffer;  in beiscsi_hdl_fwd_pdu()
1627 			dlen = pasync_handle->buffer_len;  in beiscsi_hdl_fwd_pdu()
1630 		if (!pasync_handle->buffer_len ||  in beiscsi_hdl_fwd_pdu()
1631 		    (dlen + pasync_handle->buffer_len) >  in beiscsi_hdl_fwd_pdu()
1632 		    pasync_ctx->async_data.buffer_size)  in beiscsi_hdl_fwd_pdu()
1634 		memcpy(pdata + dlen, pasync_handle->pbuffer,  in beiscsi_hdl_fwd_pdu()
1635 		       pasync_handle->buffer_len);  in beiscsi_hdl_fwd_pdu()
1636 		dlen += pasync_handle->buffer_len;  in beiscsi_hdl_fwd_pdu()
1639 	if (!plast_handle->is_final) {  in beiscsi_hdl_fwd_pdu()
1642 			    "BM_%d : cid %u %p fwd async PDU opcode %x with last handle missing - HL%u:DN%u:DR%u\n",  in beiscsi_hdl_fwd_pdu()
1643 			    beiscsi_conn->beiscsi_conn_cid, plast_handle,  in beiscsi_hdl_fwd_pdu()
1645 			    pasync_ctx->async_entry[cri].wq.hdr_len,  in beiscsi_hdl_fwd_pdu()
1646 			    pasync_ctx->async_entry[cri].wq.bytes_needed,  in beiscsi_hdl_fwd_pdu()
1647 			    pasync_ctx->async_entry[cri].wq.bytes_received);  in beiscsi_hdl_fwd_pdu()
1649 	spin_lock_bh(&session->back_lock);  in beiscsi_hdl_fwd_pdu()
1651 	spin_unlock_bh(&session->back_lock);  in beiscsi_hdl_fwd_pdu()
1662 	u16 cri = pasync_handle->cri;  in beiscsi_hdl_gather_pdu()
1668 	phba = beiscsi_conn->phba;  in beiscsi_hdl_gather_pdu()
1669 	wq = &pasync_ctx->async_entry[cri].wq;  in beiscsi_hdl_gather_pdu()
1670 	if (pasync_handle->is_header) {  in beiscsi_hdl_gather_pdu()
1672 		if (wq->hdr_len) {  in beiscsi_hdl_gather_pdu()
1676 		ppdu = pasync_handle->pbuffer;  in beiscsi_hdl_gather_pdu()
1682 		wq->hdr_len = pasync_handle->buffer_len;  in beiscsi_hdl_gather_pdu()
1683 		wq->bytes_received = 0;  in beiscsi_hdl_gather_pdu()
1684 		wq->bytes_needed = bytes_needed;  in beiscsi_hdl_gather_pdu()
1685 		list_add_tail(&pasync_handle->link, &wq->list);  in beiscsi_hdl_gather_pdu()
1691 		if (!wq->hdr_len || !wq->bytes_needed) {  in beiscsi_hdl_gather_pdu()
1695 		wq->bytes_received += pasync_handle->buffer_len;  in beiscsi_hdl_gather_pdu()
1697 		if (wq->bytes_received > wq->bytes_needed) {  in beiscsi_hdl_gather_pdu()
1701 		list_add_tail(&pasync_handle->link, &wq->list);  in beiscsi_hdl_gather_pdu()
1702 		if (wq->bytes_received == wq->bytes_needed)  in beiscsi_hdl_gather_pdu()
1710 		    "BM_%d : cid %u async PDU %s - def-%c:HL%u:DN%u:DR%u\n",  in beiscsi_hdl_gather_pdu()
1711 		    beiscsi_conn->beiscsi_conn_cid, err,  in beiscsi_hdl_gather_pdu()
1712 		    pasync_handle->is_header ? 'H' : 'D',  in beiscsi_hdl_gather_pdu()
1713 		    wq->hdr_len, wq->bytes_needed,  in beiscsi_hdl_gather_pdu()
1714 		    pasync_handle->buffer_len);  in beiscsi_hdl_gather_pdu()
1735 	phwi_ctrlr = phba->phwi_ctrlr;  in beiscsi_hdq_post_handles()
1738 		pasync_sge = pasync_ctx->async_header.ring_base;  in beiscsi_hdq_post_handles()
1739 		pi = pasync_ctx->async_header.pi;  in beiscsi_hdq_post_handles()
1740 		ring_id = phwi_ctrlr->default_pdu_hdr[ulp_num].id;  in beiscsi_hdq_post_handles()
1741 		doorbell_offset = phwi_ctrlr->default_pdu_hdr[ulp_num].  in beiscsi_hdq_post_handles()
1744 		pasync_sge = pasync_ctx->async_data.ring_base;  in beiscsi_hdq_post_handles()
1745 		pi = pasync_ctx->async_data.pi;  in beiscsi_hdq_post_handles()
1746 		ring_id = phwi_ctrlr->default_pdu_data[ulp_num].id;  in beiscsi_hdq_post_handles()
1747 		doorbell_offset = phwi_ctrlr->default_pdu_data[ulp_num].  in beiscsi_hdq_post_handles()
1753 			pasync_handle = pasync_ctx->async_entry[pi].header;  in beiscsi_hdq_post_handles()
1755 			pasync_handle = pasync_ctx->async_entry[pi].data;  in beiscsi_hdq_post_handles()
1756 		WARN_ON(pasync_handle->is_header != header);  in beiscsi_hdq_post_handles()
1757 		WARN_ON(pasync_handle->index != pi);  in beiscsi_hdq_post_handles()
1759 		if (nbuf == pasync_ctx->num_entries) {  in beiscsi_hdq_post_handles()
1761 			pasync_sge[pi].hi = pasync_handle->pa.u.a32.address_lo;  in beiscsi_hdq_post_handles()
1762 			pasync_sge[pi].lo = pasync_handle->pa.u.a32.address_hi;  in beiscsi_hdq_post_handles()
1764 		if (++pi == pasync_ctx->num_entries)  in beiscsi_hdq_post_handles()
1769 		pasync_ctx->async_header.pi = pi;  in beiscsi_hdq_post_handles()
1771 		pasync_ctx->async_data.pi = pi;  in beiscsi_hdq_post_handles()
1777 	iowrite32(doorbell, phba->db_va + doorbell_offset);  in beiscsi_hdq_post_handles()
1784 	struct beiscsi_hba *phba = beiscsi_conn->phba;  in beiscsi_hdq_process_compl()
1791 	phwi_ctrlr = phba->phwi_ctrlr;  in beiscsi_hdq_process_compl()
1792 	cid_cri = BE_GET_CRI_FROM_CID(beiscsi_conn->beiscsi_conn_cid);  in beiscsi_hdq_process_compl()
1816 	mcc_cq = &phba->ctrl.mcc_obj.cq;  in beiscsi_process_mcc_cq()
1818 	mcc_compl->flags = le32_to_cpu(mcc_compl->flags);  in beiscsi_process_mcc_cq()
1819 	while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) {  in beiscsi_process_mcc_cq()
1824 			hwi_ring_cq_db(phba, mcc_cq->id,  in beiscsi_process_mcc_cq()
1828 		if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) {  in beiscsi_process_mcc_cq()
1830 		} else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) {  in beiscsi_process_mcc_cq()
1831 			beiscsi_process_mcc_compl(&phba->ctrl, mcc_compl);  in beiscsi_process_mcc_cq()
1834 		mcc_compl->flags = 0;  in beiscsi_process_mcc_cq()
1837 		mcc_compl->flags = le32_to_cpu(mcc_compl->flags);  in beiscsi_process_mcc_cq()
1842 		hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1);  in beiscsi_process_mcc_cq()
1851 	phba = pbe_eq->phba;  in beiscsi_mcc_work()
1855 		hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);  in beiscsi_mcc_work()
1859  * beiscsi_process_cq()- Process the Completion Queue
1879 	cq = pbe_eq->cq;  in beiscsi_process_cq()
1881 	phba = pbe_eq->phba;  in beiscsi_process_cq()
1883 	while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &  in beiscsi_process_cq()
1890 		code = (sol->dw[offsetof(struct amap_sol_cqe, code) / 32] &  in beiscsi_process_cq()
1909 		ep = phba->ep_array[cri_index];  in beiscsi_process_cq()
1922 		beiscsi_ep = ep->dd_data;  in beiscsi_process_cq()
1923 		beiscsi_conn = beiscsi_ep->conn;  in beiscsi_process_cq()
1927 			hwi_ring_cq_db(phba, cq->id, 32, 0);  in beiscsi_process_cq()
1950 			spin_lock_bh(&phba->async_pdu_lock);  in beiscsi_process_cq()
1953 			spin_unlock_bh(&phba->async_pdu_lock);  in beiscsi_process_cq()
1961 			spin_lock_bh(&phba->async_pdu_lock);  in beiscsi_process_cq()
1964 			spin_unlock_bh(&phba->async_pdu_lock);  in beiscsi_process_cq()
1998 			spin_lock_bh(&phba->async_pdu_lock);  in beiscsi_process_cq()
2002 			spin_unlock_bh(&phba->async_pdu_lock);  in beiscsi_process_cq()
2024 				iscsi_conn_failure(beiscsi_conn->conn,  in beiscsi_process_cq()
2044 	hwi_ring_cq_db(phba, cq->id, num_processed, 1);  in beiscsi_process_cq()
2057 	phba = pbe_eq->phba;  in be_iopoll()
2064 	eq = &pbe_eq->q;  in be_iopoll()
2066 	while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] &  in be_iopoll()
2073 	hwi_ring_eq_db(phba, eq->id, 1, io_events, 0, 1);  in be_iopoll()
2076 	pbe_eq->cq_count += ret;  in be_iopoll()
2081 			    "BM_%d : rearm pbe_eq->q.id =%d ret %d\n",  in be_iopoll()
2082 			    pbe_eq->q.id, ret);  in be_iopoll()
2084 			hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);  in be_iopoll()
2101 		      io_task->bhs_pa.u.a32.address_lo);  in hwi_write_sgl_v2()
2103 		      io_task->bhs_pa.u.a32.address_hi);  in hwi_write_sgl_v2()
2137 	psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;  in hwi_write_sgl_v2()
2140 	AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);  in hwi_write_sgl_v2()
2143 		      io_task->bhs_pa.u.a32.address_hi);  in hwi_write_sgl_v2()
2145 		      io_task->bhs_pa.u.a32.address_lo);  in hwi_write_sgl_v2()
2180 	psgl--;  in hwi_write_sgl_v2()
2196 				      io_task->bhs_pa.u.a32.address_lo);  in hwi_write_sgl()
2198 				      io_task->bhs_pa.u.a32.address_hi);  in hwi_write_sgl()
2226 	psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;  in hwi_write_sgl()
2229 	AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);  in hwi_write_sgl()
2232 			io_task->bhs_pa.u.a32.address_hi);  in hwi_write_sgl()
2234 			io_task->bhs_pa.u.a32.address_lo);  in hwi_write_sgl()
2268 	psgl--;  in hwi_write_sgl()
2273  * hwi_write_buffer()- Populate the WRB with task info
2280 	struct beiscsi_io_task *io_task = task->dd_data;  in hwi_write_buffer()
2281 	struct beiscsi_conn *beiscsi_conn = io_task->conn;  in hwi_write_buffer()
2282 	struct beiscsi_hba *phba = beiscsi_conn->phba;  in hwi_write_buffer()
2285 	io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;  in hwi_write_buffer()
2287 				io_task->bhs_pa.u.a32.address_lo);  in hwi_write_buffer()
2289 				io_task->bhs_pa.u.a32.address_hi);  in hwi_write_buffer()
2291 	if (task->data) {  in hwi_write_buffer()
2294 		dsp_value = (task->data_count) ? 1 : 0;  in hwi_write_buffer()
2305 			io_task->mtask_addr = dma_map_single(&phba->pcidev->dev,  in hwi_write_buffer()
2306 							     task->data,  in hwi_write_buffer()
2307 							     task->data_count,  in hwi_write_buffer()
2309 			if (dma_mapping_error(&phba->pcidev->dev,  in hwi_write_buffer()
2310 						  io_task->mtask_addr))  in hwi_write_buffer()
2311 				return -ENOMEM;  in hwi_write_buffer()
2312 			io_task->mtask_data_count = task->data_count;  in hwi_write_buffer()
2314 			io_task->mtask_addr = 0;  in hwi_write_buffer()
2317 			      lower_32_bits(io_task->mtask_addr));  in hwi_write_buffer()
2319 			      upper_32_bits(io_task->mtask_addr));  in hwi_write_buffer()
2321 						task->data_count);  in hwi_write_buffer()
2326 		io_task->mtask_addr = 0;  in hwi_write_buffer()
2329 	psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;  in hwi_write_buffer()
2331 	AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);  in hwi_write_buffer()
2334 		      io_task->bhs_pa.u.a32.address_hi);  in hwi_write_buffer()
2336 		      io_task->bhs_pa.u.a32.address_lo);  in hwi_write_buffer()
2337 	if (task->data) {  in hwi_write_buffer()
2347 		if (task->data) {  in hwi_write_buffer()
2349 				      lower_32_bits(io_task->mtask_addr));  in hwi_write_buffer()
2351 				      upper_32_bits(io_task->mtask_addr));  in hwi_write_buffer()
2360  * beiscsi_find_mem_req()- Find mem needed
2370 	phba->params.hwi_ws_sz = sizeof(struct hwi_controller);  in beiscsi_find_mem_req()
2372 	phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *  in beiscsi_find_mem_req()
2374 	phba->mem_req[HWI_MEM_ADDN_CONTEXT] =  in beiscsi_find_mem_req()
2378 	phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)  in beiscsi_find_mem_req()
2379 	    * (phba->params.wrbs_per_cxn)  in beiscsi_find_mem_req()
2380 	    * phba->params.cxns_per_ctrl;  in beiscsi_find_mem_req()
2382 				 (phba->params.wrbs_per_cxn);  in beiscsi_find_mem_req()
2383 	phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *  in beiscsi_find_mem_req()
2384 				phba->params.cxns_per_ctrl);  in beiscsi_find_mem_req()
2386 	phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *  in beiscsi_find_mem_req()
2387 		phba->params.icds_per_ctrl;  in beiscsi_find_mem_req()
2388 	phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *  in beiscsi_find_mem_req()
2389 		phba->params.num_sge_per_io * phba->params.icds_per_ctrl;  in beiscsi_find_mem_req()
2391 		if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {  in beiscsi_find_mem_req()
2401 					       phba->params.defpdu_hdr_sz);  in beiscsi_find_mem_req()
2406 					       phba->params.defpdu_data_sz);  in beiscsi_find_mem_req()
2415 			phba->mem_req[mem_descr_index] =  in beiscsi_find_mem_req()
2421 			phba->mem_req[mem_descr_index] =  in beiscsi_find_mem_req()
2427 			phba->mem_req[mem_descr_index] =  in beiscsi_find_mem_req()
2433 			phba->mem_req[mem_descr_index] =  in beiscsi_find_mem_req()
2439 			phba->mem_req[mem_descr_index] =  in beiscsi_find_mem_req()
2445 			phba->mem_req[mem_descr_index] =  in beiscsi_find_mem_req()
2451 			phba->mem_req[mem_descr_index] =  in beiscsi_find_mem_req()
2457 			phba->mem_req[mem_descr_index] =  in beiscsi_find_mem_req()
2473 	phba->phwi_ctrlr = kzalloc(phba->params.hwi_ws_sz, GFP_KERNEL);  in beiscsi_alloc_mem()
2474 	if (!phba->phwi_ctrlr)  in beiscsi_alloc_mem()
2475 		return -ENOMEM;  in beiscsi_alloc_mem()
2478 	phwi_ctrlr = phba->phwi_ctrlr;  in beiscsi_alloc_mem()
2479 	phwi_ctrlr->wrb_context = kcalloc(phba->params.cxns_per_ctrl,  in beiscsi_alloc_mem()
2482 	if (!phwi_ctrlr->wrb_context) {  in beiscsi_alloc_mem()
2483 		kfree(phba->phwi_ctrlr);  in beiscsi_alloc_mem()
2484 		return -ENOMEM;  in beiscsi_alloc_mem()
2487 	phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),  in beiscsi_alloc_mem()
2489 	if (!phba->init_mem) {  in beiscsi_alloc_mem()
2490 		kfree(phwi_ctrlr->wrb_context);  in beiscsi_alloc_mem()
2491 		kfree(phba->phwi_ctrlr);  in beiscsi_alloc_mem()
2492 		return -ENOMEM;  in beiscsi_alloc_mem()
2499 		kfree(phba->init_mem);  in beiscsi_alloc_mem()
2500 		kfree(phwi_ctrlr->wrb_context);  in beiscsi_alloc_mem()
2501 		kfree(phba->phwi_ctrlr);  in beiscsi_alloc_mem()
2502 		return -ENOMEM;  in beiscsi_alloc_mem()
2505 	mem_descr = phba->init_mem;  in beiscsi_alloc_mem()
2507 		if (!phba->mem_req[i]) {  in beiscsi_alloc_mem()
2508 			mem_descr->mem_array = NULL;  in beiscsi_alloc_mem()
2515 		alloc_size = phba->mem_req[i];  in beiscsi_alloc_mem()
2520 			mem_arr->virtual_address =  in beiscsi_alloc_mem()
2521 				dma_alloc_coherent(&phba->pcidev->dev,  in beiscsi_alloc_mem()
2523 			if (!mem_arr->virtual_address) {  in beiscsi_alloc_mem()
2526 				if (curr_alloc_size -  in beiscsi_alloc_mem()
2533 				mem_arr->bus_address.u.  in beiscsi_alloc_mem()
2535 				mem_arr->size = curr_alloc_size;  in beiscsi_alloc_mem()
2536 				alloc_size -= curr_alloc_size;  in beiscsi_alloc_mem()
2543 		mem_descr->num_elements = j;  in beiscsi_alloc_mem()
2544 		mem_descr->size_in_bytes = phba->mem_req[i];  in beiscsi_alloc_mem()
2545 		mem_descr->mem_array = kmalloc_array(j, sizeof(*mem_arr),  in beiscsi_alloc_mem()
2547 		if (!mem_descr->mem_array)  in beiscsi_alloc_mem()
2550 		memcpy(mem_descr->mem_array, mem_arr_orig,  in beiscsi_alloc_mem()
2557 	mem_descr->num_elements = j;  in beiscsi_alloc_mem()
2559 		for (j = mem_descr->num_elements; j > 0; j--) {  in beiscsi_alloc_mem()
2560 			dma_free_coherent(&phba->pcidev->dev,  in beiscsi_alloc_mem()
2561 					    mem_descr->mem_array[j - 1].size,  in beiscsi_alloc_mem()
2562 					    mem_descr->mem_array[j - 1].  in beiscsi_alloc_mem()
2564 					    (unsigned long)mem_descr->  in beiscsi_alloc_mem()
2565 					    mem_array[j - 1].  in beiscsi_alloc_mem()
2569 			i--;  in beiscsi_alloc_mem()
2570 			kfree(mem_descr->mem_array);  in beiscsi_alloc_mem()
2571 			mem_descr--;  in beiscsi_alloc_mem()
2575 	kfree(phba->init_mem);  in beiscsi_alloc_mem()
2576 	kfree(phba->phwi_ctrlr->wrb_context);  in beiscsi_alloc_mem()
2577 	kfree(phba->phwi_ctrlr);  in beiscsi_alloc_mem()
2578 	return -ENOMEM;  in beiscsi_alloc_mem()
2593 	mem_descr = phba->init_mem;  in iscsi_init_global_templates()
2596 	    (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;  in iscsi_init_global_templates()
2603 	    (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].  in iscsi_init_global_templates()
2623 	mem_descr_wrbh = phba->init_mem;  in beiscsi_init_wrb_handle()
2626 	mem_descr_wrb = phba->init_mem;  in beiscsi_init_wrb_handle()
2628 	phwi_ctrlr = phba->phwi_ctrlr;  in beiscsi_init_wrb_handle()
2631 	phwi_ctxt = phwi_ctrlr->phwi_ctxt;  in beiscsi_init_wrb_handle()
2632 	phwi_ctxt->be_wrbq = kcalloc(phba->params.cxns_per_ctrl,  in beiscsi_init_wrb_handle()
2635 	if (!phwi_ctxt->be_wrbq) {  in beiscsi_init_wrb_handle()
2638 		return -ENOMEM;  in beiscsi_init_wrb_handle()
2641 	for (index = 0; index < phba->params.cxns_per_ctrl; index++) {  in beiscsi_init_wrb_handle()
2642 		pwrb_context = &phwi_ctrlr->wrb_context[index];  in beiscsi_init_wrb_handle()
2643 		pwrb_context->pwrb_handle_base =  in beiscsi_init_wrb_handle()
2644 				kcalloc(phba->params.wrbs_per_cxn,  in beiscsi_init_wrb_handle()
2647 		if (!pwrb_context->pwrb_handle_base) {  in beiscsi_init_wrb_handle()
2652 		pwrb_context->pwrb_handle_basestd =  in beiscsi_init_wrb_handle()
2653 				kcalloc(phba->params.wrbs_per_cxn,  in beiscsi_init_wrb_handle()
2656 		if (!pwrb_context->pwrb_handle_basestd) {  in beiscsi_init_wrb_handle()
2663 				mem_descr_wrbh->mem_array[idx].virtual_address;  in beiscsi_init_wrb_handle()
2664 			num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /  in beiscsi_init_wrb_handle()
2666 					 phba->params.wrbs_per_cxn));  in beiscsi_init_wrb_handle()
2669 		pwrb_context->alloc_index = 0;  in beiscsi_init_wrb_handle()
2670 		pwrb_context->wrb_handles_available = 0;  in beiscsi_init_wrb_handle()
2671 		pwrb_context->free_index = 0;  in beiscsi_init_wrb_handle()
2674 			for (j = 0; j < phba->params.wrbs_per_cxn; j++) {  in beiscsi_init_wrb_handle()
2675 				pwrb_context->pwrb_handle_base[j] = pwrb_handle;  in beiscsi_init_wrb_handle()
2676 				pwrb_context->pwrb_handle_basestd[j] =  in beiscsi_init_wrb_handle()
2678 				pwrb_context->wrb_handles_available++;  in beiscsi_init_wrb_handle()
2679 				pwrb_handle->wrb_index = j;  in beiscsi_init_wrb_handle()
2682 			num_cxn_wrbh--;  in beiscsi_init_wrb_handle()
2684 		spin_lock_init(&pwrb_context->wrb_lock);  in beiscsi_init_wrb_handle()
2687 	for (index = 0; index < phba->params.cxns_per_ctrl; index++) {  in beiscsi_init_wrb_handle()
2688 		pwrb_context = &phwi_ctrlr->wrb_context[index];  in beiscsi_init_wrb_handle()
2690 			pwrb = mem_descr_wrb->mem_array[idx].virtual_address;  in beiscsi_init_wrb_handle()
2691 			num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /  in beiscsi_init_wrb_handle()
2693 				  phba->params.wrbs_per_cxn));  in beiscsi_init_wrb_handle()
2698 			for (j = 0; j < phba->params.wrbs_per_cxn; j++) {  in beiscsi_init_wrb_handle()
2699 				pwrb_handle = pwrb_context->pwrb_handle_base[j];  in beiscsi_init_wrb_handle()
2700 				pwrb_handle->pwrb = pwrb;  in beiscsi_init_wrb_handle()
2703 			num_cxn_wrb--;  in beiscsi_init_wrb_handle()
2708 	for (j = index; j > 0; j--) {  in beiscsi_init_wrb_handle()
2709 		pwrb_context = &phwi_ctrlr->wrb_context[j];  in beiscsi_init_wrb_handle()
2710 		kfree(pwrb_context->pwrb_handle_base);  in beiscsi_init_wrb_handle()
2711 		kfree(pwrb_context->pwrb_handle_basestd);  in beiscsi_init_wrb_handle()
2713 	kfree(phwi_ctxt->be_wrbq);  in beiscsi_init_wrb_handle()
2714 	return -ENOMEM;  in beiscsi_init_wrb_handle()
2721 	struct hba_parameters *p = &phba->params;  in hwi_init_async_pdu_ctx()
2728 		if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {  in hwi_init_async_pdu_ctx()
2730 			mem_descr = (struct be_mem_descriptor *)phba->init_mem;  in hwi_init_async_pdu_ctx()
2734 			phwi_ctrlr = phba->phwi_ctrlr;  in hwi_init_async_pdu_ctx()
2735 			phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num] =  in hwi_init_async_pdu_ctx()
2737 				 mem_descr->mem_array[0].virtual_address;  in hwi_init_async_pdu_ctx()
2739 			pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num];  in hwi_init_async_pdu_ctx()
2742 			pasync_ctx->async_entry =  in hwi_init_async_pdu_ctx()
2747 			pasync_ctx->num_entries = BEISCSI_ASYNC_HDQ_SIZE(phba,  in hwi_init_async_pdu_ctx()
2750 			mem_descr = (struct be_mem_descriptor *)phba->init_mem;  in hwi_init_async_pdu_ctx()
2753 			if (mem_descr->mem_array[0].virtual_address) {  in hwi_init_async_pdu_ctx()
2758 					    mem_descr->mem_array[0].  in hwi_init_async_pdu_ctx()
2766 			pasync_ctx->async_header.pi = 0;  in hwi_init_async_pdu_ctx()
2767 			pasync_ctx->async_header.buffer_size = p->defpdu_hdr_sz;  in hwi_init_async_pdu_ctx()
2768 			pasync_ctx->async_header.va_base =  in hwi_init_async_pdu_ctx()
2769 				mem_descr->mem_array[0].virtual_address;  in hwi_init_async_pdu_ctx()
2771 			pasync_ctx->async_header.pa_base.u.a64.address =  in hwi_init_async_pdu_ctx()
2772 				mem_descr->mem_array[0].  in hwi_init_async_pdu_ctx()
2776 			mem_descr = (struct be_mem_descriptor *)phba->init_mem;  in hwi_init_async_pdu_ctx()
2779 			if (mem_descr->mem_array[0].virtual_address) {  in hwi_init_async_pdu_ctx()
2784 					    mem_descr->mem_array[0].  in hwi_init_async_pdu_ctx()
2792 			pasync_ctx->async_header.ring_base =  in hwi_init_async_pdu_ctx()
2793 				mem_descr->mem_array[0].virtual_address;  in hwi_init_async_pdu_ctx()
2796 			mem_descr = (struct be_mem_descriptor *)phba->init_mem;  in hwi_init_async_pdu_ctx()
2799 			if (mem_descr->mem_array[0].virtual_address) {  in hwi_init_async_pdu_ctx()
2804 					    mem_descr->mem_array[0].  in hwi_init_async_pdu_ctx()
2812 			pasync_ctx->async_header.handle_base =  in hwi_init_async_pdu_ctx()
2813 				mem_descr->mem_array[0].virtual_address;  in hwi_init_async_pdu_ctx()
2816 			mem_descr = (struct be_mem_descriptor *)phba->init_mem;  in hwi_init_async_pdu_ctx()
2819 			if (mem_descr->mem_array[0].virtual_address) {  in hwi_init_async_pdu_ctx()
2824 					    mem_descr->mem_array[0].  in hwi_init_async_pdu_ctx()
2832 			pasync_ctx->async_data.ring_base =  in hwi_init_async_pdu_ctx()
2833 				mem_descr->mem_array[0].virtual_address;  in hwi_init_async_pdu_ctx()
2836 			mem_descr = (struct be_mem_descriptor *)phba->init_mem;  in hwi_init_async_pdu_ctx()
2839 			if (!mem_descr->mem_array[0].virtual_address)  in hwi_init_async_pdu_ctx()
2845 			pasync_ctx->async_data.handle_base =  in hwi_init_async_pdu_ctx()
2846 				mem_descr->mem_array[0].virtual_address;  in hwi_init_async_pdu_ctx()
2850 				pasync_ctx->async_header.handle_base;  in hwi_init_async_pdu_ctx()
2853 				pasync_ctx->async_data.handle_base;  in hwi_init_async_pdu_ctx()
2856 			mem_descr = (struct be_mem_descriptor *)phba->init_mem;  in hwi_init_async_pdu_ctx()
2859 			if (mem_descr->mem_array[0].virtual_address) {  in hwi_init_async_pdu_ctx()
2864 					    mem_descr->mem_array[0].  in hwi_init_async_pdu_ctx()
2873 			pasync_ctx->async_data.pi = 0;  in hwi_init_async_pdu_ctx()
2874 			pasync_ctx->async_data.buffer_size = p->defpdu_data_sz;  in hwi_init_async_pdu_ctx()
2875 			pasync_ctx->async_data.va_base =  in hwi_init_async_pdu_ctx()
2876 				mem_descr->mem_array[idx].virtual_address;  in hwi_init_async_pdu_ctx()
2877 			pasync_ctx->async_data.pa_base.u.a64.address =  in hwi_init_async_pdu_ctx()
2878 				mem_descr->mem_array[idx].  in hwi_init_async_pdu_ctx()
2881 			num_async_data = ((mem_descr->mem_array[idx].size) /  in hwi_init_async_pdu_ctx()
2882 					phba->params.defpdu_data_sz);  in hwi_init_async_pdu_ctx()
2887 				pasync_header_h->cri = -1;  in hwi_init_async_pdu_ctx()
2888 				pasync_header_h->is_header = 1;  in hwi_init_async_pdu_ctx()
2889 				pasync_header_h->index = index;  in hwi_init_async_pdu_ctx()
2890 				INIT_LIST_HEAD(&pasync_header_h->link);  in hwi_init_async_pdu_ctx()
2891 				pasync_header_h->pbuffer =  in hwi_init_async_pdu_ctx()
2893 						 (pasync_ctx->  in hwi_init_async_pdu_ctx()
2895 						 (p->defpdu_hdr_sz * index));  in hwi_init_async_pdu_ctx()
2897 				pasync_header_h->pa.u.a64.address =  in hwi_init_async_pdu_ctx()
2898 					pasync_ctx->async_header.pa_base.u.a64.  in hwi_init_async_pdu_ctx()
2899 					address + (p->defpdu_hdr_sz * index);  in hwi_init_async_pdu_ctx()
2901 				pasync_ctx->async_entry[index].header =  in hwi_init_async_pdu_ctx()
2904 				INIT_LIST_HEAD(&pasync_ctx->async_entry[index].  in hwi_init_async_pdu_ctx()
2907 				pasync_data_h->cri = -1;  in hwi_init_async_pdu_ctx()
2908 				pasync_data_h->is_header = 0;  in hwi_init_async_pdu_ctx()
2909 				pasync_data_h->index = index;  in hwi_init_async_pdu_ctx()
2910 				INIT_LIST_HEAD(&pasync_data_h->link);  in hwi_init_async_pdu_ctx()
2915 					pasync_ctx->async_data.va_base =  in hwi_init_async_pdu_ctx()
2916 						mem_descr->mem_array[idx].  in hwi_init_async_pdu_ctx()
2918 					pasync_ctx->async_data.pa_base.u.  in hwi_init_async_pdu_ctx()
2920 						mem_descr->mem_array[idx].  in hwi_init_async_pdu_ctx()
2923 						((mem_descr->mem_array[idx].  in hwi_init_async_pdu_ctx()
2925 						 phba->params.defpdu_data_sz);  in hwi_init_async_pdu_ctx()
2927 				pasync_data_h->pbuffer =  in hwi_init_async_pdu_ctx()
2929 					(pasync_ctx->async_data.va_base) +  in hwi_init_async_pdu_ctx()
2930 					(p->defpdu_data_sz * num_per_mem));  in hwi_init_async_pdu_ctx()
2932 				pasync_data_h->pa.u.a64.address =  in hwi_init_async_pdu_ctx()
2933 					pasync_ctx->async_data.pa_base.u.a64.  in hwi_init_async_pdu_ctx()
2934 					address + (p->defpdu_data_sz *  in hwi_init_async_pdu_ctx()
2937 				num_async_data--;  in hwi_init_async_pdu_ctx()
2939 				pasync_ctx->async_entry[index].data =  in hwi_init_async_pdu_ctx()
2959 	sgl->va = virtual_address;  in be_sgl_create_contiguous()
2960 	sgl->dma = (unsigned long)physical_address;  in be_sgl_create_contiguous()
2961 	sgl->size = length;  in be_sgl_create_contiguous()
2975 	if (sgl->va)  in hwi_build_be_sgl_arr()
2978 	be_sgl_create_contiguous(pmem->virtual_address,  in hwi_build_be_sgl_arr()
2979 				 pmem->bus_address.u.a64.address,  in hwi_build_be_sgl_arr()
2980 				 pmem->size, sgl);  in hwi_build_be_sgl_arr()
2987 	if (sgl->va)  in hwi_build_be_sgl_by_offset()
2990 	be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,  in hwi_build_be_sgl_by_offset()
2991 				 pmem->bus_address.u.a64.address,  in hwi_build_be_sgl_by_offset()
2992 				 pmem->size, sgl);  in hwi_build_be_sgl_by_offset()
2998 	struct be_dma_mem *mem = &q->dma_mem;  in be_fill_queue()
3001 	q->len = len;  in be_fill_queue()
3002 	q->entry_size = entry_size;  in be_fill_queue()
3003 	mem->size = len * entry_size;  in be_fill_queue()
3004 	mem->va = vaddress;  in be_fill_queue()
3005 	if (!mem->va)  in be_fill_queue()
3006 		return -ENOMEM;  in be_fill_queue()
3007 	memset(mem->va, 0, mem->size);  in be_fill_queue()
3014 	int ret = -ENOMEM, eq_for_mcc;  in beiscsi_create_eqs()
3021 	num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries *  in beiscsi_create_eqs()
3024 	if (phba->pcidev->msix_enabled)  in beiscsi_create_eqs()
3028 	for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {  in beiscsi_create_eqs()
3029 		eq = &phwi_context->be_eq[i].q;  in beiscsi_create_eqs()
3030 		mem = &eq->dma_mem;  in beiscsi_create_eqs()
3031 		phwi_context->be_eq[i].phba = phba;  in beiscsi_create_eqs()
3032 		eq_vaddress = dma_alloc_coherent(&phba->pcidev->dev,  in beiscsi_create_eqs()
3036 			ret = -ENOMEM;  in beiscsi_create_eqs()
3040 		mem->va = eq_vaddress;  in beiscsi_create_eqs()
3041 		ret = be_fill_queue(eq, phba->params.num_eq_entries,  in beiscsi_create_eqs()
3049 		mem->dma = paddr;  in beiscsi_create_eqs()
3050 		ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,  in beiscsi_create_eqs()
3060 			    phwi_context->be_eq[i].q.id);  in beiscsi_create_eqs()
3065 	for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {  in beiscsi_create_eqs()
3066 		eq = &phwi_context->be_eq[i].q;  in beiscsi_create_eqs()
3067 		mem = &eq->dma_mem;  in beiscsi_create_eqs()
3068 		if (mem->va)  in beiscsi_create_eqs()
3069 			dma_free_coherent(&phba->pcidev->dev, num_eq_pages  in beiscsi_create_eqs()
3071 					    mem->va, mem->dma);  in beiscsi_create_eqs()
3084 	int ret = -ENOMEM;  in beiscsi_create_cqs()
3087 	num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries *  in beiscsi_create_cqs()
3090 	for (i = 0; i < phba->num_cpus; i++) {  in beiscsi_create_cqs()
3091 		cq = &phwi_context->be_cq[i];  in beiscsi_create_cqs()
3092 		eq = &phwi_context->be_eq[i].q;  in beiscsi_create_cqs()
3093 		pbe_eq = &phwi_context->be_eq[i];  in beiscsi_create_cqs()
3094 		pbe_eq->cq = cq;  in beiscsi_create_cqs()
3095 		pbe_eq->phba = phba;  in beiscsi_create_cqs()
3096 		mem = &cq->dma_mem;  in beiscsi_create_cqs()
3097 		cq_vaddress = dma_alloc_coherent(&phba->pcidev->dev,  in beiscsi_create_cqs()
3101 			ret = -ENOMEM;  in beiscsi_create_cqs()
3105 		ret = be_fill_queue(cq, phba->params.num_cq_entries,  in beiscsi_create_cqs()
3113 		mem->dma = paddr;  in beiscsi_create_cqs()
3114 		ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,  in beiscsi_create_cqs()
3123 			    "iSCSI CQ CREATED\n", cq->id, eq->id);  in beiscsi_create_cqs()
3128 	for (i = 0; i < phba->num_cpus; i++) {  in beiscsi_create_cqs()
3129 		cq = &phwi_context->be_cq[i];  in beiscsi_create_cqs()
3130 		mem = &cq->dma_mem;  in beiscsi_create_cqs()
3131 		if (mem->va)  in beiscsi_create_cqs()
3132 			dma_free_coherent(&phba->pcidev->dev, num_cq_pages  in beiscsi_create_cqs()
3134 					    mem->va, mem->dma);  in beiscsi_create_cqs()
3153 	dq = &phwi_context->be_def_hdrq[ulp_num];  in beiscsi_create_def_hdr()
3154 	cq = &phwi_context->be_cq[0];  in beiscsi_create_def_hdr()
3155 	mem = &dq->dma_mem;  in beiscsi_create_def_hdr()
3156 	mem_descr = phba->init_mem;  in beiscsi_create_def_hdr()
3159 	dq_vaddress = mem_descr->mem_array[idx].virtual_address;  in beiscsi_create_def_hdr()
3160 	ret = be_fill_queue(dq, mem_descr->mem_array[0].size /  in beiscsi_create_def_hdr()
3170 	mem->dma = (unsigned long)mem_descr->mem_array[idx].  in beiscsi_create_def_hdr()
3172 	ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,  in beiscsi_create_def_hdr()
3174 					      phba->params.defpdu_hdr_sz,  in beiscsi_create_def_hdr()
3187 		    phwi_context->be_def_hdrq[ulp_num].id);  in beiscsi_create_def_hdr()
3205 	dataq = &phwi_context->be_def_dataq[ulp_num];  in beiscsi_create_def_data()
3206 	cq = &phwi_context->be_cq[0];  in beiscsi_create_def_data()
3207 	mem = &dataq->dma_mem;  in beiscsi_create_def_data()
3208 	mem_descr = phba->init_mem;  in beiscsi_create_def_data()
3211 	dq_vaddress = mem_descr->mem_array[idx].virtual_address;  in beiscsi_create_def_data()
3212 	ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /  in beiscsi_create_def_data()
3223 	mem->dma = (unsigned long)mem_descr->mem_array[idx].  in beiscsi_create_def_data()
3225 	ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,  in beiscsi_create_def_data()
3227 					      phba->params.defpdu_data_sz,  in beiscsi_create_def_data()
3240 		    phwi_context->be_def_dataq[ulp_num].id);  in beiscsi_create_def_data()
3258 		if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {  in beiscsi_post_template_hdr()
3259 			mem_descr = (struct be_mem_descriptor *)phba->init_mem;  in beiscsi_post_template_hdr()
3262 			pm_arr = mem_descr->mem_array;  in beiscsi_post_template_hdr()
3266 				 &phba->ctrl, &sgl);  in beiscsi_post_template_hdr()
3292 	mem_descr = phba->init_mem;  in beiscsi_post_pages()
3294 	pm_arr = mem_descr->mem_array;  in beiscsi_post_pages()
3297 		if (test_bit(ulp_num, &phba->fw_config.ulp_supported))  in beiscsi_post_pages()
3300 	page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *  in beiscsi_post_pages()
3301 			phba->fw_config.iscsi_icd_start[ulp_num]) / PAGE_SIZE;  in beiscsi_post_pages()
3302 	for (i = 0; i < mem_descr->num_elements; i++) {  in beiscsi_post_pages()
3304 		status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,  in beiscsi_post_pages()
3306 						(pm_arr->size / PAGE_SIZE));  in beiscsi_post_pages()
3307 		page_offset += pm_arr->size / PAGE_SIZE;  in beiscsi_post_pages()
3322 	struct be_dma_mem *mem = &q->dma_mem;  in be_queue_free()
3323 	if (mem->va) {  in be_queue_free()
3324 		dma_free_coherent(&phba->pcidev->dev, mem->size,  in be_queue_free()
3325 			mem->va, mem->dma);  in be_queue_free()
3326 		mem->va = NULL;  in be_queue_free()
3333 	struct be_dma_mem *mem = &q->dma_mem;  in be_queue_alloc()
3336 	q->len = len;  in be_queue_alloc()
3337 	q->entry_size = entry_size;  in be_queue_alloc()
3338 	mem->size = len * entry_size;  in be_queue_alloc()
3339 	mem->va = dma_alloc_coherent(&phba->pcidev->dev, mem->size, &mem->dma,  in be_queue_alloc()
3341 	if (!mem->va)  in be_queue_alloc()
3342 		return -ENOMEM;  in be_queue_alloc()
3364 	mem_descr = phba->init_mem;  in beiscsi_create_wrb_rings()
3366 	pwrb_arr = kmalloc_array(phba->params.cxns_per_ctrl,  in beiscsi_create_wrb_rings()
3372 		return -ENOMEM;  in beiscsi_create_wrb_rings()
3374 	wrb_vaddr = mem_descr->mem_array[idx].virtual_address;  in beiscsi_create_wrb_rings()
3375 	pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;  in beiscsi_create_wrb_rings()
3376 	num_wrb_rings = mem_descr->mem_array[idx].size /  in beiscsi_create_wrb_rings()
3377 		(phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));  in beiscsi_create_wrb_rings()
3379 	for (num = 0; num < phba->params.cxns_per_ctrl; num++) {  in beiscsi_create_wrb_rings()
3383 			pwrb_arr[num].size = phba->params.wrbs_per_cxn *  in beiscsi_create_wrb_rings()
3387 			num_wrb_rings--;  in beiscsi_create_wrb_rings()
3390 			wrb_vaddr = mem_descr->mem_array[idx].virtual_address;  in beiscsi_create_wrb_rings()
3391 			pa_addr_lo = mem_descr->mem_array[idx].  in beiscsi_create_wrb_rings()
3393 			num_wrb_rings = mem_descr->mem_array[idx].size /  in beiscsi_create_wrb_rings()
3394 					(phba->params.wrbs_per_cxn *  in beiscsi_create_wrb_rings()
3398 			pwrb_arr[num].size = phba->params.wrbs_per_cxn *  in beiscsi_create_wrb_rings()
3402 			num_wrb_rings--;  in beiscsi_create_wrb_rings()
3408 		if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {  in beiscsi_create_wrb_rings()
3415 	for (i = 0; i < phba->params.cxns_per_ctrl; i++) {  in beiscsi_create_wrb_rings()
3423 			cid_count_ulp[ulp_base_num]--;  in beiscsi_create_wrb_rings()
3428 		status = be_cmd_wrbq_create(&phba->ctrl, &sgl,  in beiscsi_create_wrb_rings()
3429 					    &phwi_context->be_wrbq[i],  in beiscsi_create_wrb_rings()
3430 					    &phwi_ctrlr->wrb_context[i],  in beiscsi_create_wrb_rings()
3438 		pwrb_context = &phwi_ctrlr->wrb_context[i];  in beiscsi_create_wrb_rings()
3439 		BE_SET_CID_TO_CRI(i, pwrb_context->cid);  in beiscsi_create_wrb_rings()
3451 	phwi_ctrlr = phba->phwi_ctrlr;  in free_wrb_handles()
3452 	for (index = 0; index < phba->params.cxns_per_ctrl; index++) {  in free_wrb_handles()
3453 		pwrb_context = &phwi_ctrlr->wrb_context[index];  in free_wrb_handles()
3454 		kfree(pwrb_context->pwrb_handle_base);  in free_wrb_handles()
3455 		kfree(pwrb_context->pwrb_handle_basestd);  in free_wrb_handles()
3461 	struct be_ctrl_info *ctrl = &phba->ctrl;  in be_mcc_queues_destroy()
3466 	q = &phba->ctrl.mcc_obj.q;  in be_mcc_queues_destroy()
3470 			      &ctrl->ptag_state[tag].tag_state))  in be_mcc_queues_destroy()
3474 			     &ctrl->ptag_state[tag].tag_state)) {  in be_mcc_queues_destroy()
3475 			ptag_mem = &ctrl->ptag_state[tag].tag_mem_state;  in be_mcc_queues_destroy()
3476 			if (ptag_mem->size) {  in be_mcc_queues_destroy()
3477 				dma_free_coherent(&ctrl->pdev->dev,  in be_mcc_queues_destroy()
3478 						    ptag_mem->size,  in be_mcc_queues_destroy()
3479 						    ptag_mem->va,  in be_mcc_queues_destroy()
3480 						    ptag_mem->dma);  in be_mcc_queues_destroy()
3481 				ptag_mem->size = 0;  in be_mcc_queues_destroy()
3491 		if (waitqueue_active(&ctrl->mcc_wait[tag])) {  in be_mcc_queues_destroy()
3492 			ctrl->mcc_tag_status[tag] = MCC_STATUS_FAILED;  in be_mcc_queues_destroy()
3493 			ctrl->mcc_tag_status[tag] |= CQE_VALID_MASK;  in be_mcc_queues_destroy()
3494 			wake_up_interruptible(&ctrl->mcc_wait[tag]);  in be_mcc_queues_destroy()
3500 					&ctrl->ptag_state[tag].tag_state))  in be_mcc_queues_destroy()
3508 	if (q->created) {  in be_mcc_queues_destroy()
3513 	q = &phba->ctrl.mcc_obj.cq;  in be_mcc_queues_destroy()
3514 	if (q->created) {  in be_mcc_queues_destroy()
3524 	struct be_ctrl_info *ctrl = &phba->ctrl;  in be_mcc_queues_create()
3527 	cq = &phba->ctrl.mcc_obj.cq;  in be_mcc_queues_create()
3532 	if (phba->pcidev->msix_enabled) {  in be_mcc_queues_create()
3534 					&phwi_context->be_eq[phba->num_cpus].q,  in be_mcc_queues_create()
3538 		if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,  in be_mcc_queues_create()
3544 	q = &phba->ctrl.mcc_obj.q;  in be_mcc_queues_create()
3561 	return -ENOMEM;  in be_mcc_queues_create()
3568 	switch (phba->generation) {  in be2iscsi_enable_msix()
3574 		nvec = phba->fw_config.eqid_count;  in be2iscsi_enable_msix()
3585 		if (pci_alloc_irq_vectors_affinity(phba->pcidev, 2, nvec,  in be2iscsi_enable_msix()
3587 			phba->num_cpus = nvec - 1;  in be2iscsi_enable_msix()
3592 	phba->num_cpus = 1;  in be2iscsi_enable_msix()
3607 	phwi_ctrlr = phba->phwi_ctrlr;  in hwi_purge_eq()
3608 	phwi_context = phwi_ctrlr->phwi_ctxt;  in hwi_purge_eq()
3609 	if (phba->pcidev->msix_enabled)  in hwi_purge_eq()
3614 	for (i = 0; i < (phba->num_cpus + eq_msix); i++) {  in hwi_purge_eq()
3615 		eq = &phwi_context->be_eq[i].q;  in hwi_purge_eq()
3618 		while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]  in hwi_purge_eq()
3627 			hwi_ring_eq_db(phba, eq->id, 1,	num_processed, 1, 1);  in hwi_purge_eq()
3634 	struct be_ctrl_info *ctrl = &phba->ctrl;  in hwi_cleanup_port()
3640 		if (test_bit(ulp_num, &phba->fw_config.ulp_supported))  in hwi_cleanup_port()
3650 	phwi_ctrlr = phba->phwi_ctrlr;  in hwi_cleanup_port()
3651 	phwi_context = phwi_ctrlr->phwi_ctxt;  in hwi_cleanup_port()
3655 	for (i = 0; i < phba->params.cxns_per_ctrl; i++) {  in hwi_cleanup_port()
3656 		q = &phwi_context->be_wrbq[i];  in hwi_cleanup_port()
3657 		if (q->created)  in hwi_cleanup_port()
3660 	kfree(phwi_context->be_wrbq);  in hwi_cleanup_port()
3664 		if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {  in hwi_cleanup_port()
3666 			q = &phwi_context->be_def_hdrq[ulp_num];  in hwi_cleanup_port()
3667 			if (q->created)  in hwi_cleanup_port()
3670 			q = &phwi_context->be_def_dataq[ulp_num];  in hwi_cleanup_port()
3671 			if (q->created)  in hwi_cleanup_port()
3678 	for (i = 0; i < (phba->num_cpus); i++) {  in hwi_cleanup_port()
3679 		q = &phwi_context->be_cq[i];  in hwi_cleanup_port()
3680 		if (q->created) {  in hwi_cleanup_port()
3687 	if (phba->pcidev->msix_enabled)  in hwi_cleanup_port()
3691 	for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {  in hwi_cleanup_port()
3692 		q = &phwi_context->be_eq[i].q;  in hwi_cleanup_port()
3693 		if (q->created) {  in hwi_cleanup_port()
3701 	beiscsi_cmd_special_wrb(&phba->ctrl, 0);  in hwi_cleanup_port()
3709 	struct be_ctrl_info *ctrl = &phba->ctrl;  in hwi_init_port()
3713 	phwi_ctrlr = phba->phwi_ctrlr;  in hwi_init_port()
3714 	phwi_context = phwi_ctrlr->phwi_ctxt;  in hwi_init_port()
3716 	phba->optic_state = 0xff;  in hwi_init_port()
3744 		if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {  in hwi_init_port()
3745 			nbufs = phwi_context->pasync_ctx[ulp_num]->num_entries;  in hwi_init_port()
3803 		if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {  in hwi_init_port()
3810 			     phba->params.cxns_per_ctrl; cri++) {  in hwi_init_port()
3813 					pasync_ctx->cid_to_async_cri_map[  in hwi_init_port()
3814 					phwi_ctrlr->wrb_context[cri].cid] =  in hwi_init_port()
3835 	phwi_ctrlr = phba->phwi_ctrlr;  in hwi_init_controller()
3836 	if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {  in hwi_init_controller()
3837 		phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->  in hwi_init_controller()
3840 			    "BM_%d :  phwi_ctrlr->phwi_ctxt=%p\n",  in hwi_init_controller()
3841 			    phwi_ctrlr->phwi_ctxt);  in hwi_init_controller()
3846 		return -ENOMEM;  in hwi_init_controller()
3851 		return -ENOMEM;  in hwi_init_controller()
3856 		return -ENOMEM;  in hwi_init_controller()
3863 		return -ENOMEM;  in hwi_init_controller()
3873 	mem_descr = phba->init_mem;  in beiscsi_free_mem()
3875 		for (j = mem_descr->num_elements; j > 0; j--) {  in beiscsi_free_mem()
3876 			dma_free_coherent(&phba->pcidev->dev,  in beiscsi_free_mem()
3877 			  mem_descr->mem_array[j - 1].size,  in beiscsi_free_mem()
3878 			  mem_descr->mem_array[j - 1].virtual_address,  in beiscsi_free_mem()
3879 			  (unsigned long)mem_descr->mem_array[j - 1].  in beiscsi_free_mem()
3883 		kfree(mem_descr->mem_array);  in beiscsi_free_mem()
3886 	kfree(phba->init_mem);  in beiscsi_free_mem()
3887 	kfree(phba->phwi_ctrlr->wrb_context);  in beiscsi_free_mem()
3888 	kfree(phba->phwi_ctrlr);  in beiscsi_free_mem()
3899 	phba->io_sgl_hndl_avbl = 0;  in beiscsi_init_sgl_handle()
3900 	phba->eh_sgl_hndl_avbl = 0;  in beiscsi_init_sgl_handle()
3902 	mem_descr_sglh = phba->init_mem;  in beiscsi_init_sgl_handle()
3904 	if (1 == mem_descr_sglh->num_elements) {  in beiscsi_init_sgl_handle()
3905 		phba->io_sgl_hndl_base = kcalloc(phba->params.ios_per_ctrl,  in beiscsi_init_sgl_handle()
3908 		if (!phba->io_sgl_hndl_base) {  in beiscsi_init_sgl_handle()
3911 			return -ENOMEM;  in beiscsi_init_sgl_handle()
3913 		phba->eh_sgl_hndl_base =  in beiscsi_init_sgl_handle()
3914 			kcalloc(phba->params.icds_per_ctrl -  in beiscsi_init_sgl_handle()
3915 					phba->params.ios_per_ctrl,  in beiscsi_init_sgl_handle()
3917 		if (!phba->eh_sgl_hndl_base) {  in beiscsi_init_sgl_handle()
3918 			kfree(phba->io_sgl_hndl_base);  in beiscsi_init_sgl_handle()
3921 			return -ENOMEM;  in beiscsi_init_sgl_handle()
3927 		return -ENOMEM;  in beiscsi_init_sgl_handle()
3932 	while (idx < mem_descr_sglh->num_elements) {  in beiscsi_init_sgl_handle()
3933 		psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;  in beiscsi_init_sgl_handle()
3935 		for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /  in beiscsi_init_sgl_handle()
3937 			if (arr_index < phba->params.ios_per_ctrl) {  in beiscsi_init_sgl_handle()
3938 				phba->io_sgl_hndl_base[arr_index] = psgl_handle;  in beiscsi_init_sgl_handle()
3939 				phba->io_sgl_hndl_avbl++;  in beiscsi_init_sgl_handle()
3942 				phba->eh_sgl_hndl_base[arr_index -  in beiscsi_init_sgl_handle()
3943 					phba->params.ios_per_ctrl] =  in beiscsi_init_sgl_handle()
3946 				phba->eh_sgl_hndl_avbl++;  in beiscsi_init_sgl_handle()
3953 		    "BM_%d : phba->io_sgl_hndl_avbl=%d "  in beiscsi_init_sgl_handle()
3954 		    "phba->eh_sgl_hndl_avbl=%d\n",  in beiscsi_init_sgl_handle()
3955 		    phba->io_sgl_hndl_avbl,  in beiscsi_init_sgl_handle()
3956 		    phba->eh_sgl_hndl_avbl);  in beiscsi_init_sgl_handle()
3958 	mem_descr_sg = phba->init_mem;  in beiscsi_init_sgl_handle()
3961 		    "\n BM_%d : mem_descr_sg->num_elements=%d\n",  in beiscsi_init_sgl_handle()
3962 		    mem_descr_sg->num_elements);  in beiscsi_init_sgl_handle()
3965 		if (test_bit(ulp_num, &phba->fw_config.ulp_supported))  in beiscsi_init_sgl_handle()
3968 	ulp_icd_start = phba->fw_config.iscsi_icd_start[ulp_num];  in beiscsi_init_sgl_handle()
3972 	while (idx < mem_descr_sg->num_elements) {  in beiscsi_init_sgl_handle()
3973 		pfrag = mem_descr_sg->mem_array[idx].virtual_address;  in beiscsi_init_sgl_handle()
3976 		     i < (mem_descr_sg->mem_array[idx].size) /  in beiscsi_init_sgl_handle()
3977 		     (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);  in beiscsi_init_sgl_handle()
3979 			if (arr_index < phba->params.ios_per_ctrl)  in beiscsi_init_sgl_handle()
3980 				psgl_handle = phba->io_sgl_hndl_base[arr_index];  in beiscsi_init_sgl_handle()
3982 				psgl_handle = phba->eh_sgl_hndl_base[arr_index -  in beiscsi_init_sgl_handle()
3983 						phba->params.ios_per_ctrl];  in beiscsi_init_sgl_handle()
3984 			psgl_handle->pfrag = pfrag;  in beiscsi_init_sgl_handle()
3987 			pfrag += phba->params.num_sge_per_io;  in beiscsi_init_sgl_handle()
3988 			psgl_handle->sgl_index = ulp_icd_start + arr_index++;  in beiscsi_init_sgl_handle()
3992 	phba->io_sgl_free_index = 0;  in beiscsi_init_sgl_handle()
3993 	phba->io_sgl_alloc_index = 0;  in beiscsi_init_sgl_handle()
3994 	phba->eh_sgl_free_index = 0;  in beiscsi_init_sgl_handle()
3995 	phba->eh_sgl_alloc_index = 0;  in beiscsi_init_sgl_handle()
4006 		if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {  in hba_setup_cid_tbls()
4011 				ret = -ENOMEM;  in hba_setup_cid_tbls()
4016 			ptr_cid_info->cid_array =  in hba_setup_cid_tbls()
4018 					sizeof(*ptr_cid_info->cid_array),  in hba_setup_cid_tbls()
4020 			if (!ptr_cid_info->cid_array) {  in hba_setup_cid_tbls()
4023 				ret = -ENOMEM;  in hba_setup_cid_tbls()
4027 			ptr_cid_info->avlbl_cids = BEISCSI_GET_CID_COUNT(  in hba_setup_cid_tbls()
4031 			phba->cid_array_info[ulp_num] = ptr_cid_info;  in hba_setup_cid_tbls()
4034 	phba->ep_array = kcalloc(phba->params.cxns_per_ctrl,  in hba_setup_cid_tbls()
4037 	if (!phba->ep_array) {  in hba_setup_cid_tbls()
4038 		ret = -ENOMEM;  in hba_setup_cid_tbls()
4043 	phba->conn_table = kcalloc(phba->params.cxns_per_ctrl,  in hba_setup_cid_tbls()
4046 	if (!phba->conn_table) {  in hba_setup_cid_tbls()
4047 		kfree(phba->ep_array);  in hba_setup_cid_tbls()
4048 		phba->ep_array = NULL;  in hba_setup_cid_tbls()
4049 		ret = -ENOMEM;  in hba_setup_cid_tbls()
4054 	for (i = 0; i < phba->params.cxns_per_ctrl; i++) {  in hba_setup_cid_tbls()
4055 		ulp_num = phba->phwi_ctrlr->wrb_context[i].ulp_num;  in hba_setup_cid_tbls()
4057 		ptr_cid_info = phba->cid_array_info[ulp_num];  in hba_setup_cid_tbls()
4058 		ptr_cid_info->cid_array[ptr_cid_info->cid_alloc++] =  in hba_setup_cid_tbls()
4059 			phba->phwi_ctrlr->wrb_context[i].cid;  in hba_setup_cid_tbls()
4064 		if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {  in hba_setup_cid_tbls()
4065 			ptr_cid_info = phba->cid_array_info[ulp_num];  in hba_setup_cid_tbls()
4067 			ptr_cid_info->cid_alloc = 0;  in hba_setup_cid_tbls()
4068 			ptr_cid_info->cid_free = 0;  in hba_setup_cid_tbls()
4075 		if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {  in hba_setup_cid_tbls()
4076 			ptr_cid_info = phba->cid_array_info[ulp_num];  in hba_setup_cid_tbls()
4079 				kfree(ptr_cid_info->cid_array);  in hba_setup_cid_tbls()
4081 				phba->cid_array_info[ulp_num] = NULL;  in hba_setup_cid_tbls()
4091 	struct be_ctrl_info *ctrl = &phba->ctrl;  in hwi_enable_intr()
4099 	phwi_ctrlr = phba->phwi_ctrlr;  in hwi_enable_intr()
4100 	phwi_context = phwi_ctrlr->phwi_ctxt;  in hwi_enable_intr()
4102 	addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +  in hwi_enable_intr()
4114 	if (!phba->pcidev->msix_enabled) {  in hwi_enable_intr()
4115 		eq = &phwi_context->be_eq[0].q;  in hwi_enable_intr()
4117 			    "BM_%d : eq->id=%d\n", eq->id);  in hwi_enable_intr()
4119 		hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);  in hwi_enable_intr()
4121 		for (i = 0; i <= phba->num_cpus; i++) {  in hwi_enable_intr()
4122 			eq = &phwi_context->be_eq[i].q;  in hwi_enable_intr()
4124 				    "BM_%d : eq->id=%d\n", eq->id);  in hwi_enable_intr()
4125 			hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);  in hwi_enable_intr()
4132 	struct be_ctrl_info *ctrl = &phba->ctrl;  in hwi_disable_intr()
4134 	u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;  in hwi_disable_intr()
4167 		kfree(phba->io_sgl_hndl_base);  in beiscsi_init_port()
4168 		kfree(phba->eh_sgl_hndl_base);  in beiscsi_init_port()
4183 	kfree(phba->io_sgl_hndl_base);  in beiscsi_cleanup_port()
4184 	kfree(phba->eh_sgl_hndl_base);  in beiscsi_cleanup_port()
4185 	kfree(phba->ep_array);  in beiscsi_cleanup_port()
4186 	kfree(phba->conn_table);  in beiscsi_cleanup_port()
4189 		if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {  in beiscsi_cleanup_port()
4190 			ptr_cid_info = phba->cid_array_info[ulp_num];  in beiscsi_cleanup_port()
4193 				kfree(ptr_cid_info->cid_array);  in beiscsi_cleanup_port()
4195 				phba->cid_array_info[ulp_num] = NULL;  in beiscsi_cleanup_port()
4202  * beiscsi_free_mgmt_task_handles()- Free driver CXN resources
4213 	struct beiscsi_hba *phba = beiscsi_conn->phba;  in beiscsi_free_mgmt_task_handles()
4217 				beiscsi_conn->beiscsi_conn_cid);  in beiscsi_free_mgmt_task_handles()
4219 	phwi_ctrlr = phba->phwi_ctrlr;  in beiscsi_free_mgmt_task_handles()
4220 	pwrb_context = &phwi_ctrlr->wrb_context[cri_index];  in beiscsi_free_mgmt_task_handles()
4222 	io_task = task->dd_data;  in beiscsi_free_mgmt_task_handles()
4224 	if (io_task->pwrb_handle) {  in beiscsi_free_mgmt_task_handles()
4225 		free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);  in beiscsi_free_mgmt_task_handles()
4226 		io_task->pwrb_handle = NULL;  in beiscsi_free_mgmt_task_handles()
4229 	if (io_task->psgl_handle) {  in beiscsi_free_mgmt_task_handles()
4230 		free_mgmt_sgl_handle(phba, io_task->psgl_handle);  in beiscsi_free_mgmt_task_handles()
4231 		io_task->psgl_handle = NULL;  in beiscsi_free_mgmt_task_handles()
4234 	if (io_task->mtask_addr) {  in beiscsi_free_mgmt_task_handles()
4235 		dma_unmap_single(&phba->pcidev->dev,  in beiscsi_free_mgmt_task_handles()
4236 				 io_task->mtask_addr,  in beiscsi_free_mgmt_task_handles()
4237 				 io_task->mtask_data_count,  in beiscsi_free_mgmt_task_handles()
4239 		io_task->mtask_addr = 0;  in beiscsi_free_mgmt_task_handles()
4244  * beiscsi_cleanup_task()- Free driver resources of the task
4250 	struct beiscsi_io_task *io_task = task->dd_data;  in beiscsi_cleanup_task()
4251 	struct iscsi_conn *conn = task->conn;  in beiscsi_cleanup_task()
4252 	struct beiscsi_conn *beiscsi_conn = conn->dd_data;  in beiscsi_cleanup_task()
4253 	struct beiscsi_hba *phba = beiscsi_conn->phba;  in beiscsi_cleanup_task()
4254 	struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;  in beiscsi_cleanup_task()
4258 			     beiscsi_conn->beiscsi_conn_cid);  in beiscsi_cleanup_task()
4260 	phwi_ctrlr = phba->phwi_ctrlr;  in beiscsi_cleanup_task()
4261 	pwrb_context = &phwi_ctrlr->wrb_context[cri_index];  in beiscsi_cleanup_task()
4263 	if (io_task->cmd_bhs) {  in beiscsi_cleanup_task()
4264 		dma_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,  in beiscsi_cleanup_task()
4265 			      io_task->bhs_pa.u.a64.address);  in beiscsi_cleanup_task()
4266 		io_task->cmd_bhs = NULL;  in beiscsi_cleanup_task()
4267 		task->hdr = NULL;  in beiscsi_cleanup_task()
4270 	if (task->sc) {  in beiscsi_cleanup_task()
4271 		if (io_task->pwrb_handle) {  in beiscsi_cleanup_task()
4273 					io_task->pwrb_handle);  in beiscsi_cleanup_task()
4274 			io_task->pwrb_handle = NULL;  in beiscsi_cleanup_task()
4277 		if (io_task->psgl_handle) {  in beiscsi_cleanup_task()
4278 			free_io_sgl_handle(phba, io_task->psgl_handle);  in beiscsi_cleanup_task()
4279 			io_task->psgl_handle = NULL;  in beiscsi_cleanup_task()
4282 		if (io_task->scsi_cmnd) {  in beiscsi_cleanup_task()
4283 			if (io_task->num_sg)  in beiscsi_cleanup_task()
4284 				scsi_dma_unmap(io_task->scsi_cmnd);  in beiscsi_cleanup_task()
4285 			io_task->scsi_cmnd = NULL;  in beiscsi_cleanup_task()
4288 		if (!beiscsi_conn->login_in_progress)  in beiscsi_cleanup_task()
4299 	struct beiscsi_hba *phba = beiscsi_conn->phba;  in beiscsi_offload_connection()
4300 	struct iscsi_task *task = beiscsi_conn->task;  in beiscsi_offload_connection()
4301 	struct iscsi_session *session = task->conn->session;  in beiscsi_offload_connection()
4305 	 * We can always use 0 here because it is reserved by libiscsi for  in beiscsi_offload_connection()
4308 	beiscsi_conn->login_in_progress = 0;  in beiscsi_offload_connection()
4309 	spin_lock_bh(&session->back_lock);  in beiscsi_offload_connection()
4311 	spin_unlock_bh(&session->back_lock);  in beiscsi_offload_connection()
4313 	pwrb_handle = alloc_wrb_handle(phba, beiscsi_conn->beiscsi_conn_cid,  in beiscsi_offload_connection()
4319 				       phba->init_mem,  in beiscsi_offload_connection()
4325 	be_dws_le_to_cpu(pwrb_handle->pwrb,  in beiscsi_offload_connection()
4328 	doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;  in beiscsi_offload_connection()
4329 	doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)  in beiscsi_offload_connection()
4332 	iowrite32(doorbell, phba->db_va +  in beiscsi_offload_connection()
4333 		  beiscsi_conn->doorbell_offset);  in beiscsi_offload_connection()
4342 			       phba->params.wrbs_per_cxn);  in beiscsi_offload_connection()
4346 		    pwrb_handle, pwrb_context->free_index,  in beiscsi_offload_connection()
4347 		    pwrb_context->wrb_handles_available);  in beiscsi_offload_connection()
4355 		*age = conn->session->age;  in beiscsi_parse_pdu()
4359  * beiscsi_alloc_pdu - allocates pdu and related resources
4360  * @task: libiscsi task
4366  * the pdu itt to the libiscsi task itt.
4370 	struct beiscsi_io_task *io_task = task->dd_data;  in beiscsi_alloc_pdu()
4371 	struct iscsi_conn *conn = task->conn;  in beiscsi_alloc_pdu()
4372 	struct beiscsi_conn *beiscsi_conn = conn->dd_data;  in beiscsi_alloc_pdu()
4373 	struct beiscsi_hba *phba = beiscsi_conn->phba;  in beiscsi_alloc_pdu()
4378 	struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;  in beiscsi_alloc_pdu()
4381 	io_task->cmd_bhs = dma_pool_alloc(beiscsi_sess->bhs_pool,  in beiscsi_alloc_pdu()
4383 	if (!io_task->cmd_bhs)  in beiscsi_alloc_pdu()
4384 		return -ENOMEM;  in beiscsi_alloc_pdu()
4385 	io_task->bhs_pa.u.a64.address = paddr;  in beiscsi_alloc_pdu()
4386 	io_task->libiscsi_itt = (itt_t)task->itt;  in beiscsi_alloc_pdu()
4387 	io_task->conn = beiscsi_conn;  in beiscsi_alloc_pdu()
4389 	task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;  in beiscsi_alloc_pdu()
4390 	task->hdr_max = sizeof(struct be_cmd_bhs);  in beiscsi_alloc_pdu()
4391 	io_task->psgl_handle = NULL;  in beiscsi_alloc_pdu()
4392 	io_task->pwrb_handle = NULL;  in beiscsi_alloc_pdu()
4394 	if (task->sc) {  in beiscsi_alloc_pdu()
4395 		io_task->psgl_handle = alloc_io_sgl_handle(phba);  in beiscsi_alloc_pdu()
4396 		if (!io_task->psgl_handle) {  in beiscsi_alloc_pdu()
4401 				    beiscsi_conn->beiscsi_conn_cid);  in beiscsi_alloc_pdu()
4404 		io_task->pwrb_handle = alloc_wrb_handle(phba,  in beiscsi_alloc_pdu()
4405 					beiscsi_conn->beiscsi_conn_cid,  in beiscsi_alloc_pdu()
4406 					&io_task->pwrb_context);  in beiscsi_alloc_pdu()
4407 		if (!io_task->pwrb_handle) {  in beiscsi_alloc_pdu()
4412 				    beiscsi_conn->beiscsi_conn_cid);  in beiscsi_alloc_pdu()
4416 		io_task->scsi_cmnd = NULL;  in beiscsi_alloc_pdu()
4418 			beiscsi_conn->task = task;  in beiscsi_alloc_pdu()
4419 			if (!beiscsi_conn->login_in_progress) {  in beiscsi_alloc_pdu()
4420 				io_task->psgl_handle = (struct sgl_handle *)  in beiscsi_alloc_pdu()
4422 				if (!io_task->psgl_handle) {  in beiscsi_alloc_pdu()
4428 						    beiscsi_conn->beiscsi_conn_cid);  in beiscsi_alloc_pdu()
4432 				beiscsi_conn->login_in_progress = 1;  in beiscsi_alloc_pdu()
4433 				beiscsi_conn->plogin_sgl_handle =  in beiscsi_alloc_pdu()
4434 							io_task->psgl_handle;  in beiscsi_alloc_pdu()
4435 				io_task->pwrb_handle =  in beiscsi_alloc_pdu()
4437 					beiscsi_conn->beiscsi_conn_cid,  in beiscsi_alloc_pdu()
4438 					&io_task->pwrb_context);  in beiscsi_alloc_pdu()
4439 				if (!io_task->pwrb_handle) {  in beiscsi_alloc_pdu()
4445 						    beiscsi_conn->beiscsi_conn_cid);  in beiscsi_alloc_pdu()
4448 				beiscsi_conn->plogin_wrb_handle =  in beiscsi_alloc_pdu()
4449 							io_task->pwrb_handle;  in beiscsi_alloc_pdu()
4452 				io_task->psgl_handle =  in beiscsi_alloc_pdu()
4453 						beiscsi_conn->plogin_sgl_handle;  in beiscsi_alloc_pdu()
4454 				io_task->pwrb_handle =  in beiscsi_alloc_pdu()
4455 						beiscsi_conn->plogin_wrb_handle;  in beiscsi_alloc_pdu()
4458 			io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);  in beiscsi_alloc_pdu()
4459 			if (!io_task->psgl_handle) {  in beiscsi_alloc_pdu()
4465 					    beiscsi_conn->beiscsi_conn_cid);  in beiscsi_alloc_pdu()
4468 			io_task->pwrb_handle =  in beiscsi_alloc_pdu()
4470 					beiscsi_conn->beiscsi_conn_cid,  in beiscsi_alloc_pdu()
4471 					&io_task->pwrb_context);  in beiscsi_alloc_pdu()
4472 			if (!io_task->pwrb_handle) {  in beiscsi_alloc_pdu()
4477 					    beiscsi_conn->beiscsi_conn_cid);  in beiscsi_alloc_pdu()
4483 	itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->  in beiscsi_alloc_pdu()
4485 				(io_task->psgl_handle->sgl_index));  in beiscsi_alloc_pdu()
4486 	io_task->pwrb_handle->pio_handle = task;  in beiscsi_alloc_pdu()
4488 	io_task->cmd_bhs->iscsi_hdr.itt = itt;  in beiscsi_alloc_pdu()
4492 	free_io_sgl_handle(phba, io_task->psgl_handle);  in beiscsi_alloc_pdu()
4495 	free_mgmt_sgl_handle(phba, io_task->psgl_handle);  in beiscsi_alloc_pdu()
4496 	io_task->psgl_handle = NULL;  in beiscsi_alloc_pdu()
4498 	phwi_ctrlr = phba->phwi_ctrlr;  in beiscsi_alloc_pdu()
4500 	beiscsi_conn->beiscsi_conn_cid);  in beiscsi_alloc_pdu()
4501 	pwrb_context = &phwi_ctrlr->wrb_context[cri_index];  in beiscsi_alloc_pdu()
4502 	if (io_task->pwrb_handle)  in beiscsi_alloc_pdu()
4503 		free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);  in beiscsi_alloc_pdu()
4504 	io_task->pwrb_handle = NULL;  in beiscsi_alloc_pdu()
4505 	dma_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,  in beiscsi_alloc_pdu()
4506 		      io_task->bhs_pa.u.a64.address);  in beiscsi_alloc_pdu()
4507 	io_task->cmd_bhs = NULL;  in beiscsi_alloc_pdu()
4508 	return -ENOMEM;  in beiscsi_alloc_pdu()
4515 	struct beiscsi_io_task *io_task = task->dd_data;  in beiscsi_iotask_v2()
4516 	struct iscsi_conn *conn = task->conn;  in beiscsi_iotask_v2()
4517 	struct beiscsi_conn *beiscsi_conn = conn->dd_data;  in beiscsi_iotask_v2()
4518 	struct beiscsi_hba *phba = beiscsi_conn->phba;  in beiscsi_iotask_v2()
4522 	pwrb = io_task->pwrb_handle->pwrb;  in beiscsi_iotask_v2()
4524 	io_task->bhs_len = sizeof(struct be_cmd_bhs);  in beiscsi_iotask_v2()
4536 	io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb_v2,  in beiscsi_iotask_v2()
4541 		      &io_task->cmd_bhs->iscsi_hdr.lun));  in beiscsi_iotask_v2()
4544 		      io_task->pwrb_handle->wrb_index);  in beiscsi_iotask_v2()
4546 		      be32_to_cpu(task->cmdsn));  in beiscsi_iotask_v2()
4548 		      io_task->psgl_handle->sgl_index);  in beiscsi_iotask_v2()
4552 		      io_task->pwrb_handle->wrb_index);  in beiscsi_iotask_v2()
4553 	if (io_task->pwrb_context->plast_wrb)  in beiscsi_iotask_v2()
4555 			      io_task->pwrb_context->plast_wrb,  in beiscsi_iotask_v2()
4556 			      io_task->pwrb_handle->wrb_index);  in beiscsi_iotask_v2()
4557 	io_task->pwrb_context->plast_wrb = pwrb;  in beiscsi_iotask_v2()
4561 	doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;  in beiscsi_iotask_v2()
4562 	doorbell |= (io_task->pwrb_handle->wrb_index &  in beiscsi_iotask_v2()
4566 	iowrite32(doorbell, phba->db_va +  in beiscsi_iotask_v2()
4567 		  beiscsi_conn->doorbell_offset);  in beiscsi_iotask_v2()
4576 	struct beiscsi_io_task *io_task = task->dd_data;  in beiscsi_iotask()
4577 	struct iscsi_conn *conn = task->conn;  in beiscsi_iotask()
4578 	struct beiscsi_conn *beiscsi_conn = conn->dd_data;  in beiscsi_iotask()
4579 	struct beiscsi_hba *phba = beiscsi_conn->phba;  in beiscsi_iotask()
4583 	pwrb = io_task->pwrb_handle->pwrb;  in beiscsi_iotask()
4584 	io_task->bhs_len = sizeof(struct be_cmd_bhs);  in beiscsi_iotask()
4596 	io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb,  in beiscsi_iotask()
4601 				  &io_task->cmd_bhs->iscsi_hdr.lun));  in beiscsi_iotask()
4604 		      io_task->pwrb_handle->wrb_index);  in beiscsi_iotask()
4606 		      be32_to_cpu(task->cmdsn));  in beiscsi_iotask()
4608 		      io_task->psgl_handle->sgl_index);  in beiscsi_iotask()
4613 		      io_task->pwrb_handle->wrb_index);  in beiscsi_iotask()
4614 	if (io_task->pwrb_context->plast_wrb)  in beiscsi_iotask()
4616 			      io_task->pwrb_context->plast_wrb,  in beiscsi_iotask()
4617 			      io_task->pwrb_handle->wrb_index);  in beiscsi_iotask()
4618 	io_task->pwrb_context->plast_wrb = pwrb;  in beiscsi_iotask()
4622 	doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;  in beiscsi_iotask()
4623 	doorbell |= (io_task->pwrb_handle->wrb_index &  in beiscsi_iotask()
4627 	iowrite32(doorbell, phba->db_va +  in beiscsi_iotask()
4628 		  beiscsi_conn->doorbell_offset);  in beiscsi_iotask()
4634 	struct beiscsi_io_task *io_task = task->dd_data;  in beiscsi_mtask()
4635 	struct iscsi_conn *conn = task->conn;  in beiscsi_mtask()
4636 	struct beiscsi_conn *beiscsi_conn = conn->dd_data;  in beiscsi_mtask()
4637 	struct beiscsi_hba *phba = beiscsi_conn->phba;  in beiscsi_mtask()
4644 	cid = beiscsi_conn->beiscsi_conn_cid;  in beiscsi_mtask()
4645 	pwrb = io_task->pwrb_handle->pwrb;  in beiscsi_mtask()
4649 			      be32_to_cpu(task->cmdsn));  in beiscsi_mtask()
4651 			      io_task->pwrb_handle->wrb_index);  in beiscsi_mtask()
4653 			      io_task->psgl_handle->sgl_index);  in beiscsi_mtask()
4655 			      task->data_count);  in beiscsi_mtask()
4657 			      io_task->pwrb_handle->wrb_index);  in beiscsi_mtask()
4658 		if (io_task->pwrb_context->plast_wrb)  in beiscsi_mtask()
4660 				      io_task->pwrb_context->plast_wrb,  in beiscsi_mtask()
4661 				      io_task->pwrb_handle->wrb_index);  in beiscsi_mtask()
4662 		io_task->pwrb_context->plast_wrb = pwrb;  in beiscsi_mtask()
4667 			      be32_to_cpu(task->cmdsn));  in beiscsi_mtask()
4669 			      io_task->pwrb_handle->wrb_index);  in beiscsi_mtask()
4671 			      io_task->psgl_handle->sgl_index);  in beiscsi_mtask()
4673 			      task->data_count);  in beiscsi_mtask()
4675 			      io_task->pwrb_handle->wrb_index);  in beiscsi_mtask()
4676 		if (io_task->pwrb_context->plast_wrb)  in beiscsi_mtask()
4678 				      io_task->pwrb_context->plast_wrb,  in beiscsi_mtask()
4679 				      io_task->pwrb_handle->wrb_index);  in beiscsi_mtask()
4680 		io_task->pwrb_context->plast_wrb = pwrb;  in beiscsi_mtask()
4686 	switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {  in beiscsi_mtask()
4693 		if (task->hdr->ttt != ISCSI_RESERVED_TAG) {  in beiscsi_mtask()
4728 			    task->hdr->opcode & ISCSI_OPCODE_MASK);  in beiscsi_mtask()
4730 		return -EINVAL;  in beiscsi_mtask()
4737 	io_task->wrb_type = (is_chip_be2_be3r(phba)) ?  in beiscsi_mtask()
4742 	doorbell |= (io_task->pwrb_handle->wrb_index &  in beiscsi_mtask()
4745 	iowrite32(doorbell, phba->db_va +  in beiscsi_mtask()
4746 		  beiscsi_conn->doorbell_offset);  in beiscsi_mtask()
4752 	struct beiscsi_io_task *io_task = task->dd_data;  in beiscsi_task_xmit()
4753 	struct scsi_cmnd *sc = task->sc;  in beiscsi_task_xmit()
4759 	phba = io_task->conn->phba;  in beiscsi_task_xmit()
4766 		return -EIO;  in beiscsi_task_xmit()
4768 	if (!io_task->conn->login_in_progress)  in beiscsi_task_xmit()
4769 		task->hdr->exp_statsn = 0;  in beiscsi_task_xmit()
4774 	io_task->scsi_cmnd = sc;  in beiscsi_task_xmit()
4775 	io_task->num_sg = 0;  in beiscsi_task_xmit()
4782 			    be32_to_cpu(io_task->cmd_bhs->iscsi_hdr.itt),  in beiscsi_task_xmit()
4783 			    io_task->libiscsi_itt, scsi_bufflen(sc));  in beiscsi_task_xmit()
4791 	io_task->num_sg = num_sg;  in beiscsi_task_xmit()
4794 	if (sc->sc_data_direction == DMA_TO_DEVICE)  in beiscsi_task_xmit()
4799 	return phba->iotask_fn(task, sg, num_sg, xferlen, writedir);  in beiscsi_task_xmit()
4803  * beiscsi_bsg_request - handle bsg request from ISCSI transport
4810 	struct iscsi_bsg_request *bsg_req = job->request;  in beiscsi_bsg_request()
4811 	int rc = -EINVAL;  in beiscsi_bsg_request()
4815 	struct iscsi_bsg_reply *bsg_reply = job->reply;  in beiscsi_bsg_request()
4823 			    "BM_%d : HBA in error 0x%lx\n", phba->state);  in beiscsi_bsg_request()
4824 		return -ENXIO;  in beiscsi_bsg_request()
4827 	switch (bsg_req->msgcode) {  in beiscsi_bsg_request()
4829 		nonemb_cmd.va = dma_alloc_coherent(&phba->ctrl.pdev->dev,  in beiscsi_bsg_request()
4830 					job->request_payload.payload_len,  in beiscsi_bsg_request()
4836 			return -ENOMEM;  in beiscsi_bsg_request()
4838 		tag = mgmt_vendor_specific_fw_cmd(&phba->ctrl, phba, job,  in beiscsi_bsg_request()
4844 			dma_free_coherent(&phba->ctrl.pdev->dev, nonemb_cmd.size,  in beiscsi_bsg_request()
4846 			return -EAGAIN;  in beiscsi_bsg_request()
4850 					phba->ctrl.mcc_wait[tag],  in beiscsi_bsg_request()
4851 					phba->ctrl.mcc_tag_status[tag],  in beiscsi_bsg_request()
4855 		if (!test_bit(BEISCSI_HBA_ONLINE, &phba->state)) {  in beiscsi_bsg_request()
4857 				  &phba->ctrl.ptag_state[tag].tag_state);  in beiscsi_bsg_request()
4858 			dma_free_coherent(&phba->ctrl.pdev->dev, nonemb_cmd.size,  in beiscsi_bsg_request()
4860 			return -EIO;  in beiscsi_bsg_request()
4862 		extd_status = (phba->ctrl.mcc_tag_status[tag] &  in beiscsi_bsg_request()
4864 		status = phba->ctrl.mcc_tag_status[tag] & CQE_STATUS_MASK;  in beiscsi_bsg_request()
4865 		free_mcc_wrb(&phba->ctrl, tag);  in beiscsi_bsg_request()
4867 		sg_copy_from_buffer(job->reply_payload.sg_list,  in beiscsi_bsg_request()
4868 				    job->reply_payload.sg_cnt,  in beiscsi_bsg_request()
4869 				    nonemb_cmd.va, (resp->response_length  in beiscsi_bsg_request()
4871 		bsg_reply->reply_payload_rcv_len = resp->response_length;  in beiscsi_bsg_request()
4872 		bsg_reply->result = status;  in beiscsi_bsg_request()
4873 		bsg_job_done(job, bsg_reply->result,  in beiscsi_bsg_request()
4874 			     bsg_reply->reply_payload_rcv_len);  in beiscsi_bsg_request()
4875 		dma_free_coherent(&phba->ctrl.pdev->dev, nonemb_cmd.size,  in beiscsi_bsg_request()
4883 			return -EIO;  in beiscsi_bsg_request()
4892 				bsg_req->msgcode);  in beiscsi_bsg_request()
4907 	if (phba->boot_struct.boot_kset)  in beiscsi_start_boot_work()
4911 	if (test_and_set_bit(BEISCSI_HBA_BOOT_WORK, &phba->state))  in beiscsi_start_boot_work()
4914 	phba->boot_struct.retry = 3;  in beiscsi_start_boot_work()
4915 	phba->boot_struct.tag = 0;  in beiscsi_start_boot_work()
4916 	phba->boot_struct.s_handle = s_handle;  in beiscsi_start_boot_work()
4917 	phba->boot_struct.action = BEISCSI_BOOT_GET_SHANDLE;  in beiscsi_start_boot_work()
4918 	schedule_work(&phba->boot_work);  in beiscsi_start_boot_work()
4924  * Boot flag info for iscsi-utilities
4931 	struct mgmt_session_info *boot_sess = &phba->boot_struct.boot_sess;  in beiscsi_show_boot_tgt_info()
4932 	struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0];  in beiscsi_show_boot_tgt_info()
4934 	int rc = -EPERM;  in beiscsi_show_boot_tgt_info()
4939 			    (int)strlen(boot_sess->target_name),  in beiscsi_show_boot_tgt_info()
4940 			    (char *)&boot_sess->target_name);  in beiscsi_show_boot_tgt_info()
4943 		if (boot_conn->dest_ipaddr.ip_type == BEISCSI_IP_TYPE_V4)  in beiscsi_show_boot_tgt_info()
4945 				(char *)&boot_conn->dest_ipaddr.addr);  in beiscsi_show_boot_tgt_info()
4948 				(char *)&boot_conn->dest_ipaddr.addr);  in beiscsi_show_boot_tgt_info()
4951 		rc = sprintf(str, "%d\n", boot_conn->dest_port);  in beiscsi_show_boot_tgt_info()
4956 			     boot_conn->negotiated_login_options.auth_data.chap.  in beiscsi_show_boot_tgt_info()
4958 			     (char *)&boot_conn->negotiated_login_options.  in beiscsi_show_boot_tgt_info()
4963 			     boot_conn->negotiated_login_options.auth_data.chap.  in beiscsi_show_boot_tgt_info()
4965 			     (char *)&boot_conn->negotiated_login_options.  in beiscsi_show_boot_tgt_info()
4970 			     boot_conn->negotiated_login_options.auth_data.chap.  in beiscsi_show_boot_tgt_info()
4972 			     (char *)&boot_conn->negotiated_login_options.  in beiscsi_show_boot_tgt_info()
4977 			     boot_conn->negotiated_login_options.auth_data.chap.  in beiscsi_show_boot_tgt_info()
4979 			     (char *)&boot_conn->negotiated_login_options.  in beiscsi_show_boot_tgt_info()
4996 	int rc = -EPERM;  in beiscsi_show_boot_ini_info()
5001 			     phba->boot_struct.boot_sess.initiator_iscsiname);  in beiscsi_show_boot_ini_info()
5011 	int rc = -EPERM;  in beiscsi_show_boot_eth_info()
5077 	scsi_host_put(phba->shost);  in beiscsi_boot_kobj_release()
5082 	struct boot_struct *bs = &phba->boot_struct;  in beiscsi_boot_create_kset()
5085 	if (bs->boot_kset) {  in beiscsi_boot_create_kset()
5091 	bs->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no);  in beiscsi_boot_create_kset()
5092 	if (!bs->boot_kset) {  in beiscsi_boot_create_kset()
5095 		return -ENOMEM;  in beiscsi_boot_create_kset()
5099 	if (!scsi_host_get(phba->shost))  in beiscsi_boot_create_kset()
5102 	boot_kobj = iscsi_boot_create_target(bs->boot_kset, 0, phba,  in beiscsi_boot_create_kset()
5109 	if (!scsi_host_get(phba->shost))  in beiscsi_boot_create_kset()
5112 	boot_kobj = iscsi_boot_create_initiator(bs->boot_kset, 0, phba,  in beiscsi_boot_create_kset()
5119 	if (!scsi_host_get(phba->shost))  in beiscsi_boot_create_kset()
5122 	boot_kobj = iscsi_boot_create_ethernet(bs->boot_kset, 0, phba,  in beiscsi_boot_create_kset()
5132 	scsi_host_put(phba->shost);  in beiscsi_boot_create_kset()
5134 	iscsi_boot_destroy_kset(bs->boot_kset);  in beiscsi_boot_create_kset()
5135 	bs->boot_kset = NULL;  in beiscsi_boot_create_kset()
5136 	return -ENOMEM;  in beiscsi_boot_create_kset()
5143 	struct boot_struct *bs = &phba->boot_struct;  in beiscsi_boot_work()
5152 		    __func__, phba->boot_struct.action);  in beiscsi_boot_work()
5154 	switch (phba->boot_struct.action) {  in beiscsi_boot_work()
5174 		clear_bit(BEISCSI_HBA_BOOT_WORK, &phba->state);  in beiscsi_boot_work()
5178 		if (bs->retry--)  in beiscsi_boot_work()
5179 			schedule_work(&phba->boot_work);  in beiscsi_boot_work()
5181 			clear_bit(BEISCSI_HBA_BOOT_WORK, &phba->state);  in beiscsi_boot_work()
5201 	phwi_ctrlr = phba->phwi_ctrlr;  in beiscsi_eqd_update_work()
5202 	phwi_context = phwi_ctrlr->phwi_ctxt;  in beiscsi_eqd_update_work()
5204 	for (i = 0; i <= phba->num_cpus; i++) {  in beiscsi_eqd_update_work()
5205 		aic = &phba->aic_obj[i];  in beiscsi_eqd_update_work()
5206 		pbe_eq = &phwi_context->be_eq[i];  in beiscsi_eqd_update_work()
5208 		if (!aic->jiffies || time_before(now, aic->jiffies) ||  in beiscsi_eqd_update_work()
5209 		    pbe_eq->cq_count < aic->eq_prev) {  in beiscsi_eqd_update_work()
5210 			aic->jiffies = now;  in beiscsi_eqd_update_work()
5211 			aic->eq_prev = pbe_eq->cq_count;  in beiscsi_eqd_update_work()
5214 		delta = jiffies_to_msecs(now - aic->jiffies);  in beiscsi_eqd_update_work()
5215 		pps = (((u32)(pbe_eq->cq_count - aic->eq_prev) * 1000) / delta);  in beiscsi_eqd_update_work()
5223 		aic->jiffies = now;  in beiscsi_eqd_update_work()
5224 		aic->eq_prev = pbe_eq->cq_count;  in beiscsi_eqd_update_work()
5226 		if (eqd != aic->prev_eqd) {  in beiscsi_eqd_update_work()
5228 			set_eqd[num].eq_id = pbe_eq->q.id;  in beiscsi_eqd_update_work()
5229 			aic->prev_eqd = eqd;  in beiscsi_eqd_update_work()
5237 	schedule_delayed_work(&phba->eqd_update,  in beiscsi_eqd_update_work()
5252 	if (phba->ue2rp > BEISCSI_UE_DETECT_INTERVAL)  in beiscsi_hw_tpe_check()
5253 		wait = phba->ue2rp - BEISCSI_UE_DETECT_INTERVAL;  in beiscsi_hw_tpe_check()
5254 	queue_delayed_work(phba->wq, &phba->recover_port,  in beiscsi_hw_tpe_check()
5265 			      "BM_%d : port in error: %lx\n", phba->state);  in beiscsi_hw_health_check()
5267 		queue_work(phba->wq, &phba->sess_work);  in beiscsi_hw_health_check()
5270 		if (!test_bit(BEISCSI_HBA_UER_SUPP, &phba->state))  in beiscsi_hw_health_check()
5273 		phba->hw_check.function = beiscsi_hw_tpe_check;  in beiscsi_hw_health_check()
5276 	mod_timer(&phba->hw_check,  in beiscsi_hw_health_check()
5281  * beiscsi_enable_port()- Enables the disabled port.
5295 	if (test_bit(BEISCSI_HBA_ONLINE, &phba->state)) {  in beiscsi_enable_port()
5298 			      __func__, phba->state);  in beiscsi_enable_port()
5310 	/* Re-enable UER. If different TPE occurs then it is recoverable. */  in beiscsi_enable_port()
5313 	phba->shost->max_id = phba->params.cxns_per_ctrl - 1;  in beiscsi_enable_port()
5314 	phba->shost->can_queue = phba->params.ios_per_ctrl;  in beiscsi_enable_port()
5323 		init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);  in beiscsi_enable_port()
5324 		phba->ctrl.mcc_tag[i] = i + 1;  in beiscsi_enable_port()
5325 		phba->ctrl.mcc_tag_status[i + 1] = 0;  in beiscsi_enable_port()
5326 		phba->ctrl.mcc_tag_available++;  in beiscsi_enable_port()
5329 	phwi_ctrlr = phba->phwi_ctrlr;  in beiscsi_enable_port()
5330 	phwi_context = phwi_ctrlr->phwi_ctxt;  in beiscsi_enable_port()
5331 	for (i = 0; i < phba->num_cpus; i++) {  in beiscsi_enable_port()
5332 		pbe_eq = &phwi_context->be_eq[i];  in beiscsi_enable_port()
5333 		irq_poll_init(&pbe_eq->iopoll, be_iopoll_budget, be_iopoll);  in beiscsi_enable_port()
5336 	i = (phba->pcidev->msix_enabled) ? i : 0;  in beiscsi_enable_port()
5338 	pbe_eq = &phwi_context->be_eq[i];  in beiscsi_enable_port()
5339 	INIT_WORK(&pbe_eq->mcc_work, beiscsi_mcc_work);  in beiscsi_enable_port()
5349 	set_bit(BEISCSI_HBA_ONLINE, &phba->state);  in beiscsi_enable_port()
5351 		      "BM_%d : port online: 0x%lx\n", phba->state);  in beiscsi_enable_port()
5354 	schedule_delayed_work(&phba->eqd_update,  in beiscsi_enable_port()
5361 	phba->hw_check.function = beiscsi_hw_health_check;  in beiscsi_enable_port()
5362 	mod_timer(&phba->hw_check,  in beiscsi_enable_port()
5367 	for (i = 0; i < phba->num_cpus; i++) {  in beiscsi_enable_port()
5368 		pbe_eq = &phwi_context->be_eq[i];  in beiscsi_enable_port()
5369 		irq_poll_disable(&pbe_eq->iopoll);  in beiscsi_enable_port()
5374 	pci_free_irq_vectors(phba->pcidev);  in beiscsi_enable_port()
5379  * beiscsi_disable_port()- Disable port and cleanup driver resources.
5393 	if (!test_and_clear_bit(BEISCSI_HBA_ONLINE, &phba->state))  in beiscsi_disable_port()
5396 	phwi_ctrlr = phba->phwi_ctrlr;  in beiscsi_disable_port()
5397 	phwi_context = phwi_ctrlr->phwi_ctxt;  in beiscsi_disable_port()
5400 	pci_free_irq_vectors(phba->pcidev);  in beiscsi_disable_port()
5402 	for (i = 0; i < phba->num_cpus; i++) {  in beiscsi_disable_port()
5403 		pbe_eq = &phwi_context->be_eq[i];  in beiscsi_disable_port()
5404 		irq_poll_disable(&pbe_eq->iopoll);  in beiscsi_disable_port()
5406 	cancel_delayed_work_sync(&phba->eqd_update);  in beiscsi_disable_port()
5407 	cancel_work_sync(&phba->boot_work);  in beiscsi_disable_port()
5410 		pbe_eq = &phwi_context->be_eq[i];  in beiscsi_disable_port()
5411 		cancel_work_sync(&pbe_eq->mcc_work);  in beiscsi_disable_port()
5424 	 * Old sessions are gone so need to be re-established.  in beiscsi_sess_work()
5427 	iscsi_host_for_each_session(phba->shost, beiscsi_session_fail);  in beiscsi_sess_work()
5445 	set_bit(BEISCSI_HBA_PCI_ERR, &phba->state);  in beiscsi_eeh_err_detected()
5451 	timer_delete_sync(&phba->hw_check);  in beiscsi_eeh_err_detected()
5452 	cancel_delayed_work_sync(&phba->recover_port);  in beiscsi_eeh_err_detected()
5455 	iscsi_host_for_each_session(phba->shost, beiscsi_session_fail);  in beiscsi_eeh_err_detected()
5472 	if (pdev->devfn == 0)  in beiscsi_eeh_err_detected()
5535 		dev_err(&pcidev->dev,  in beiscsi_dev_probe()
5536 			"beiscsi_dev_probe - Failed to enable pci device\n");  in beiscsi_dev_probe()
5542 		dev_err(&pcidev->dev,  in beiscsi_dev_probe()
5543 			"beiscsi_dev_probe - Failed in beiscsi_hba_alloc\n");  in beiscsi_dev_probe()
5544 		ret = -ENOMEM;  in beiscsi_dev_probe()
5553 	phba->mac_addr_set = false;  in beiscsi_dev_probe()
5555 	switch (pcidev->device) {  in beiscsi_dev_probe()
5559 		phba->generation = BE_GEN2;  in beiscsi_dev_probe()
5560 		phba->iotask_fn = beiscsi_iotask;  in beiscsi_dev_probe()
5561 		dev_warn(&pcidev->dev,  in beiscsi_dev_probe()
5566 		phba->generation = BE_GEN3;  in beiscsi_dev_probe()
5567 		phba->iotask_fn = beiscsi_iotask;  in beiscsi_dev_probe()
5570 		phba->generation = BE_GEN4;  in beiscsi_dev_probe()
5571 		phba->iotask_fn = beiscsi_iotask_v2;  in beiscsi_dev_probe()
5574 		phba->generation = 0;  in beiscsi_dev_probe()
5588 	spin_lock_init(&phba->io_sgl_lock);  in beiscsi_dev_probe()
5589 	spin_lock_init(&phba->mgmt_sgl_lock);  in beiscsi_dev_probe()
5590 	spin_lock_init(&phba->async_pdu_lock);  in beiscsi_dev_probe()
5591 	ret = beiscsi_get_fw_config(&phba->ctrl, phba);  in beiscsi_dev_probe()
5597 	beiscsi_get_port_name(&phba->ctrl, phba);  in beiscsi_dev_probe()
5606 		    phba->num_cpus);  in beiscsi_dev_probe()
5608 	phba->shost->max_id = phba->params.cxns_per_ctrl;  in beiscsi_dev_probe()
5609 	phba->shost->can_queue = phba->params.ios_per_ctrl;  in beiscsi_dev_probe()
5626 		init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);  in beiscsi_dev_probe()
5627 		phba->ctrl.mcc_tag[i] = i + 1;  in beiscsi_dev_probe()
5628 		phba->ctrl.mcc_tag_status[i + 1] = 0;  in beiscsi_dev_probe()
5629 		phba->ctrl.mcc_tag_available++;  in beiscsi_dev_probe()
5630 		memset(&phba->ctrl.ptag_state[i].tag_mem_state, 0,  in beiscsi_dev_probe()
5634 	phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0;  in beiscsi_dev_probe()
5636 	phba->wq = alloc_workqueue("beiscsi_%02x_wq", WQ_MEM_RECLAIM, 1,  in beiscsi_dev_probe()
5637 				   phba->shost->host_no);  in beiscsi_dev_probe()
5638 	if (!phba->wq) {  in beiscsi_dev_probe()
5640 			    "BM_%d : beiscsi_dev_probe-"  in beiscsi_dev_probe()
5642 		ret = -ENOMEM;  in beiscsi_dev_probe()
5646 	INIT_DELAYED_WORK(&phba->eqd_update, beiscsi_eqd_update_work);  in beiscsi_dev_probe()
5648 	phwi_ctrlr = phba->phwi_ctrlr;  in beiscsi_dev_probe()
5649 	phwi_context = phwi_ctrlr->phwi_ctxt;  in beiscsi_dev_probe()
5651 	for (i = 0; i < phba->num_cpus; i++) {  in beiscsi_dev_probe()
5652 		pbe_eq = &phwi_context->be_eq[i];  in beiscsi_dev_probe()
5653 		irq_poll_init(&pbe_eq->iopoll, be_iopoll_budget, be_iopoll);  in beiscsi_dev_probe()
5656 	i = (phba->pcidev->msix_enabled) ? i : 0;  in beiscsi_dev_probe()
5658 	pbe_eq = &phwi_context->be_eq[i];  in beiscsi_dev_probe()
5659 	INIT_WORK(&pbe_eq->mcc_work, beiscsi_mcc_work);  in beiscsi_dev_probe()
5664 			    "BM_%d : beiscsi_dev_probe-"  in beiscsi_dev_probe()
5670 	ret = iscsi_host_add(phba->shost, &phba->pcidev->dev);  in beiscsi_dev_probe()
5675 	set_bit(BEISCSI_HBA_ONLINE, &phba->state);  in beiscsi_dev_probe()
5677 		      "BM_%d : port online: 0x%lx\n", phba->state);  in beiscsi_dev_probe()
5679 	INIT_WORK(&phba->boot_work, beiscsi_boot_work);  in beiscsi_dev_probe()
5688 		set_bit(BEISCSI_HBA_BOOT_FOUND, &phba->state);  in beiscsi_dev_probe()
5692 	schedule_delayed_work(&phba->eqd_update,  in beiscsi_dev_probe()
5695 	INIT_WORK(&phba->sess_work, beiscsi_sess_work);  in beiscsi_dev_probe()
5696 	INIT_DELAYED_WORK(&phba->recover_port, beiscsi_recover_port);  in beiscsi_dev_probe()
5701 	timer_setup(&phba->hw_check, beiscsi_hw_health_check, 0);  in beiscsi_dev_probe()
5702 	mod_timer(&phba->hw_check,  in beiscsi_dev_probe()
5705 		    "\n\n\n BM_%d : SUCCESS - DRIVER LOADED\n\n\n");  in beiscsi_dev_probe()
5712 	for (i = 0; i < phba->num_cpus; i++) {  in beiscsi_dev_probe()
5713 		pbe_eq = &phwi_context->be_eq[i];  in beiscsi_dev_probe()
5714 		irq_poll_disable(&pbe_eq->iopoll);  in beiscsi_dev_probe()
5716 	destroy_workqueue(phba->wq);  in beiscsi_dev_probe()
5722 	dma_free_coherent(&phba->pcidev->dev,  in beiscsi_dev_probe()
5723 			    phba->ctrl.mbox_mem_alloced.size,  in beiscsi_dev_probe()
5724 			    phba->ctrl.mbox_mem_alloced.va,  in beiscsi_dev_probe()
5725 			    phba->ctrl.mbox_mem_alloced.dma);  in beiscsi_dev_probe()
5728 	pci_disable_msix(phba->pcidev);  in beiscsi_dev_probe()
5729 	pci_dev_put(phba->pcidev);  in beiscsi_dev_probe()
5730 	iscsi_host_free(phba->shost);  in beiscsi_dev_probe()
5744 		dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n");  in beiscsi_remove()
5749 	timer_delete_sync(&phba->hw_check);  in beiscsi_remove()
5750 	cancel_delayed_work_sync(&phba->recover_port);  in beiscsi_remove()
5751 	cancel_work_sync(&phba->sess_work);  in beiscsi_remove()
5754 	iscsi_host_remove(phba->shost, false);  in beiscsi_remove()
5758 	iscsi_boot_destroy_kset(phba->boot_struct.boot_kset);  in beiscsi_remove()
5761 	destroy_workqueue(phba->wq);  in beiscsi_remove()
5766 	dma_free_coherent(&phba->pcidev->dev,  in beiscsi_remove()
5767 			    phba->ctrl.mbox_mem_alloced.size,  in beiscsi_remove()
5768 			    phba->ctrl.mbox_mem_alloced.va,  in beiscsi_remove()
5769 			    phba->ctrl.mbox_mem_alloced.dma);  in beiscsi_remove()
5771 	pci_dev_put(phba->pcidev);  in beiscsi_remove()
5772 	iscsi_host_free(phba->shost);  in beiscsi_remove()
5835 		       "beiscsi_module_init - Unable to  register beiscsi transport.\n");  in beiscsi_module_init()
5836 		return -ENOMEM;  in beiscsi_module_init()
5844 		       "beiscsi_module_init - Unable to  register beiscsi pci driver.\n");  in beiscsi_module_init()