1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2015 - 2021 Intel Corporation */ 3 #include "main.h" 4 5 MODULE_ALIAS("i40iw"); 6 MODULE_DESCRIPTION("Intel(R) Ethernet Protocol Driver for RDMA"); 7 MODULE_LICENSE("Dual BSD/GPL"); 8 9 static struct notifier_block irdma_inetaddr_notifier = { 10 .notifier_call = irdma_inetaddr_event 11 }; 12 13 static struct notifier_block irdma_inetaddr6_notifier = { 14 .notifier_call = irdma_inet6addr_event 15 }; 16 17 static struct notifier_block irdma_net_notifier = { 18 .notifier_call = irdma_net_event 19 }; 20 21 static struct notifier_block irdma_netdevice_notifier = { 22 .notifier_call = irdma_netdevice_event 23 }; 24 25 static void irdma_register_notifiers(void) 26 { 27 register_inetaddr_notifier(&irdma_inetaddr_notifier); 28 register_inet6addr_notifier(&irdma_inetaddr6_notifier); 29 register_netevent_notifier(&irdma_net_notifier); 30 register_netdevice_notifier(&irdma_netdevice_notifier); 31 } 32 33 static void irdma_unregister_notifiers(void) 34 { 35 unregister_netevent_notifier(&irdma_net_notifier); 36 unregister_inetaddr_notifier(&irdma_inetaddr_notifier); 37 unregister_inet6addr_notifier(&irdma_inetaddr6_notifier); 38 unregister_netdevice_notifier(&irdma_netdevice_notifier); 39 } 40 41 static void irdma_prep_tc_change(struct irdma_device *iwdev) 42 { 43 iwdev->vsi.tc_change_pending = true; 44 irdma_sc_suspend_resume_qps(&iwdev->vsi, IRDMA_OP_SUSPEND); 45 46 /* Wait for all qp's to suspend */ 47 wait_event_timeout(iwdev->suspend_wq, 48 !atomic_read(&iwdev->vsi.qp_suspend_reqs), 49 msecs_to_jiffies(IRDMA_EVENT_TIMEOUT_MS)); 50 irdma_ws_reset(&iwdev->vsi); 51 } 52 53 static void irdma_log_invalid_mtu(u16 mtu, struct irdma_sc_dev *dev) 54 { 55 if (mtu < IRDMA_MIN_MTU_IPV4) 56 ibdev_warn(to_ibdev(dev), "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 576 for IPv4\n", mtu); 57 else if (mtu < IRDMA_MIN_MTU_IPV6) 58 ibdev_warn(to_ibdev(dev), "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 1280 for IPv6\\n", mtu); 59 } 60 61 static void irdma_fill_qos_info(struct irdma_l2params *l2params, 62 struct iidc_rdma_qos_params *qos_info) 63 { 64 int i; 65 66 l2params->num_tc = qos_info->num_tc; 67 l2params->vsi_prio_type = qos_info->vport_priority_type; 68 l2params->vsi_rel_bw = qos_info->vport_relative_bw; 69 for (i = 0; i < l2params->num_tc; i++) { 70 l2params->tc_info[i].egress_virt_up = 71 qos_info->tc_info[i].egress_virt_up; 72 l2params->tc_info[i].ingress_virt_up = 73 qos_info->tc_info[i].ingress_virt_up; 74 l2params->tc_info[i].prio_type = qos_info->tc_info[i].prio_type; 75 l2params->tc_info[i].rel_bw = qos_info->tc_info[i].rel_bw; 76 l2params->tc_info[i].tc_ctx = qos_info->tc_info[i].tc_ctx; 77 } 78 for (i = 0; i < IIDC_MAX_USER_PRIORITY; i++) 79 l2params->up2tc[i] = qos_info->up2tc[i]; 80 if (qos_info->pfc_mode == IIDC_DSCP_PFC_MODE) { 81 l2params->dscp_mode = true; 82 memcpy(l2params->dscp_map, qos_info->dscp_map, sizeof(l2params->dscp_map)); 83 } 84 } 85 86 static void irdma_iidc_event_handler(struct iidc_rdma_core_dev_info *cdev_info, 87 struct iidc_rdma_event *event) 88 { 89 struct irdma_device *iwdev = dev_get_drvdata(&cdev_info->adev->dev); 90 struct irdma_l2params l2params = {}; 91 92 if (*event->type & BIT(IIDC_RDMA_EVENT_AFTER_MTU_CHANGE)) { 93 ibdev_dbg(&iwdev->ibdev, "CLNT: new MTU = %d\n", iwdev->netdev->mtu); 94 if (iwdev->vsi.mtu != iwdev->netdev->mtu) { 95 l2params.mtu = iwdev->netdev->mtu; 96 l2params.mtu_changed = true; 97 irdma_log_invalid_mtu(l2params.mtu, &iwdev->rf->sc_dev); 98 irdma_change_l2params(&iwdev->vsi, &l2params); 99 } 100 } else if (*event->type & BIT(IIDC_RDMA_EVENT_BEFORE_TC_CHANGE)) { 101 if (iwdev->vsi.tc_change_pending) 102 return; 103 104 irdma_prep_tc_change(iwdev); 105 } else if (*event->type & BIT(IIDC_RDMA_EVENT_AFTER_TC_CHANGE)) { 106 struct iidc_rdma_priv_dev_info *iidc_priv = cdev_info->iidc_priv; 107 108 if (!iwdev->vsi.tc_change_pending) 109 return; 110 111 l2params.tc_changed = true; 112 ibdev_dbg(&iwdev->ibdev, "CLNT: TC Change\n"); 113 114 irdma_fill_qos_info(&l2params, &iidc_priv->qos_info); 115 if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY) 116 iwdev->dcb_vlan_mode = 117 l2params.num_tc > 1 && !l2params.dscp_mode; 118 irdma_change_l2params(&iwdev->vsi, &l2params); 119 } else if (*event->type & BIT(IIDC_RDMA_EVENT_CRIT_ERR)) { 120 ibdev_warn(&iwdev->ibdev, "ICE OICR event notification: oicr = 0x%08x\n", 121 event->reg); 122 if (event->reg & IRDMAPFINT_OICR_PE_CRITERR_M) { 123 u32 pe_criterr; 124 125 pe_criterr = readl(iwdev->rf->sc_dev.hw_regs[IRDMA_GLPE_CRITERR]); 126 #define IRDMA_Q1_RESOURCE_ERR 0x0001024d 127 if (pe_criterr != IRDMA_Q1_RESOURCE_ERR) { 128 ibdev_err(&iwdev->ibdev, "critical PE Error, GLPE_CRITERR=0x%08x\n", 129 pe_criterr); 130 iwdev->rf->reset = true; 131 } else { 132 ibdev_warn(&iwdev->ibdev, "Q1 Resource Check\n"); 133 } 134 } 135 if (event->reg & IRDMAPFINT_OICR_HMC_ERR_M) { 136 ibdev_err(&iwdev->ibdev, "HMC Error\n"); 137 iwdev->rf->reset = true; 138 } 139 if (event->reg & IRDMAPFINT_OICR_PE_PUSH_M) { 140 ibdev_err(&iwdev->ibdev, "PE Push Error\n"); 141 iwdev->rf->reset = true; 142 } 143 if (iwdev->rf->reset) 144 iwdev->rf->gen_ops.request_reset(iwdev->rf); 145 } 146 } 147 148 /** 149 * irdma_request_reset - Request a reset 150 * @rf: RDMA PCI function 151 */ 152 static void irdma_request_reset(struct irdma_pci_f *rf) 153 { 154 ibdev_warn(&rf->iwdev->ibdev, "Requesting a reset\n"); 155 ice_rdma_request_reset(rf->cdev, IIDC_FUNC_RESET); 156 } 157 158 /** 159 * irdma_lan_register_qset - Register qset with LAN driver 160 * @vsi: vsi structure 161 * @tc_node: Traffic class node 162 */ 163 static int irdma_lan_register_qset(struct irdma_sc_vsi *vsi, 164 struct irdma_ws_node *tc_node) 165 { 166 struct irdma_device *iwdev = vsi->back_vsi; 167 struct iidc_rdma_core_dev_info *cdev_info; 168 struct iidc_rdma_qset_params qset = {}; 169 int ret; 170 171 cdev_info = iwdev->rf->cdev; 172 qset.qs_handle = tc_node->qs_handle; 173 qset.tc = tc_node->traffic_class; 174 qset.vport_id = vsi->vsi_idx; 175 ret = ice_add_rdma_qset(cdev_info, &qset); 176 if (ret) { 177 ibdev_dbg(&iwdev->ibdev, "WS: LAN alloc_res for rdma qset failed.\n"); 178 return ret; 179 } 180 181 tc_node->l2_sched_node_id = qset.teid; 182 vsi->qos[tc_node->user_pri].l2_sched_node_id = qset.teid; 183 184 return 0; 185 } 186 187 /** 188 * irdma_lan_unregister_qset - Unregister qset with LAN driver 189 * @vsi: vsi structure 190 * @tc_node: Traffic class node 191 */ 192 static void irdma_lan_unregister_qset(struct irdma_sc_vsi *vsi, 193 struct irdma_ws_node *tc_node) 194 { 195 struct irdma_device *iwdev = vsi->back_vsi; 196 struct iidc_rdma_core_dev_info *cdev_info; 197 struct iidc_rdma_qset_params qset = {}; 198 199 cdev_info = iwdev->rf->cdev; 200 qset.qs_handle = tc_node->qs_handle; 201 qset.tc = tc_node->traffic_class; 202 qset.vport_id = vsi->vsi_idx; 203 qset.teid = tc_node->l2_sched_node_id; 204 205 if (ice_del_rdma_qset(cdev_info, &qset)) 206 ibdev_dbg(&iwdev->ibdev, "WS: LAN free_res for rdma qset failed.\n"); 207 } 208 209 static int irdma_init_interrupts(struct irdma_pci_f *rf, struct iidc_rdma_core_dev_info *cdev) 210 { 211 int i; 212 213 rf->msix_count = num_online_cpus() + IRDMA_NUM_AEQ_MSIX; 214 rf->msix_entries = kcalloc(rf->msix_count, sizeof(*rf->msix_entries), 215 GFP_KERNEL); 216 if (!rf->msix_entries) 217 return -ENOMEM; 218 219 for (i = 0; i < rf->msix_count; i++) 220 if (ice_alloc_rdma_qvector(cdev, &rf->msix_entries[i])) 221 break; 222 223 if (i < IRDMA_MIN_MSIX) { 224 while (--i >= 0) 225 ice_free_rdma_qvector(cdev, &rf->msix_entries[i]); 226 227 kfree(rf->msix_entries); 228 return -ENOMEM; 229 } 230 231 rf->msix_count = i; 232 233 return 0; 234 } 235 236 static void irdma_deinit_interrupts(struct irdma_pci_f *rf, struct iidc_rdma_core_dev_info *cdev) 237 { 238 int i; 239 240 for (i = 0; i < rf->msix_count; i++) 241 ice_free_rdma_qvector(cdev, &rf->msix_entries[i]); 242 243 kfree(rf->msix_entries); 244 } 245 246 static void irdma_remove(struct auxiliary_device *aux_dev) 247 { 248 struct irdma_device *iwdev = auxiliary_get_drvdata(aux_dev); 249 struct iidc_rdma_core_auxiliary_dev *iidc_adev; 250 struct iidc_rdma_core_dev_info *cdev_info; 251 252 iidc_adev = container_of(aux_dev, struct iidc_rdma_core_auxiliary_dev, adev); 253 cdev_info = iidc_adev->cdev_info; 254 255 ice_rdma_update_vsi_filter(cdev_info, iwdev->vsi_num, false); 256 irdma_ib_unregister_device(iwdev); 257 irdma_deinit_interrupts(iwdev->rf, cdev_info); 258 259 kfree(iwdev->rf); 260 261 pr_debug("INIT: Gen2 PF[%d] device remove success\n", PCI_FUNC(cdev_info->pdev->devfn)); 262 } 263 264 static void irdma_fill_device_info(struct irdma_device *iwdev, 265 struct iidc_rdma_core_dev_info *cdev_info) 266 { 267 struct iidc_rdma_priv_dev_info *iidc_priv = cdev_info->iidc_priv; 268 struct irdma_pci_f *rf = iwdev->rf; 269 270 rf->sc_dev.hw = &rf->hw; 271 rf->iwdev = iwdev; 272 rf->cdev = cdev_info; 273 rf->hw.hw_addr = iidc_priv->hw_addr; 274 rf->pcidev = cdev_info->pdev; 275 rf->hw.device = &rf->pcidev->dev; 276 rf->pf_id = iidc_priv->pf_id; 277 rf->gen_ops.register_qset = irdma_lan_register_qset; 278 rf->gen_ops.unregister_qset = irdma_lan_unregister_qset; 279 280 rf->default_vsi.vsi_idx = iidc_priv->vport_id; 281 rf->protocol_used = 282 cdev_info->rdma_protocol == IIDC_RDMA_PROTOCOL_ROCEV2 ? 283 IRDMA_ROCE_PROTOCOL_ONLY : IRDMA_IWARP_PROTOCOL_ONLY; 284 rf->rdma_ver = IRDMA_GEN_2; 285 rf->rsrc_profile = IRDMA_HMC_PROFILE_DEFAULT; 286 rf->rst_to = IRDMA_RST_TIMEOUT_HZ; 287 rf->gen_ops.request_reset = irdma_request_reset; 288 rf->limits_sel = 7; 289 rf->iwdev = iwdev; 290 291 mutex_init(&iwdev->ah_tbl_lock); 292 293 iwdev->netdev = iidc_priv->netdev; 294 iwdev->vsi_num = iidc_priv->vport_id; 295 iwdev->init_state = INITIAL_STATE; 296 iwdev->roce_cwnd = IRDMA_ROCE_CWND_DEFAULT; 297 iwdev->roce_ackcreds = IRDMA_ROCE_ACKCREDS_DEFAULT; 298 iwdev->rcv_wnd = IRDMA_CM_DEFAULT_RCV_WND_SCALED; 299 iwdev->rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE; 300 if (rf->protocol_used == IRDMA_ROCE_PROTOCOL_ONLY) 301 iwdev->roce_mode = true; 302 } 303 304 static int irdma_probe(struct auxiliary_device *aux_dev, const struct auxiliary_device_id *id) 305 { 306 struct iidc_rdma_core_auxiliary_dev *iidc_adev; 307 struct iidc_rdma_core_dev_info *cdev_info; 308 struct iidc_rdma_priv_dev_info *iidc_priv; 309 struct irdma_l2params l2params = {}; 310 struct irdma_device *iwdev; 311 struct irdma_pci_f *rf; 312 int err; 313 314 iidc_adev = container_of(aux_dev, struct iidc_rdma_core_auxiliary_dev, adev); 315 cdev_info = iidc_adev->cdev_info; 316 iidc_priv = cdev_info->iidc_priv; 317 318 iwdev = ib_alloc_device(irdma_device, ibdev); 319 if (!iwdev) 320 return -ENOMEM; 321 iwdev->rf = kzalloc(sizeof(*rf), GFP_KERNEL); 322 if (!iwdev->rf) { 323 ib_dealloc_device(&iwdev->ibdev); 324 return -ENOMEM; 325 } 326 327 irdma_fill_device_info(iwdev, cdev_info); 328 rf = iwdev->rf; 329 330 err = irdma_init_interrupts(rf, cdev_info); 331 if (err) 332 goto err_init_interrupts; 333 334 err = irdma_ctrl_init_hw(rf); 335 if (err) 336 goto err_ctrl_init; 337 338 l2params.mtu = iwdev->netdev->mtu; 339 irdma_fill_qos_info(&l2params, &iidc_priv->qos_info); 340 if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY) 341 iwdev->dcb_vlan_mode = l2params.num_tc > 1 && !l2params.dscp_mode; 342 343 err = irdma_rt_init_hw(iwdev, &l2params); 344 if (err) 345 goto err_rt_init; 346 347 err = irdma_ib_register_device(iwdev); 348 if (err) 349 goto err_ibreg; 350 351 ice_rdma_update_vsi_filter(cdev_info, iwdev->vsi_num, true); 352 353 ibdev_dbg(&iwdev->ibdev, "INIT: Gen2 PF[%d] device probe success\n", PCI_FUNC(rf->pcidev->devfn)); 354 auxiliary_set_drvdata(aux_dev, iwdev); 355 356 return 0; 357 358 err_ibreg: 359 irdma_rt_deinit_hw(iwdev); 360 err_rt_init: 361 irdma_ctrl_deinit_hw(rf); 362 err_ctrl_init: 363 irdma_deinit_interrupts(rf, cdev_info); 364 err_init_interrupts: 365 kfree(iwdev->rf); 366 ib_dealloc_device(&iwdev->ibdev); 367 368 return err; 369 } 370 371 static const struct auxiliary_device_id irdma_auxiliary_id_table[] = { 372 {.name = "ice.iwarp", }, 373 {.name = "ice.roce", }, 374 {}, 375 }; 376 377 MODULE_DEVICE_TABLE(auxiliary, irdma_auxiliary_id_table); 378 379 static struct iidc_rdma_core_auxiliary_drv irdma_auxiliary_drv = { 380 .adrv = { 381 .id_table = irdma_auxiliary_id_table, 382 .probe = irdma_probe, 383 .remove = irdma_remove, 384 }, 385 .event_handler = irdma_iidc_event_handler, 386 }; 387 388 static int __init irdma_init_module(void) 389 { 390 int ret; 391 392 ret = auxiliary_driver_register(&i40iw_auxiliary_drv); 393 if (ret) { 394 pr_err("Failed i40iw(gen_1) auxiliary_driver_register() ret=%d\n", 395 ret); 396 return ret; 397 } 398 399 ret = auxiliary_driver_register(&irdma_auxiliary_drv.adrv); 400 if (ret) { 401 auxiliary_driver_unregister(&i40iw_auxiliary_drv); 402 pr_err("Failed irdma auxiliary_driver_register() ret=%d\n", 403 ret); 404 return ret; 405 } 406 407 irdma_register_notifiers(); 408 409 return 0; 410 } 411 412 static void __exit irdma_exit_module(void) 413 { 414 irdma_unregister_notifiers(); 415 auxiliary_driver_unregister(&irdma_auxiliary_drv.adrv); 416 auxiliary_driver_unregister(&i40iw_auxiliary_drv); 417 } 418 419 module_init(irdma_init_module); 420 module_exit(irdma_exit_module); 421